feat: add HMAC challenge/response authentication for relay

- Implement HMAC‑SHA256 challenge/response for relay registration and agent connections, rejecting unauthenticated peers.
- Add deadline‑based timeout to protect against hanging handshakes and replay attacks.
- Refactor secure handshake to use per‑connection random challenge instead of a static string.
- Update README with detailed security guidance and replay protection notes.
- Introduce extensive unit tests for relay authentication and secure connection round‑trip.
- Bump Go version to 1.25 and clean up .gitignore.

Author: devnullvoid

.gitignore

@@ -6,6 +6,3 @@
 
 # ignore vscode workspace files
 .vscode/
-
-# ignore test files 
-*_test.go

AGENTS.md

@@ -0,0 +1,5 @@
+# Agent Activity Log
+
+| Date (UTC) | Agent | Summary |
+| --- | --- | --- |
+| 2025-11-03 | Code | Reviewed tunnel handshake, added HMAC challenge/response (including relay auth), expanded regression tests, refreshed security guidance, and updated Go module metadata. |

README.md

@@ -156,9 +156,11 @@ tunnel_addr: proxy.host:9000
 
 ## Security
 
-- Uses AES-CTR with separate IVs for encrypt/decrypt.
-- HMAC-SHA256 handshake to authenticate peers.
+- Uses AES-CTR with separate IVs for encrypt/decrypt and fresh IVs per connection.
+- Performs an HMAC-SHA256 challenge/response handshake to authenticate peers and block replay attempts.
 - High-entropy ciphertext; no plaintext leaks over the tunnel.
+- Provide a high-entropy shared secret (at least 32 random bytes) and rotate it periodically.
+- Relay registrations and agent attachments complete a pre-tunnel HMAC challenge so only trusted peers can bind to the relay.
 
 ## License
 

go.mod

@@ -1,6 +1,6 @@
 module github.com/lonepie/reverse-soxy
 
-go 1.24
+go 1.25
 
 require (
 	github.com/fatih/color v1.13.0

internal/proxy/agent.go

@@ -53,6 +53,12 @@ func RunAgentRelay(relayAddr, secret string, maxRetries int) {
 			time.Sleep(5 * time.Second)
 			continue
 		}
+		if err := answerRelayChallenge(rawConn, secret); err != nil {
+			rawConn.Close()
+			logger.Error("AgentRelay auth failed: %v", err)
+			time.Sleep(5 * time.Second)
+			continue
+		}
 		// secure handshake
 		secureConn, err := NewSecureClientConn(rawConn, secret)
 		if err != nil {

internal/proxy/proxy.go

@@ -336,6 +336,9 @@ func RunProxyRelay(relayAddr string, socksAddr string, secret string) {
 	if _, err := rawConn.Write([]byte("REGISTER")); err != nil {
 		logger.Fatalf("Register header send error: %v", err)
 	}
+	if err := answerRelayChallenge(rawConn, secret); err != nil {
+		logger.Fatalf("Relay auth failed: %v", err)
+	}
 	// secure handshake as server
 	secureConn, err := NewSecureServerConn(rawConn, secret)
 	if err != nil {

internal/proxy/relay.go

@@ -1,11 +1,15 @@
 package proxy
 
 import (
+	"crypto/hmac"
+	"crypto/rand"
+	"crypto/sha256"
 	"fmt"
 	"io"
 	"net"
 	"strings"
 	"sync"
+	"time"
 
 	"github.com/lonepie/reverse-soxy/internal/logger"
 )
@@ -43,14 +47,79 @@ func handleRelayConn(conn net.Conn, secret string) {
 	header := strings.TrimSpace(string(hdr))
 	switch header {
 	case "REGISTER":
+		if err := verifyRelayPeer(conn, secret); err != nil {
+			logger.Error("Relay REGISTER auth failed: %v", err)
+			return
+		}
 		registerProxy(conn)
 	case "AGENT":
+		if err := verifyRelayPeer(conn, secret); err != nil {
+			logger.Error("Relay AGENT auth failed: %v", err)
+			return
+		}
 		handleAgent(conn)
 	default:
 		logger.Error("Unknown relay header: %s", header)
 	}
 }
 
+const relayAuthTimeout = 10 * time.Second
+
+func verifyRelayPeer(conn net.Conn, secret string) error {
+	if secret == "" {
+		return errAuthFailed
+	}
+	if err := conn.SetDeadline(time.Now().Add(relayAuthTimeout)); err != nil {
+		return err
+	}
+	defer conn.SetDeadline(time.Time{})
+
+	challenge := make([]byte, handshakeChallengeSize)
+	if _, err := rand.Read(challenge); err != nil {
+		return err
+	}
+	if _, err := conn.Write(challenge); err != nil {
+		return err
+	}
+
+	expected := hmac.New(sha256.New, deriveKey(secret))
+	expected.Write(challenge)
+
+	response := make([]byte, expected.Size())
+	if _, err := io.ReadFull(conn, response); err != nil {
+		return err
+	}
+	if !hmac.Equal(response, expected.Sum(nil)) {
+		return errAuthFailed
+	}
+
+	return nil
+}
+
+func answerRelayChallenge(conn net.Conn, secret string) error {
+	if secret == "" {
+		return errAuthFailed
+	}
+	if err := conn.SetDeadline(time.Now().Add(relayAuthTimeout)); err != nil {
+		return err
+	}
+	defer conn.SetDeadline(time.Time{})
+
+	challenge := make([]byte, handshakeChallengeSize)
+	if _, err := io.ReadFull(conn, challenge); err != nil {
+		return err
+	}
+
+	mac := hmac.New(sha256.New, deriveKey(secret))
+	mac.Write(challenge)
+
+	if _, err := conn.Write(mac.Sum(nil)); err != nil {
+		return err
+	}
+
+	return nil
+}
+
 func registerProxy(conn net.Conn) {
 	regMu.Lock()
 	registry = append(registry, conn)
@@ -117,6 +186,9 @@ func RunRegister(relayAddr, secret string) {
 	}
 	defer conn.Close()
 	conn.Write([]byte("REGISTER"))
+	if err := answerRelayChallenge(conn, secret); err != nil {
+		logger.Fatalf("Register auth failed: %v", err)
+	}
 	logger.Info("Registered with relay %s", relayAddr)
 	select {} // hold open
 }

internal/proxy/relay_test.go

@@ -0,0 +1,102 @@
+package proxy
+
+import (
+	"errors"
+	"net"
+	"testing"
+	"time"
+)
+
+func TestRegisterProxy(t *testing.T) {
+	// reset registry
+	regMu.Lock()
+	registry = nil
+	regMu.Unlock()
+	// simulate a proxy registration
+	c1, c2 := net.Pipe()
+	// run registerProxy in goroutine to avoid blocking
+	go registerProxy(c1)
+	// allow goroutine to append
+	time.Sleep(10 * time.Millisecond)
+	regMu.Lock()
+	if len(registry) != 1 {
+		t.Fatalf("expected registry length 1, got %d", len(registry))
+	}
+	if registry[0] != c1 {
+		t.Fatalf("expected registry[0] to be c1")
+	}
+	// cleanup
+	registry = nil
+	regMu.Unlock()
+	c1.Close()
+	c2.Close()
+}
+
+func TestHandleRelayConnRegister(t *testing.T) {
+	// reset registry
+	regMu.Lock()
+	registry = nil
+	regMu.Unlock()
+	client, server := net.Pipe()
+	// start handler before writing to avoid blocking
+	go handleRelayConn(server, "secret")
+	// write REGISTER header (8 bytes)
+	if _, err := client.Write([]byte("REGISTER")); err != nil {
+		t.Fatalf("failed to write header: %v", err)
+	}
+	if err := answerRelayChallenge(client, "secret"); err != nil {
+		t.Fatalf("challenge response failed: %v", err)
+	}
+	// allow time for processing
+	time.Sleep(10 * time.Millisecond)
+	regMu.Lock()
+	if len(registry) != 1 {
+		t.Fatalf("expected registry length 1 after handleRelayConn, got %d", len(registry))
+	}
+	if registry[0] != server {
+		t.Fatalf("expected registered conn to be server")
+	}
+	t.Log("Registry registered successfully")
+	regMu.Unlock()
+	client.Close()
+	server.Close()
+}
+
+func TestVerifyRelayPeerRejectsWrongSecret(t *testing.T) {
+	server, client := net.Pipe()
+	defer server.Close()
+	defer client.Close()
+
+	errCh := make(chan error, 1)
+	go func() {
+		errCh <- verifyRelayPeer(server, "secret")
+	}()
+
+	if err := answerRelayChallenge(client, "wrong"); err != nil {
+		t.Fatalf("answerRelayChallenge failed: %v", err)
+	}
+
+	err := <-errCh
+	if !errors.Is(err, errAuthFailed) {
+		t.Fatalf("expected errAuthFailed, got %v", err)
+	}
+}
+
+func TestRelayChallengeSuccess(t *testing.T) {
+	server, client := net.Pipe()
+	defer server.Close()
+	defer client.Close()
+
+	errCh := make(chan error, 1)
+	go func() {
+		errCh <- verifyRelayPeer(server, "secret")
+	}()
+
+	if err := answerRelayChallenge(client, "secret"); err != nil {
+		t.Fatalf("answerRelayChallenge failed: %v", err)
+	}
+
+	if err := <-errCh; err != nil {
+		t.Fatalf("verifyRelayPeer returned error: %v", err)
+	}
+}

internal/proxy/secure.go

@@ -14,6 +14,9 @@ import (
 
 var errAuthFailed = errors.New("authentication failed")
 
+const handshakeTimeout = 2 * time.Minute
+const handshakeChallengeSize = sha256.Size
+
 func deriveKey(secret string) []byte {
 	h := sha256.Sum256([]byte(secret))
 	return h[:]
@@ -36,9 +39,15 @@ func (s *secureConn) Write(p []byte) (int, error) {
 // NewSecureClientConn performs HMAC auth and AES-CTR encryption on a client-side tunnel connection
 func NewSecureClientConn(conn net.Conn, secret string) (net.Conn, error) {
 	key := deriveKey(secret)
-	// send HMAC handshake
+	conn.SetDeadline(time.Now().Add(handshakeTimeout))
+	defer conn.SetDeadline(time.Time{})
+	// read server challenge and respond with HMAC to prevent replay
+	challenge := make([]byte, handshakeChallengeSize)
+	if _, err := io.ReadFull(conn, challenge); err != nil {
+		return nil, err
+	}
 	mac := hmac.New(sha256.New, key)
-	mac.Write([]byte("handshake"))
+	mac.Write(challenge)
 	if _, err := conn.Write(mac.Sum(nil)); err != nil {
 		return nil, err
 	}
@@ -62,8 +71,6 @@ func NewSecureClientConn(conn net.Conn, secret string) (net.Conn, error) {
 	if _, err := conn.Write(ivDec); err != nil {
 		return nil, err
 	}
-	// clear deadlines after handshake
-	conn.SetDeadline(time.Time{})
 	// create independent CTR streams
 	enc := cipher.NewCTR(block, ivEnc)
 	dec := cipher.NewCTR(block, ivDec)
@@ -73,15 +80,23 @@ func NewSecureClientConn(conn net.Conn, secret string) (net.Conn, error) {
 // NewSecureServerConn performs HMAC auth and AES-CTR encryption on a server-side tunnel connection
 func NewSecureServerConn(conn net.Conn, secret string) (net.Conn, error) {
 	key := deriveKey(secret)
-	// read HMAC handshake
-	expectedMacLen := sha256.Size
-	bufMac := make([]byte, expectedMacLen)
-	if _, err := io.ReadFull(conn, bufMac); err != nil {
+	conn.SetDeadline(time.Now().Add(handshakeTimeout))
+	defer conn.SetDeadline(time.Time{})
+	// send challenge to defend against replayed handshakes
+	challenge := make([]byte, handshakeChallengeSize)
+	if _, err := rand.Read(challenge); err != nil {
+		return nil, err
+	}
+	if _, err := conn.Write(challenge); err != nil {
 		return nil, err
 	}
 	mac := hmac.New(sha256.New, key)
-	mac.Write([]byte("handshake"))
-	if !hmac.Equal(bufMac, mac.Sum(nil)) {
+	mac.Write(challenge)
+	recvMac := make([]byte, mac.Size())
+	if _, err := io.ReadFull(conn, recvMac); err != nil {
+		return nil, err
+	}
+	if !hmac.Equal(recvMac, mac.Sum(nil)) {
 		return nil, errAuthFailed
 	}
 	// read IVs from client