fix: patch inconsistency and minor QUIC bug

Signed-off-by: Gaukas Wang <[email protected]>

fingerprint_hash.go

@@ -76,7 +76,7 @@ func (gci *GatheredClientInitials) calcNumericID() uint64 {
 	// merge, deduplicate, and sort all frames from all packets
 	var allFrameIDs []uint8
 	for _, p := range gci.Packets {
-		allFrameIDs = append(allFrameIDs, p.Frames.FrameTypesUint8()...)
+		allFrameIDs = append(allFrameIDs, p.frames.FrameTypesUint8()...)
 	}
 	dedupAllFrameIDs := utils.DedupIntArr(allFrameIDs)
 	sort.Slice(dedupAllFrameIDs, func(i, j int) bool {

modcaddy/app/reservoir.go

@@ -2,6 +2,7 @@ package app
 
 import (
 	"errors"
+	"sync"
 	"time"
 
 	"github.com/caddyserver/caddy/v2"
@@ -34,8 +35,9 @@ type Reservoir struct {
 	// sufficient.
 	CleanInterval caddy.Duration `json:"clean_interval,omitempty"`
 
-	tlsFingerprinter  *clienthellod.TLSFingerprinter
-	quicFingerprinter *clienthellod.QUICFingerprinter
+	tlsFingerprinter        *clienthellod.TLSFingerprinter
+	quicFingerprinter       *clienthellod.QUICFingerprinter
+	mapLastQUICVisitorPerIP *sync.Map // sometimes even when a complete QUIC handshake is done, client decide to connect using HTTP/2
 
 	logger *zap.Logger
 }
@@ -66,6 +68,26 @@ func (r *Reservoir) QUICFingerprinter() *clienthellod.QUICFingerprinter { // ski
 	return r.quicFingerprinter
 }
 
+// NewQUICVisitor updates the map entry for the given IP address.
+func (r *Reservoir) NewQUICVisitor(ip, fullKey string) { // skipcq: GO-W1029
+	r.mapLastQUICVisitorPerIP.Store(ip, fullKey)
+
+	// delete it after validfor if not updated
+	time.AfterFunc(time.Duration(r.ValidFor), func() {
+		r.mapLastQUICVisitorPerIP.CompareAndDelete(ip, fullKey)
+	})
+}
+
+// GetLastQUICVisitor returns the last QUIC visitor for the given IP address.
+func (r *Reservoir) GetLastQUICVisitor(ip string) (string, bool) { // skipcq: GO-W1029
+	if v, ok := r.mapLastQUICVisitorPerIP.Load(ip); ok {
+		if fullKey, ok := v.(string); ok {
+			return fullKey, true
+		}
+	}
+	return "", false
+}
+
 // Start implements Start() of caddy.App.
 func (r *Reservoir) Start() error { // skipcq: GO-W1029
 	if r.ValidFor <= 0 {
@@ -93,6 +115,7 @@ func (r *Reservoir) Provision(ctx caddy.Context) error { // skipcq: GO-W1029
 	r.logger = ctx.Logger(r)
 	r.tlsFingerprinter = clienthellod.NewTLSFingerprinterWithTimeout(time.Duration(r.ValidFor))
 	r.quicFingerprinter = clienthellod.NewQUICFingerprinterWithTimeout(time.Duration(r.ValidFor))
+	r.mapLastQUICVisitorPerIP = new(sync.Map)
 
 	r.logger.Info("clienthellod reservoir is provisioned")
 	return nil

modcaddy/handler/handler.go

@@ -4,6 +4,7 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"net"
 	"net/http"
 
 	"github.com/caddyserver/caddy/v2"
@@ -76,8 +77,13 @@ func (h *Handler) ServeHTTP(wr http.ResponseWriter, req *http.Request, next cadd
 
 	if h.TLS && req.ProtoMajor <= 2 { // HTTP/1.0, HTTP/1.1, H2
 		return h.serveHTTP12(wr, req, next) // TLS ClientHello capture enabled, serve ClientHello
-	} else if h.QUIC && req.ProtoMajor == 3 { // QUIC
-		return h.serveQUIC(wr, req, next)
+	} else if h.QUIC {
+		if req.ProtoMajor == 3 { // QUIC
+			return h.serveQUIC(wr, req, next)
+		} else {
+			h.logger.Debug("Serving QUIC Fingerprint over TLS")
+			return h.serveQUICFingerprintOverTLS(wr, req, next)
+		}
 	}
 	return next.ServeHTTP(wr, req)
 }
@@ -142,6 +148,14 @@ func (h *Handler) serveQUIC(wr http.ResponseWriter, req *http.Request, next cadd
 
 	h.logger.Debug(fmt.Sprintf("Extracted QUIC fingerprint for %s", req.RemoteAddr))
 
+	// Get IP part of the RemoteAddr
+	ip, _, err := net.SplitHostPort(req.RemoteAddr)
+	if err == nil {
+		h.reservoir.NewQUICVisitor(ip, req.RemoteAddr)
+	} else {
+		h.logger.Error(fmt.Sprintf("Can't extract IP from %s: %v", req.RemoteAddr, err))
+	}
+
 	qfp.UserAgent = req.UserAgent()
 
 	// dump JSON
@@ -167,6 +181,55 @@ func (h *Handler) serveQUIC(wr http.ResponseWriter, req *http.Request, next cadd
 	return nil
 }
 
+func (h *Handler) serveQUICFingerprintOverTLS(wr http.ResponseWriter, req *http.Request, next caddyhttp.Handler) error { // skipcq: GO-W1029
+	// Get IP part of the RemoteAddr
+	ip, _, err := net.SplitHostPort(req.RemoteAddr)
+	if err != nil {
+		h.logger.Error(fmt.Sprintf("Can't extract IP from %s: %v", req.RemoteAddr, err))
+		return next.ServeHTTP(wr, req)
+	}
+
+	// Get the last QUIC visitor
+	fullKey, ok := h.reservoir.GetLastQUICVisitor(ip)
+	if !ok {
+		h.logger.Debug(fmt.Sprintf("Can't find last QUIC visitor for %s", ip))
+		return next.ServeHTTP(wr, req)
+	}
+
+	// Get the client hello from the reservoir
+	// get the client hello from the reservoir
+	qfp, err := h.reservoir.QUICFingerprinter().LookupAwait(fullKey)
+	if err != nil {
+		h.logger.Error(fmt.Sprintf("Can't extract QUIC fingerprint sent by %s: %v", ip, err))
+		return next.ServeHTTP(wr, req)
+	}
+
+	h.logger.Debug(fmt.Sprintf("Extracted QUIC fingerprint for %s", fullKey))
+	// qfp.UserAgent = req.UserAgent() // Should have been updated
+
+	// dump JSON
+	var b []byte
+	if req.URL.Query().Get("beautify") == "true" {
+		b, err = json.MarshalIndent(qfp, "", "  ")
+	} else {
+		b, err = json.Marshal(qfp)
+	}
+	if err != nil {
+		h.logger.Error("failed to marshal QUIC fingerprint into JSON", zap.Error(err))
+		return next.ServeHTTP(wr, req)
+	}
+
+	// write JSON to response
+	wr.Header().Set("Content-Type", "application/json")
+	wr.Header().Set("Connection", "close")
+	_, err = wr.Write(b)
+	if err != nil {
+		h.logger.Error("failed to write response", zap.Error(err))
+		return next.ServeHTTP(wr, req)
+	}
+	return nil
+}
+
 // UnmarshalCaddyfile unmarshals Caddyfile tokens into h.
 func (h *Handler) UnmarshalCaddyfile(d *caddyfile.Dispenser) error { // skipcq: GO-W1029
 	for d.Next() {

quic_client_initial.go

@@ -2,8 +2,10 @@ package clienthellod
 
 import (
 	"context"
+	"encoding/json"
 	"errors"
 	"fmt"
+	"log"
 	"sort"
 	"sync"
 	"sync/atomic"
@@ -14,7 +16,8 @@ import (
 
 type ClientInitial struct {
 	Header *QUICHeader `json:"header,omitempty"` // QUIC header
-	Frames QUICFrames  `json:"frames,omitempty"` // frames in order
+	Frames []uint64    `json:"frames,omitempty"` // frames ID in order
+	frames QUICFrames  // frames in order
 	raw    []byte
 }
 
@@ -26,11 +29,13 @@ func UnmarshalQUICClientInitialPacket(p []byte) (ci *ClientInitial, err error) {
 		raw: p,
 	}
 
-	ci.Header, ci.Frames, err = DecodeQUICHeaderAndFrames(p)
+	ci.Header, ci.frames, err = DecodeQUICHeaderAndFrames(p)
 	if err != nil {
 		return
 	}
 
+	ci.Frames = ci.frames.FrameTypes()
+
 	// reassembledCRYPTOFrame, err := ReassembleCRYPTOFrames(cip.Header.Frames())
 	// if err != nil {
 	// 	return cip, err
@@ -128,7 +133,7 @@ func (gci *GatheredClientInitials) AddPacket(cip *ClientInitial) error {
 		return gci.Packets[i].Header.initialPacketNumber < gci.Packets[j].Header.initialPacketNumber
 	})
 
-	if err := gci.clientHelloReconstructor.FromFrames(cip.Frames); err != nil {
+	if err := gci.clientHelloReconstructor.FromFrames(cip.frames); err != nil {
 		if errors.Is(err, ErrNeedMoreFrames) {
 			return nil // abort early, need more frames before ClientHello can be reconstructed
 		} else {
@@ -165,6 +170,12 @@ func (gci *GatheredClientInitials) lockedGatherComplete() error {
 	// Finally, mark the completion
 	gci.completed.Store(true)
 
+	b, err := json.Marshal(gci)
+	if err != nil {
+		return err
+	}
+	log.Printf("GatheredClientInitials: %s", string(b))
+
 	return nil
 }
 

quic_common.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"io"
 
+	"github.com/gaukas/clienthellod/internal/utils"
 	"golang.org/x/crypto/cryptobyte"
 )
 
@@ -110,7 +111,8 @@ func DecodeQUICHeaderAndFrames(p []byte) (hdr *QUICHeader, frames QUICFrames, er
 
 	// LSB of the first byte is protected, we will resolve it later
 
-	hdr.Version = p[1:5]
+	hdr.Version = make(utils.Uint8Arr, 4)
+	copy(hdr.Version, p[1:5])
 	s := cryptobyte.String(p[5:])
 	initialRandom := new(cryptobyte.String)
 	if !s.ReadUint8LengthPrefixed(initialRandom) {

quic_fingerprint.go

@@ -3,8 +3,10 @@ package clienthellod
 import (
 	"crypto/sha1" // skipcq: GSC-G505
 	"encoding/binary"
+	"encoding/json"
 	"errors"
 	"io"
+	"log"
 	"net"
 	"sync"
 	"sync/atomic"
@@ -35,7 +37,7 @@ func GenerateQUICFingerprint(gci *GatheredClientInitials) (*QUICFingerprint, err
 	// TODO: calculate hash
 	h := sha1.New() // skipcq: GO-S1025, GSC-G401
 	updateU64(h, gci.NumID)
-	updateU64(h, uint64(gci.ClientHello.NumID))
+	updateU64(h, uint64(gci.ClientHello.NormNumID))
 	updateU64(h, gci.TransportParameters.NumID)
 
 	qfp.NumID = binary.BigEndian.Uint64(h.Sum(nil))
@@ -84,6 +86,11 @@ func (qfp *QUICFingerprinter) HandlePacket(from string, p []byte) error {
 		}
 		return err
 	}
+	b, err := json.Marshal(ci)
+	if err != nil {
+		return err
+	}
+	log.Printf("ClientInitial: %s", string(b))
 
 	var testGci *GatheredClientInitials
 	if qfp.timeout == time.Duration(0) {
@@ -102,6 +109,7 @@ func (qfp *QUICFingerprinter) HandlePacket(from string, p []byte) error {
 				<-time.After(qfp.timeout)
 			}
 			qfp.mapGatheringClientInitials.Delete(from)
+			log.Printf("GatheredClientInitials for %s expired", from)
 		}()
 	}
 
@@ -161,7 +169,7 @@ func (qfp *QUICFingerprinter) HandleIPConn(ipc *net.IPConn) error {
 }
 
 func (qfp *QUICFingerprinter) Lookup(from string) *QUICFingerprint {
-	gci, ok := qfp.mapGatheringClientInitials.LoadAndDelete(from)
+	gci, ok := qfp.mapGatheringClientInitials.Load(from) // when using LoadAndDelete, some implementations "wasting" QUIC connections will fail
 	if !ok {
 		return nil
 	}
@@ -184,7 +192,7 @@ func (qfp *QUICFingerprinter) Lookup(from string) *QUICFingerprint {
 }
 
 func (qfp *QUICFingerprinter) LookupAwait(from string) (*QUICFingerprint, error) {
-	gci, ok := qfp.mapGatheringClientInitials.LoadAndDelete(from)
+	gci, ok := qfp.mapGatheringClientInitials.Load(from) // when using LoadAndDelete, some implementations "wasting" QUIC connections will fail
 	if !ok {
 		return nil, errors.New("GatheredClientInitials not found for the given key")
 	}
@@ -194,6 +202,12 @@ func (qfp *QUICFingerprinter) LookupAwait(from string) (*QUICFingerprint, error)
 		return nil, errors.New("GatheredClientInitials loaded from sync.Map failed type assertion")
 	}
 
+	b, err := json.Marshal(gatheredCI)
+	if err != nil {
+		return nil, err
+	}
+	log.Printf("Found GatheredClientInitials: %s", string(b))
+
 	qf, err := GenerateQUICFingerprint(gatheredCI)
 	if err != nil {
 		return nil, err