fix/deep review improvements

CHANGELOG.md

@@ -21,6 +21,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
   (`hawk`, `eyrie`, `yaad`, `sight`, `inspect`). No git tag has been cut yet.
 
 ### Removed
+- Unused internal config structs `QuestionAwareLayerConfig`,
+  `DensityAdaptiveLayerConfig`, `NumericalQuantLayerConfig`, and
+  `DynamicRatioLayerConfig` (and their fields on the internal `LayerConfig`)
+  that were marked "reserved for future implementation" and never referenced.
 - `scripts/deploy.sh` — referenced a Dockerfile, k8s manifests, and an
   `internal/integration/` package that never existed. tok is a Go library
   (no binary, no HTTP server); deployment happens via `go get`.

internal/fastops/simd.go

@@ -168,15 +168,18 @@ func FastLower(s string) string {
 		return s
 	}
 
-	// Convert using unsafe for zero-copy when possible
-	b := []byte(s)
+	// Convert using unsafe to avoid a second copy on the way back to string.
+	b := []byte(s) // fresh copy of s; we own this backing array exclusively
 	for i := 0; i < len(b); i++ {
 		c := b[i]
 		if c >= 'A' && c <= 'Z' {
 			b[i] = c + ('a' - 'A')
 		}
 	}
-	return *(*string)(unsafe.Pointer(&b))
+	// safe: b is a private copy created above; no other reference to its
+	// backing array exists and it is never mutated after this point, so
+	// aliasing it as a string preserves string immutability.
+	return unsafe.String(unsafe.SliceData(b), len(b))
 }
 
 // FastEqual compares strings with early exit

internal/filter/pipeline_types.go

@@ -225,41 +225,12 @@ type AgentMemoryLayerConfig struct {
 	ConsolidationMax   int
 }
 
-// QuestionAwareLayerConfig groups T12 settings.
-// NOTE: Currently unused - reserved for future implementation.
-type QuestionAwareLayerConfig struct {
-	Enabled   bool
-	Threshold float64
-}
-
-// DensityAdaptiveLayerConfig groups T17 settings.
-// NOTE: Currently unused - reserved for future implementation.
-type DensityAdaptiveLayerConfig struct {
-	Enabled     bool
-	TargetRatio float64
-	Threshold   float64
-}
-
 // TFIDFLayerConfig groups TF-IDF filter settings.
 type TFIDFLayerConfig struct {
 	Enabled   bool
 	Threshold float64
 }
 
-// NumericalQuantLayerConfig groups numerical quantization settings.
-// NOTE: Currently unused - reserved for future implementation.
-type NumericalQuantLayerConfig struct {
-	Enabled       bool
-	DecimalPlaces int
-}
-
-// DynamicRatioLayerConfig groups dynamic compression ratio settings.
-// NOTE: Currently unused - reserved for future implementation.
-type DynamicRatioLayerConfig struct {
-	Enabled bool
-	Base    float64
-}
-
 // LayerConfig groups per-layer config structs.
 type LayerConfig struct {
 	Core              CoreLayersConfig
@@ -273,11 +244,7 @@ type LayerConfig struct {
 	LazyPruner        LazyPrunerLayerConfig
 	SemanticAnchor    SemanticAnchorLayerConfig
 	AgentMemory       AgentMemoryLayerConfig
-	QuestionAware     QuestionAwareLayerConfig
-	DensityAdaptive   DensityAdaptiveLayerConfig
 	TFIDF             TFIDFLayerConfig
-	NumericalQuant    NumericalQuantLayerConfig
-	DynamicRatio      DynamicRatioLayerConfig
 	SymbolicCompress  bool
 	PhraseGrouping    bool
 	Hypernym          bool

internal/filter/zerocopy.go

@@ -32,7 +32,10 @@ func (z *ZeroCopyBuffer) String() string {
 	if len(z.data) == 0 {
 		return ""
 	}
-	return *(*string)(unsafe.Pointer(&z.data))
+	// safe: the string header aliases z.data's backing array. The invariant
+	// (documented above) is that the backing array is not mutated while the
+	// returned string is live — Append/Reset after String() violate it.
+	return unsafe.String(unsafe.SliceData(z.data), len(z.data))
 }
 
 func (z *ZeroCopyBuffer) Reset() {
@@ -51,7 +54,11 @@ func StringToBytes(s string) []byte {
 	if len(s) == 0 {
 		return nil
 	}
-	return *(*[]byte)(unsafe.Pointer(&s))
+	// safe: the slice aliases the string's backing array (len == cap == len(s)).
+	// The invariant is that the backing array is never mutated through the
+	// returned slice — callers treat it as read-only, preserving string
+	// immutability.
+	return unsafe.Slice(unsafe.StringData(s), len(s))
 }
 
 // BytesToString converts []byte to string without allocation.
@@ -62,5 +69,8 @@ func BytesToString(b []byte) string {
 	if len(b) == 0 {
 		return ""
 	}
-	return *(*string)(unsafe.Pointer(&b))
+	// safe: the string header aliases b's backing array. The invariant is that
+	// the backing array is not mutated (through b or any other alias) while the
+	// returned string is live, preserving string immutability.
+	return unsafe.String(unsafe.SliceData(b), len(b))
 }

internal/tracking/tracking.go

@@ -44,6 +44,7 @@ type Event struct {
 // Tracker is the public type. Construct with New() or NewAt(path).
 type Tracker struct {
 	db        *sql.DB
+	path      string
 	mu        sync.Mutex
 	retenDays int
 	closed    bool
@@ -77,16 +78,16 @@ func NewAt(ctx context.Context, path string) (*Tracker, error) {
 		return nil, errors.New("tracking: empty path")
 	}
 	if err := mkdirAll(filepath.Dir(path), 0o700); err != nil {
-		return nil, fmt.Errorf("tracking: create dir: %w", err)
+		return nil, fmt.Errorf("tracking: create dir %q: %w", filepath.Dir(path), err)
 	}
 	db, err := sql.Open("sqlite", path)
 	if err != nil {
-		return nil, fmt.Errorf("tracking: open db: %w", err)
+		return nil, fmt.Errorf("tracking: open db at %q: %w", path, err)
 	}
 	// SQLite is single-writer; use a per-connection mutex to serialize
 	// writes without serializing reads.
 	db.SetMaxOpenConns(1)
-	t := &Tracker{db: db, retenDays: 90}
+	t := &Tracker{db: db, path: path, retenDays: 90}
 	if err := t.initSchema(ctx); err != nil {
 		_ = db.Close()
 		return nil, err
@@ -127,7 +128,7 @@ func (t *Tracker) initSchema(ctx context.Context) error {
 		CREATE INDEX IF NOT EXISTS idx_events_command ON events(command);
 	`)
 	if err != nil {
-		return fmt.Errorf("tracking: init schema: %w", err)
+		return fmt.Errorf("tracking: init schema at %q: %w", t.path, err)
 	}
 	return nil
 }
@@ -137,7 +138,7 @@ func (t *Tracker) initSchema(ctx context.Context) error {
 func (t *Tracker) enableWAL(ctx context.Context) error {
 	_, err := t.db.ExecContext(ctx, `PRAGMA journal_mode=WAL;`)
 	if err != nil {
-		return fmt.Errorf("tracking: enable WAL: %w", err)
+		return fmt.Errorf("tracking: enable WAL at %q: %w", t.path, err)
 	}
 	return nil
 }
@@ -166,7 +167,7 @@ func (t *Tracker) Record(ctx context.Context, ev Event) error {
 		ev.Mode, ev.Tier, ev.Model,
 	)
 	if err != nil {
-		return fmt.Errorf("tracking: record: %w", err)
+		return fmt.Errorf("tracking: record event in db at %q: %w", t.path, err)
 	}
 	return nil
 }
@@ -209,7 +210,7 @@ func (t *Tracker) Aggregate(ctx context.Context, days int) (Aggregate, error) {
 		WHERE ts >= ?
 	`, since)
 	if err := row.Scan(&agg.EventCount, &agg.TotalBytesSaved, &agg.AvgSavingsPct); err != nil {
-		return agg, fmt.Errorf("tracking: aggregate: %w", err)
+		return agg, fmt.Errorf("tracking: aggregate query on db at %q: %w", t.path, err)
 	}
 	agg.PeriodStart = time.Unix(since, 0)
 	agg.PeriodEnd = time.Now()
@@ -235,7 +236,7 @@ func (t *Tracker) Recent(ctx context.Context, n int) ([]Event, error) {
 		LIMIT ?
 	`, n)
 	if err != nil {
-		return nil, fmt.Errorf("tracking: recent: %w", err)
+		return nil, fmt.Errorf("tracking: recent query on db at %q: %w", t.path, err)
 	}
 	defer func() { _ = rows.Close() }()
 	var out []Event
@@ -246,7 +247,7 @@ func (t *Tracker) Recent(ctx context.Context, n int) ([]Event, error) {
 			&ev.OriginalBytes, &ev.CompressedBytes,
 			&ev.OriginalTokens, &ev.CompressedTokens,
 			&ev.Mode, &ev.Tier, &ev.Model); err != nil {
-			return nil, err
+			return nil, fmt.Errorf("tracking: scan event row from db at %q: %w", t.path, err)
 		}
 		ev.Timestamp = time.Unix(ts, 0)
 		if ev.OriginalBytes > 0 {
@@ -266,7 +267,7 @@ func (t *Tracker) Prune(ctx context.Context) (int64, error) {
 	cutoff := time.Now().Add(-time.Duration(t.retenDays) * 24 * time.Hour).Unix()
 	res, err := t.db.ExecContext(ctx, `DELETE FROM events WHERE ts < ?`, cutoff)
 	if err != nil {
-		return 0, fmt.Errorf("tracking: prune: %w", err)
+		return 0, fmt.Errorf("tracking: prune db at %q: %w", t.path, err)
 	}
 	return res.RowsAffected()
 }