diff --git a/.gitignore b/.gitignore index cf8abcdf3..6fbb2fc74 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,8 @@ *.dylib build/ *.c1z +# Frozen test fixtures are the exception (version-skew corpus etc.). +!pkg/sync/testdata/*.c1z *.db *.csv *.xlsx diff --git a/cmd/baton/main.go b/cmd/baton/main.go index e0e8da82c..fcd6515db 100644 --- a/cmd/baton/main.go +++ b/cmd/baton/main.go @@ -34,6 +34,7 @@ func main() { cliCmd.AddCommand(accessCmd()) cliCmd.AddCommand(dumpDBCmd()) cliCmd.AddCommand(syncsCmd()) + cliCmd.AddCommand(sourceCacheCmd()) cliCmd.AddCommand(optimizeDb()) cliCmd.AddCommand(toPebbleCmd()) cliCmd.AddCommand(explorerCmd()) diff --git a/cmd/baton/source_cache.go b/cmd/baton/source_cache.go new file mode 100644 index 000000000..1c086e361 --- /dev/null +++ b/cmd/baton/source_cache.go @@ -0,0 +1,255 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "os" + "sort" + "strings" + "text/tabwriter" + "time" + + "github.com/spf13/cobra" + + "github.com/conductorone/baton-sdk/pkg/dotc1z" + "github.com/conductorone/baton-sdk/pkg/logging" + "github.com/conductorone/baton-sdk/pkg/sourcecache" + sdkSync "github.com/conductorone/baton-sdk/pkg/sync" +) + +// source-cache: the attribution surface for source-cache replay. Rows in +// a c1z are stamped with the scope that produced them and the manifest +// records each scope's validator + write time — this command joins the +// two so "why is this row missing / stale?" becomes a per-scope answer: +// when was the scope last actually fetched, what validator did it record, +// how many rows does it hold, and is its index/manifest state consistent. +// +// Exits non-zero when any orphan-index scope is found (ingestion +// invariant I6) on a file that is ELIGIBLE as a replay source: such a +// file would poison a future sync's replay. Files the metadata gate +// already refuses (compaction folds — which legitimately clear the +// manifest while keeping scope stamps — and partial/derived syncs) can +// never be replayed from, so their index/manifest divergence is reported +// as "stale-index" without failing the audit. +func sourceCacheCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "source-cache", + Short: "Audit the C1Z's source-cache replay state (scopes, validators, row counts, consistency)", + RunE: runSourceCacheAudit, + } + return cmd +} + +type sourceCacheScopeReport struct { + RowKind string `json:"row_kind"` + ScopeKey string `json:"scope_key"` + Rows int `json:"rows"` + Validator string `json:"cache_validator,omitempty"` + DiscoveredAt time.Time `json:"discovered_at,omitempty"` + // Status: "ok" (manifest + rows), "zero-row" (manifest entry with no + // stamped rows — legal: the scope legitimately returned nothing), + // "invalidated" (the dangling-reference drops removed rows from this + // scope, so its entry is marked to miss and re-fetch cold next sync — + // legal, informational), "orphan-index" (stamped rows with NO + // manifest entry on a replay-eligible file — an invariant violation + // that would poison the next sync's replay; see ingestion invariant + // I6), or "stale-index" (the same shape on a file the replay metadata + // gate already refuses, e.g. a compaction fold whose manifest was + // deliberately cleared — informational, never a failure). + Status string `json:"status"` +} + +type sourceCacheAuditReport struct { + SyncID string `json:"sync_id,omitempty"` + // SyncType / Compacted / UsableAsReplaySource describe whether a + // FUTURE sync may use this file as its replay source (the metadata + // gate; see c1zstore.SyncRun.UsableAsReplaySource, plus the + // compaction-provenance token check the syncer applies for artifacts + // produced by pre-compacted-flag compactors). + SyncType string `json:"sync_type,omitempty"` + Compacted bool `json:"compacted"` + UsableAsReplaySource bool `json:"usable_as_replay_source"` + ReplayUnusableReason string `json:"replay_unusable_reason,omitempty"` + Scopes []sourceCacheScopeReport `json:"scopes"` + OrphanIndexScopes int `json:"orphan_index_scopes"` + StaleIndexScopes int `json:"stale_index_scopes,omitempty"` +} + +func runSourceCacheAudit(cmd *cobra.Command, args []string) error { + ctx, err := logging.Init(context.Background(), logging.WithLogFormat("console"), logging.WithLogLevel("error")) + if err != nil { + return err + } + c1zPath, err := cmd.Flags().GetString("file") + if err != nil { + return err + } + outputFormat, err := cmd.Flags().GetString("output-format") + if err != nil { + return err + } + + store, err := openReadOnlyC1ZStore(ctx, c1zPath) + if err != nil { + return err + } + defer store.Close(ctx) + + inspector, ok := store.(dotc1z.SourceCacheInspector) + if !ok { + return fmt.Errorf("source-cache state exists only in pebble-format c1z files; %q is not one", c1zPath) + } + + report := sourceCacheAuditReport{} + // The newest FINISHED sync of any type: it is the sync the store + // opened (see openReadOnlyC1ZStore's SetCurrentSync) and the one the + // replay metadata gate would evaluate — LatestFullSync could describe + // an older sync than the state being inspected, or nothing at all on + // partial/derived artifacts. + // + // A metadata READ FAILURE is fatal: replay eligibility gates whether + // orphan indexes are a hard failure or informational, so silently + // defaulting eligibility to false would let the audit exit zero on a + // file it could not actually judge. + latest, err := store.SyncMeta().LatestFinishedSyncOfAnyType(ctx) + if err != nil { + return fmt.Errorf("reading sync run metadata (needed to judge replay eligibility): %w", err) + } + if latest != nil { + report.SyncID = latest.ID + report.SyncType = string(latest.Type) + report.Compacted = latest.Compacted + // The syncer's own metadata gate (type, compacted flag, + // compaction-provenance token) — shared, so audit and syncer can + // never drift. + report.ReplayUnusableReason = sdkSync.ReplaySourceRunUnusableReason(latest) + report.UsableAsReplaySource = report.ReplayUnusableReason == "" + } + if report.UsableAsReplaySource { + // The replay-compatibility gate's offline-decidable half: key + // PRESENCE. An exact-match verdict depends on the NEXT run's + // connector declarations (cache generation, config fingerprint, + // selection), which no offline audit can know — but an ABSENT + // key can never match any future key, so the syncer will always + // run cold from this file (pre-compat-key SDKs, compaction + // folds). Reporting such a file usable=true would be wrong. + if _, compatFound, compatErr := inspector.GetSourceCacheCompat(ctx); compatErr != nil { + return fmt.Errorf("reading replay-compatibility key: %w", compatErr) + } else if !compatFound { + report.UsableAsReplaySource = false + report.ReplayUnusableReason = "no replay-compatibility key recorded (pre-key SDK or compaction fold); the syncer treats an absent key as a mismatch and always runs cold" + } + } + + manifest, err := inspector.SourceCacheManifestSnapshot(ctx) + if err != nil { + return fmt.Errorf("reading source-cache manifest: %w", err) + } + indexCounts, err := inspector.SourceScopeIndexSnapshot(ctx) + if err != nil { + return fmt.Errorf("reading scope indexes: %w", err) + } + orphans, err := inspector.SourceCacheOrphanScopes(ctx) + if err != nil { + return fmt.Errorf("checking scope consistency: %w", err) + } + + for key, snapEntry := range manifest { + kind, scope, found := strings.Cut(key, "\x00") + if !found { + return fmt.Errorf("malformed manifest snapshot key %q", key) + } + rep := sourceCacheScopeReport{RowKind: kind, ScopeKey: scope, Rows: indexCounts[kind][scope]} + entry, entryFound, err := inspector.LookupSourceCacheEntry(ctx, sourcecache.RowKind(kind), scope) + if err != nil { + return fmt.Errorf("reading manifest entry for %s/%q: %w", kind, scope, err) + } + if entryFound { + rep.Validator = entry.CacheValidator + rep.DiscoveredAt = entry.DiscoveredAt + } + switch { + case snapEntry.Invalidated: + // The lookup surface reports invalidated entries as misses; + // surface the retained (stale) validator for forensics. + rep.Validator = snapEntry.CacheValidator + rep.Status = "invalidated" + case rep.Rows == 0: + rep.Status = "zero-row" + default: + rep.Status = "ok" + } + report.Scopes = append(report.Scopes, rep) + } + for kind, scopes := range orphans { + for _, scope := range scopes { + status := "orphan-index" + if !report.UsableAsReplaySource { + // Replay from this file is already impossible (compaction + // fold, partial/derived sync): the divergence cannot poison + // anything. Compaction folds in particular ALWAYS look like + // this — the fold clears the manifest and keeps scope + // stamps by design (see pebble.ClearSourceCacheEntries). + status = "stale-index" + } + report.Scopes = append(report.Scopes, sourceCacheScopeReport{ + RowKind: kind, ScopeKey: scope, Rows: indexCounts[kind][scope], Status: status, + }) + if status == "orphan-index" { + report.OrphanIndexScopes++ + } else { + report.StaleIndexScopes++ + } + } + } + sort.Slice(report.Scopes, func(i, j int) bool { + if report.Scopes[i].RowKind != report.Scopes[j].RowKind { + return report.Scopes[i].RowKind < report.Scopes[j].RowKind + } + return report.Scopes[i].ScopeKey < report.Scopes[j].ScopeKey + }) + + if outputFormat == "json" { + enc := json.NewEncoder(os.Stdout) + enc.SetIndent("", " ") + if err := enc.Encode(report); err != nil { + return err + } + } else if err := printSourceCacheAudit(report); err != nil { + return err + } + if report.OrphanIndexScopes > 0 { + return fmt.Errorf("%d orphan-index scope(s) found (ingestion invariant I6); a future sync replaying from this file could silently drop or resurrect rows", report.OrphanIndexScopes) + } + return nil +} + +func printSourceCacheAudit(report sourceCacheAuditReport) error { + if report.SyncID != "" { + fmt.Fprintf(os.Stdout, "sync %s type=%s compacted=%v usable-as-replay-source=%v\n", + report.SyncID, report.SyncType, report.Compacted, report.UsableAsReplaySource) + if report.ReplayUnusableReason != "" { + fmt.Fprintf(os.Stdout, " (not usable: %s)\n", report.ReplayUnusableReason) + } + fmt.Fprintln(os.Stdout) + } + if len(report.Scopes) == 0 { + fmt.Fprintln(os.Stdout, "no source-cache state (cold artifact: every scope will fetch fresh next sync)") + return nil + } + w := tabwriter.NewWriter(os.Stdout, 2, 4, 2, ' ', 0) + fmt.Fprintln(w, "KIND\tSCOPE\tROWS\tSTATUS\tDISCOVERED\tVALIDATOR") + for _, r := range report.Scopes { + validator := r.Validator + if len(validator) > 48 { + validator = validator[:45] + "..." + } + discovered := "" + if !r.DiscoveredAt.IsZero() { + discovered = r.DiscoveredAt.UTC().Format(time.RFC3339) + } + fmt.Fprintf(w, "%s\t%s\t%d\t%s\t%s\t%s\n", r.RowKind, r.ScopeKey, r.Rows, r.Status, discovered, validator) + } + return w.Flush() +} diff --git a/internal/connector/connector.go b/internal/connector/connector.go index 04170efd6..336a8949e 100644 --- a/internal/connector/connector.go +++ b/internal/connector/connector.go @@ -23,11 +23,13 @@ import ( connectorV2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" connectorwrapperV1 "github.com/conductorone/baton-sdk/pb/c1/connector_wrapper/v1" + batonV1 "github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1" ratelimitV1 "github.com/conductorone/baton-sdk/pb/c1/ratelimit/v1" tlsV1 "github.com/conductorone/baton-sdk/pb/c1/utls/v1" "github.com/conductorone/baton-sdk/pkg/bid" ratelimit2 "github.com/conductorone/baton-sdk/pkg/ratelimit" "github.com/conductorone/baton-sdk/pkg/session" + "github.com/conductorone/baton-sdk/pkg/sourcecache" "github.com/conductorone/baton-sdk/pkg/types" "github.com/conductorone/baton-sdk/pkg/types/sessions" "github.com/conductorone/baton-sdk/pkg/ugrpc" @@ -54,20 +56,43 @@ type connectorClient struct { connectorV2.TicketsServiceClient connectorV2.ActionServiceClient - sessionStoreSetter sessions.SetSessionStore // this is the session store server + sessionStoreSetter sessions.SetSessionStore // this is the session store server + sourceCacheSetter sourcecache.SourceCacheSetter // this is the source-cache lookup server } var _ sessions.SetSessionStore = (*connectorClient)(nil) +var _ sourcecache.SourceCacheSetter = (*connectorClient)(nil) var _ SetSessionStoreSetter = (*connectorClient)(nil) +var _ SetSourceCacheSetter = (*connectorClient)(nil) type SetSessionStoreSetter interface { SetSessionStoreSetter(setsessionStoreSetter sessions.SetSessionStore) } +type SetSourceCacheSetter interface { + SetSourceCacheSetter(sourceCacheSetter sourcecache.SourceCacheSetter) +} + func (c *connectorClient) SetSessionStoreSetter(sessionStoreSetter sessions.SetSessionStore) { c.sessionStoreSetter = sessionStoreSetter } +func (c *connectorClient) SetSourceCacheSetter(sourceCacheSetter sourcecache.SourceCacheSetter) { + c.sourceCacheSetter = sourceCacheSetter +} + +// SetSourceCache forwards the syncer's per-sync lookup to whatever +// receives it: the subprocess-mode BatonSourceCacheService server, or the +// in-process builder. A nil setter means the connector never opted in; +// the syncer calls this unconditionally, so stay quiet at debug level. +func (c *connectorClient) SetSourceCache(ctx context.Context, lookup sourcecache.Lookup) { + if c.sourceCacheSetter == nil { + ctxzap.Extract(ctx).Debug("connectorClient's source cache setter is nil — connector did not opt into source caching") + return + } + c.sourceCacheSetter.SetSourceCache(ctx, lookup) +} + func (c *connectorClient) SetSessionStore(ctx context.Context, store sessions.SessionStore) { if c.sessionStoreSetter == nil { // Demoted from Warn to Debug: this path is the normal case for @@ -109,7 +134,8 @@ type wrapper struct { now func() time.Time - SessionServer sessions.SetSessionStore + SessionServer sessions.SetSessionStore + SourceCacheServer sourcecache.SourceCacheSetter } type Option func(ctx context.Context, w *wrapper) error @@ -197,6 +223,12 @@ func NewWrapper(ctx context.Context, server interface{}, opts ...Option) (*wrapp server: connectorServer, now: time.Now, } + // In-process delivery: a connectorbuilder-based server implements + // sourcecache.SourceCacheSetter itself, so the syncer's per-sync lookup can be + // installed without the subprocess gRPC hop. + if sourceCacheServer, ok := connectorServer.(sourcecache.SourceCacheSetter); ok { + w.SourceCacheServer = sourceCacheServer + } for _, o := range opts { err := o(ctx, w) @@ -276,18 +308,29 @@ func (cw *wrapper) runServer(ctx context.Context, serverCred *tlsV1.Credential) return 0, fmt.Errorf("failed to create session listener config: %w", err) } - // TODO(kans): block until we send a request or something/error handling in general. + // One listener serves BatonSessionService (connector session data) + // and BatonSourceCacheService (source-cache scope lookups). Keeping + // them as separate RPCs keeps validator lookups out of the + // connector's local MemorySessionCache and its TTL/eviction rules. l.Info("starting session store server") - server := session.NewGRPCSessionServer() - cw.SessionServer = server + sessionServer := session.NewGRPCSessionServer() + sourceCacheServer := sourcecache.NewGRPCServer() + cw.SessionServer = sessionServer + cw.SourceCacheServer = sourceCacheServer go func() { defer sessionListener.Close() - serverErr := session.StartGRPCSessionServerWithOptions(ctx, sessionListener, server, + grpcServer := grpc.NewServer( grpc.Creds(credentials.NewTLS(tlsConfig)), grpc.ChainUnaryInterceptor(ugrpc.UnaryServerInterceptor(ctx)...), ) - if serverErr != nil { - l.Error("failed to create session store server", zap.Error(serverErr)) + batonV1.RegisterBatonSessionServiceServer(grpcServer, sessionServer) + batonV1.RegisterBatonSourceCacheServiceServer(grpcServer, sourceCacheServer) + go func() { + <-ctx.Done() + grpcServer.GracefulStop() + }() + if serveErr := grpcServer.Serve(sessionListener); serveErr != nil { + l.Error("session/source-cache server stopped", zap.Error(serveErr)) return } }() @@ -440,6 +483,7 @@ func (cw *wrapper) C(ctx context.Context) (types.ConnectorClient, error) { cw.conn = conn client := NewConnectorClient(ctx, cw.conn) client.SetSessionStoreSetter(cw.SessionServer) + client.SetSourceCacheSetter(cw.SourceCacheServer) cw.client = client return client, nil diff --git a/pb/c1/connector/v2/annotation_source_cache.pb.go b/pb/c1/connector/v2/annotation_source_cache.pb.go new file mode 100644 index 000000000..93e5092a5 --- /dev/null +++ b/pb/c1/connector/v2/annotation_source_cache.pb.go @@ -0,0 +1,1008 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.10 +// protoc (unknown) +// source: c1/connector/v2/annotation_source_cache.proto + +//go:build !protoopaque + +package v2 + +import ( + _ "github.com/envoyproxy/protoc-gen-validate/validate" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type SourceCacheCapability_Mode int32 + +const ( + SourceCacheCapability_MODE_UNSPECIFIED SourceCacheCapability_Mode = 0 + SourceCacheCapability_MODE_DISABLED SourceCacheCapability_Mode = 1 + SourceCacheCapability_MODE_READ_WRITE SourceCacheCapability_Mode = 2 +) + +// Enum value maps for SourceCacheCapability_Mode. +var ( + SourceCacheCapability_Mode_name = map[int32]string{ + 0: "MODE_UNSPECIFIED", + 1: "MODE_DISABLED", + 2: "MODE_READ_WRITE", + } + SourceCacheCapability_Mode_value = map[string]int32{ + "MODE_UNSPECIFIED": 0, + "MODE_DISABLED": 1, + "MODE_READ_WRITE": 2, + } +) + +func (x SourceCacheCapability_Mode) Enum() *SourceCacheCapability_Mode { + p := new(SourceCacheCapability_Mode) + *p = x + return p +} + +func (x SourceCacheCapability_Mode) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (SourceCacheCapability_Mode) Descriptor() protoreflect.EnumDescriptor { + return file_c1_connector_v2_annotation_source_cache_proto_enumTypes[0].Descriptor() +} + +func (SourceCacheCapability_Mode) Type() protoreflect.EnumType { + return &file_c1_connector_v2_annotation_source_cache_proto_enumTypes[0] +} + +func (x SourceCacheCapability_Mode) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// SourceCacheCapability is attached to ConnectorServiceValidateResponse +// annotations to opt in to source-cache replay. Absent or any mode other +// than MODE_READ_WRITE means all source-cache annotations are ignored. +type SourceCacheCapability struct { + state protoimpl.MessageState `protogen:"hybrid.v1"` + Mode SourceCacheCapability_Mode `protobuf:"varint,1,opt,name=mode,proto3,enum=c1.connector.v2.SourceCacheCapability_Mode" json:"mode,omitempty"` + // cache_generation is the connector's replay-compatibility generation: + // bump it whenever the connector changes how it computes scopes, + // validators, or rows in a way that makes a PRIOR artifact's cached + // rows unsafe to replay (schema of emitted rows, scope partitioning, + // id construction). The SDK never interprets the value — replay is + // permitted only when the previous artifact recorded a byte-identical + // generation (any mismatch runs the sync cold). Never derive this from + // a semver: compatibility is an explicit declaration, not an + // inference. Empty is a valid (constant) generation. + CacheGeneration string `protobuf:"bytes,2,opt,name=cache_generation,json=cacheGeneration,proto3" json:"cache_generation,omitempty"` + // config_fingerprint is an opaque connector-computed digest of every + // configuration and permission input that changes what upstream data + // the connector CAN see (credentials scope, tenant/org selection, + // API-permission grants, include/exclude config). Replay is permitted + // only when the previous artifact recorded a byte-identical + // fingerprint: a config change means cached scopes may cover a + // different universe, and replaying them would resurrect rows the new + // configuration can no longer observe (or hide rows it now can). + // Empty means "connector declares no config sensitivity" and matches + // only empty. + ConfigFingerprint string `protobuf:"bytes,3,opt,name=config_fingerprint,json=configFingerprint,proto3" json:"config_fingerprint,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SourceCacheCapability) Reset() { + *x = SourceCacheCapability{} + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SourceCacheCapability) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SourceCacheCapability) ProtoMessage() {} + +func (x *SourceCacheCapability) ProtoReflect() protoreflect.Message { + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *SourceCacheCapability) GetMode() SourceCacheCapability_Mode { + if x != nil { + return x.Mode + } + return SourceCacheCapability_MODE_UNSPECIFIED +} + +func (x *SourceCacheCapability) GetCacheGeneration() string { + if x != nil { + return x.CacheGeneration + } + return "" +} + +func (x *SourceCacheCapability) GetConfigFingerprint() string { + if x != nil { + return x.ConfigFingerprint + } + return "" +} + +func (x *SourceCacheCapability) SetMode(v SourceCacheCapability_Mode) { + x.Mode = v +} + +func (x *SourceCacheCapability) SetCacheGeneration(v string) { + x.CacheGeneration = v +} + +func (x *SourceCacheCapability) SetConfigFingerprint(v string) { + x.ConfigFingerprint = v +} + +type SourceCacheCapability_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + Mode SourceCacheCapability_Mode + // cache_generation is the connector's replay-compatibility generation: + // bump it whenever the connector changes how it computes scopes, + // validators, or rows in a way that makes a PRIOR artifact's cached + // rows unsafe to replay (schema of emitted rows, scope partitioning, + // id construction). The SDK never interprets the value — replay is + // permitted only when the previous artifact recorded a byte-identical + // generation (any mismatch runs the sync cold). Never derive this from + // a semver: compatibility is an explicit declaration, not an + // inference. Empty is a valid (constant) generation. + CacheGeneration string + // config_fingerprint is an opaque connector-computed digest of every + // configuration and permission input that changes what upstream data + // the connector CAN see (credentials scope, tenant/org selection, + // API-permission grants, include/exclude config). Replay is permitted + // only when the previous artifact recorded a byte-identical + // fingerprint: a config change means cached scopes may cover a + // different universe, and replaying them would resurrect rows the new + // configuration can no longer observe (or hide rows it now can). + // Empty means "connector declares no config sensitivity" and matches + // only empty. + ConfigFingerprint string +} + +func (b0 SourceCacheCapability_builder) Build() *SourceCacheCapability { + m0 := &SourceCacheCapability{} + b, x := &b0, m0 + _, _ = b, x + x.Mode = b.Mode + x.CacheGeneration = b.CacheGeneration + x.ConfigFingerprint = b.ConfigFingerprint + return m0 +} + +// SourceCacheRecord is attached to a list-response page whose rows were +// freshly fetched from upstream. The SDK stamps the page's rows with +// scope_key so a future sync can replay them as a unit. +type SourceCacheRecord struct { + state protoimpl.MessageState `protogen:"hybrid.v1"` + // Connector-computed stable identifier for the canonical scope. Must be + // byte-stable across syncs for the same logical scope. Prefer an ORDERED + // natural identifier (e.g. the request URL, or "groups/{id}/members") + // over a random hash: scope-index writes lead with this value, so + // identifiers that correlate with fetch order keep index writes nearly + // append-ordered at scale. sourcecache.HashScope is the fallback when no + // compact natural form exists. + ScopeKey string `protobuf:"bytes,1,opt,name=scope_key,json=scopeKey,proto3" json:"scope_key,omitempty"` + // Opaque validator to persist for this scope. May be empty on interim + // pages of a multi-page scope (e.g. Graph @odata.nextLink pages); the + // SDK writes the scope's manifest entry when a non-empty cache_validator arrives. + // A 200 response with zero rows still persists the entry. + CacheValidator string `protobuf:"bytes,2,opt,name=cache_validator,json=cacheValidator,proto3" json:"cache_validator,omitempty"` + // Tombstones applied after this page's rows commit. Lets every page of + // a multi-page delta round carry its own deletions as the provider + // delivers them, instead of buffering a whole round onto the first + // (replay-annotated) page. Same formats as SourceCacheReplay. + // + // PRECONDITION for tombstones anywhere in a round: the provider's delta + // must be coalesced — at most one add-or-tombstone per object per round + // (Microsoft Graph guarantees this by returning final object state). + // With interleaved add/remove events for one object, per-page ordering + // is deterministic (a page's rows upsert before its deletions apply) + // but cross-page re-adds after a tombstone are the connector's + // responsibility to order. + DeletedIds []string `protobuf:"bytes,3,rep,name=deleted_ids,json=deletedIds,proto3" json:"deleted_ids,omitempty"` + // Principal-scoped grant tombstones: for RowKindGrants pages, each + // entry deletes EVERY grant row stamped with this scope whose principal + // id equals the entry — no principal resource type and no canonical + // grant-id reconstruction required (delta tombstones usually carry only + // a bare object id, and the object may no longer exist to look up). + // + // PRECONDITION: the scope must be partitioned so that "principal + // removed from scope" means every grant they have in the scope is gone + // — one scope per navigation with independent removal semantics (e.g. + // members and owners of a group are separate scopes, or membership + // removal would take the owner grant with it). + // + // For RowKindResources pages, each entry deletes the resource row(s) + // stamped with this scope whose resource id equals the entry, any + // resource type. + DeletedPrincipalIds []string `protobuf:"bytes,4,rep,name=deleted_principal_ids,json=deletedPrincipalIds,proto3" json:"deleted_principal_ids,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SourceCacheRecord) Reset() { + *x = SourceCacheRecord{} + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SourceCacheRecord) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SourceCacheRecord) ProtoMessage() {} + +func (x *SourceCacheRecord) ProtoReflect() protoreflect.Message { + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *SourceCacheRecord) GetScopeKey() string { + if x != nil { + return x.ScopeKey + } + return "" +} + +func (x *SourceCacheRecord) GetCacheValidator() string { + if x != nil { + return x.CacheValidator + } + return "" +} + +func (x *SourceCacheRecord) GetDeletedIds() []string { + if x != nil { + return x.DeletedIds + } + return nil +} + +func (x *SourceCacheRecord) GetDeletedPrincipalIds() []string { + if x != nil { + return x.DeletedPrincipalIds + } + return nil +} + +func (x *SourceCacheRecord) SetScopeKey(v string) { + x.ScopeKey = v +} + +func (x *SourceCacheRecord) SetCacheValidator(v string) { + x.CacheValidator = v +} + +func (x *SourceCacheRecord) SetDeletedIds(v []string) { + x.DeletedIds = v +} + +func (x *SourceCacheRecord) SetDeletedPrincipalIds(v []string) { + x.DeletedPrincipalIds = v +} + +type SourceCacheRecord_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + // Connector-computed stable identifier for the canonical scope. Must be + // byte-stable across syncs for the same logical scope. Prefer an ORDERED + // natural identifier (e.g. the request URL, or "groups/{id}/members") + // over a random hash: scope-index writes lead with this value, so + // identifiers that correlate with fetch order keep index writes nearly + // append-ordered at scale. sourcecache.HashScope is the fallback when no + // compact natural form exists. + ScopeKey string + // Opaque validator to persist for this scope. May be empty on interim + // pages of a multi-page scope (e.g. Graph @odata.nextLink pages); the + // SDK writes the scope's manifest entry when a non-empty cache_validator arrives. + // A 200 response with zero rows still persists the entry. + CacheValidator string + // Tombstones applied after this page's rows commit. Lets every page of + // a multi-page delta round carry its own deletions as the provider + // delivers them, instead of buffering a whole round onto the first + // (replay-annotated) page. Same formats as SourceCacheReplay. + // + // PRECONDITION for tombstones anywhere in a round: the provider's delta + // must be coalesced — at most one add-or-tombstone per object per round + // (Microsoft Graph guarantees this by returning final object state). + // With interleaved add/remove events for one object, per-page ordering + // is deterministic (a page's rows upsert before its deletions apply) + // but cross-page re-adds after a tombstone are the connector's + // responsibility to order. + DeletedIds []string + // Principal-scoped grant tombstones: for RowKindGrants pages, each + // entry deletes EVERY grant row stamped with this scope whose principal + // id equals the entry — no principal resource type and no canonical + // grant-id reconstruction required (delta tombstones usually carry only + // a bare object id, and the object may no longer exist to look up). + // + // PRECONDITION: the scope must be partitioned so that "principal + // removed from scope" means every grant they have in the scope is gone + // — one scope per navigation with independent removal semantics (e.g. + // members and owners of a group are separate scopes, or membership + // removal would take the owner grant with it). + // + // For RowKindResources pages, each entry deletes the resource row(s) + // stamped with this scope whose resource id equals the entry, any + // resource type. + DeletedPrincipalIds []string +} + +func (b0 SourceCacheRecord_builder) Build() *SourceCacheRecord { + m0 := &SourceCacheRecord{} + b, x := &b0, m0 + _, _ = b, x + x.ScopeKey = b.ScopeKey + x.CacheValidator = b.CacheValidator + x.DeletedIds = b.DeletedIds + x.DeletedPrincipalIds = b.DeletedPrincipalIds + return m0 +} + +// SourceCacheReplay is attached to a list-response page to tell the SDK to +// copy the previous sync's rows for scope_key into the current sync. +// +// The row kind (resources, entitlements, grants) is determined by which +// RPC the annotation arrived on, never by the annotation itself. +// +// The connector must only emit this for a scope whose validator it received +// from the SDK's source-cache lookup during this same sync. A replay for +// an unknown scope fails the sync: the connector has already skipped row +// generation, so there is nothing to fall back to. +type SourceCacheReplay struct { + state protoimpl.MessageState `protogen:"hybrid.v1"` + ScopeKey string `protobuf:"bytes,1,opt,name=scope_key,json=scopeKey,proto3" json:"scope_key,omitempty"` + // Validator to persist for this scope in the current sync. For an HTTP + // 304 this is the unchanged validator. For a delta query this is the NEW + // token; it may instead be supplied by the final overlay page's + // SourceCacheRecord.cache_validator, in which case this field may be left empty. + CacheValidator string `protobuf:"bytes,2,opt,name=cache_validator,json=cacheValidator,proto3" json:"cache_validator,omitempty"` + // When true, the response (and subsequent pages carrying + // SourceCacheRecord with the same scope_key) contains changed rows to + // upsert on top of the replayed base. When false the response must + // contain no rows for this scope. + // + // TRANSITIONAL: the SDK currently tolerates rows on a non-overlay + // replay page — it warns and upserts them with overlay semantics — + // while the model is validated against real providers. Do not rely on + // this: the tolerance will become a hard error, because a connector + // that fetched fresh rows and still attached a replay annotation keeps + // resurrecting the replayed base every sync (upstream deletions never + // propagate). + Overlay bool `protobuf:"varint,3,opt,name=overlay,proto3" json:"overlay,omitempty"` + // Public canonical IDs (grant/entitlement IDs, or resource BIDs for + // RowKindResources) to delete from the current sync after the replay + // copy and this page's upserts. Used for delta-query tombstones (e.g. + // Microsoft Graph @removed entries). Subsequent pages of the round + // carry their tombstones on SourceCacheRecord.deleted_ids. + DeletedIds []string `protobuf:"bytes,4,rep,name=deleted_ids,json=deletedIds,proto3" json:"deleted_ids,omitempty"` + // Principal-scoped tombstones for this page; see + // SourceCacheRecord.deleted_principal_ids for semantics and + // preconditions. + DeletedPrincipalIds []string `protobuf:"bytes,5,rep,name=deleted_principal_ids,json=deletedPrincipalIds,proto3" json:"deleted_principal_ids,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SourceCacheReplay) Reset() { + *x = SourceCacheReplay{} + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SourceCacheReplay) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SourceCacheReplay) ProtoMessage() {} + +func (x *SourceCacheReplay) ProtoReflect() protoreflect.Message { + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *SourceCacheReplay) GetScopeKey() string { + if x != nil { + return x.ScopeKey + } + return "" +} + +func (x *SourceCacheReplay) GetCacheValidator() string { + if x != nil { + return x.CacheValidator + } + return "" +} + +func (x *SourceCacheReplay) GetOverlay() bool { + if x != nil { + return x.Overlay + } + return false +} + +func (x *SourceCacheReplay) GetDeletedIds() []string { + if x != nil { + return x.DeletedIds + } + return nil +} + +func (x *SourceCacheReplay) GetDeletedPrincipalIds() []string { + if x != nil { + return x.DeletedPrincipalIds + } + return nil +} + +func (x *SourceCacheReplay) SetScopeKey(v string) { + x.ScopeKey = v +} + +func (x *SourceCacheReplay) SetCacheValidator(v string) { + x.CacheValidator = v +} + +func (x *SourceCacheReplay) SetOverlay(v bool) { + x.Overlay = v +} + +func (x *SourceCacheReplay) SetDeletedIds(v []string) { + x.DeletedIds = v +} + +func (x *SourceCacheReplay) SetDeletedPrincipalIds(v []string) { + x.DeletedPrincipalIds = v +} + +type SourceCacheReplay_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + ScopeKey string + // Validator to persist for this scope in the current sync. For an HTTP + // 304 this is the unchanged validator. For a delta query this is the NEW + // token; it may instead be supplied by the final overlay page's + // SourceCacheRecord.cache_validator, in which case this field may be left empty. + CacheValidator string + // When true, the response (and subsequent pages carrying + // SourceCacheRecord with the same scope_key) contains changed rows to + // upsert on top of the replayed base. When false the response must + // contain no rows for this scope. + // + // TRANSITIONAL: the SDK currently tolerates rows on a non-overlay + // replay page — it warns and upserts them with overlay semantics — + // while the model is validated against real providers. Do not rely on + // this: the tolerance will become a hard error, because a connector + // that fetched fresh rows and still attached a replay annotation keeps + // resurrecting the replayed base every sync (upstream deletions never + // propagate). + Overlay bool + // Public canonical IDs (grant/entitlement IDs, or resource BIDs for + // RowKindResources) to delete from the current sync after the replay + // copy and this page's upserts. Used for delta-query tombstones (e.g. + // Microsoft Graph @removed entries). Subsequent pages of the round + // carry their tombstones on SourceCacheRecord.deleted_ids. + DeletedIds []string + // Principal-scoped tombstones for this page; see + // SourceCacheRecord.deleted_principal_ids for semantics and + // preconditions. + DeletedPrincipalIds []string +} + +func (b0 SourceCacheReplay_builder) Build() *SourceCacheReplay { + m0 := &SourceCacheReplay{} + b, x := &b0, m0 + _, _ = b, x + x.ScopeKey = b.ScopeKey + x.CacheValidator = b.CacheValidator + x.Overlay = b.Overlay + x.DeletedIds = b.DeletedIds + x.DeletedPrincipalIds = b.DeletedPrincipalIds + return m0 +} + +// SourceCacheLookupOffer is attached by the SDK to list REQUESTS when the +// syncer can answer source-cache lookup asks: the connector declared +// SourceCacheCapability and a warm previous-sync lookup is installed. +// Its presence is the connector's permission to answer with +// SourceCacheLookupAsk instead of rows. Connectors with a direct lookup +// (in-process, subprocess) never need to defer and may ignore it. +type SourceCacheLookupOffer struct { + state protoimpl.MessageState `protogen:"hybrid.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SourceCacheLookupOffer) Reset() { + *x = SourceCacheLookupOffer{} + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SourceCacheLookupOffer) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SourceCacheLookupOffer) ProtoMessage() {} + +func (x *SourceCacheLookupOffer) ProtoReflect() protoreflect.Message { + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +type SourceCacheLookupOffer_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + +} + +func (b0 SourceCacheLookupOffer_builder) Build() *SourceCacheLookupOffer { + m0 := &SourceCacheLookupOffer{} + b, x := &b0, m0 + _, _ = b, x + return m0 +} + +// SourceCacheLookupAsk is attached to a list RESPONSE in place of rows: +// the connector needs previous-sync validators before it can serve the +// page. An ask response must carry NO rows, NO next page token, and no +// other source-cache annotations; the syncer consumes it (it never +// reaches page handling), resolves every query, and re-invokes the same +// request with SourceCacheLookupAnswers attached. +// +// Only legal on responses to requests that carried +// SourceCacheLookupOffer. Bounces are capped per REQUEST (a page-token +// advance is a new request and resets the cap, so a multi-page action +// may legally ask once per page); a connector that keeps asking without +// progressing past the cap fails the sync loudly. +type SourceCacheLookupAsk struct { + state protoimpl.MessageState `protogen:"hybrid.v1"` + Queries []*SourceCacheLookupAsk_Query `protobuf:"bytes,1,rep,name=queries,proto3" json:"queries,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SourceCacheLookupAsk) Reset() { + *x = SourceCacheLookupAsk{} + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SourceCacheLookupAsk) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SourceCacheLookupAsk) ProtoMessage() {} + +func (x *SourceCacheLookupAsk) ProtoReflect() protoreflect.Message { + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[4] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *SourceCacheLookupAsk) GetQueries() []*SourceCacheLookupAsk_Query { + if x != nil { + return x.Queries + } + return nil +} + +func (x *SourceCacheLookupAsk) SetQueries(v []*SourceCacheLookupAsk_Query) { + x.Queries = v +} + +type SourceCacheLookupAsk_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + Queries []*SourceCacheLookupAsk_Query +} + +func (b0 SourceCacheLookupAsk_builder) Build() *SourceCacheLookupAsk { + m0 := &SourceCacheLookupAsk{} + b, x := &b0, m0 + _, _ = b, x + x.Queries = b.Queries + return m0 +} + +// SourceCacheLookupAnswers is attached by the SDK to the re-invoked +// REQUEST, carrying the resolution of a prior ask's queries. +// +// Every query of the prior ask is answered — found=true with the +// previous sync's validator, or found=false meaning fetch fresh. A found +// answer whose validator does not fit the per-request answer size budget is +// delivered as found=false: the scope degrades to a cold fetch +// (correct, just slower) instead of staying unresolved, because answers +// accumulate across bounces and a validator that did not fit once can never +// fit a later re-invoke. An ABSENT query (a phase-2 lookup for a scope +// the connector did not ask before) means unresolved: the connector may +// ask again, subject to the bounce cap. +type SourceCacheLookupAnswers struct { + state protoimpl.MessageState `protogen:"hybrid.v1"` + // Cap = ask max_items (4096) × the per-request bounce cap (4): answers + // accumulate across bounces, so a request may legally carry every + // answer its asks produced. Keep in sync with + // sourcecache.MaxLookupBouncesPerRequest / maxAnswersPerMessage. + Answers []*SourceCacheLookupAnswers_Answer `protobuf:"bytes,1,rep,name=answers,proto3" json:"answers,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SourceCacheLookupAnswers) Reset() { + *x = SourceCacheLookupAnswers{} + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SourceCacheLookupAnswers) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SourceCacheLookupAnswers) ProtoMessage() {} + +func (x *SourceCacheLookupAnswers) ProtoReflect() protoreflect.Message { + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[5] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *SourceCacheLookupAnswers) GetAnswers() []*SourceCacheLookupAnswers_Answer { + if x != nil { + return x.Answers + } + return nil +} + +func (x *SourceCacheLookupAnswers) SetAnswers(v []*SourceCacheLookupAnswers_Answer) { + x.Answers = v +} + +type SourceCacheLookupAnswers_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + // Cap = ask max_items (4096) × the per-request bounce cap (4): answers + // accumulate across bounces, so a request may legally carry every + // answer its asks produced. Keep in sync with + // sourcecache.MaxLookupBouncesPerRequest / maxAnswersPerMessage. + Answers []*SourceCacheLookupAnswers_Answer +} + +func (b0 SourceCacheLookupAnswers_builder) Build() *SourceCacheLookupAnswers { + m0 := &SourceCacheLookupAnswers{} + b, x := &b0, m0 + _, _ = b, x + x.Answers = b.Answers + return m0 +} + +type SourceCacheLookupAsk_Query struct { + state protoimpl.MessageState `protogen:"hybrid.v1"` + // One of the sourcecache.RowKind values: "resources", + // "entitlements", "grants". + RowKind string `protobuf:"bytes,1,opt,name=row_kind,json=rowKind,proto3" json:"row_kind,omitempty"` + ScopeKey string `protobuf:"bytes,2,opt,name=scope_key,json=scopeKey,proto3" json:"scope_key,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SourceCacheLookupAsk_Query) Reset() { + *x = SourceCacheLookupAsk_Query{} + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SourceCacheLookupAsk_Query) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SourceCacheLookupAsk_Query) ProtoMessage() {} + +func (x *SourceCacheLookupAsk_Query) ProtoReflect() protoreflect.Message { + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[6] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *SourceCacheLookupAsk_Query) GetRowKind() string { + if x != nil { + return x.RowKind + } + return "" +} + +func (x *SourceCacheLookupAsk_Query) GetScopeKey() string { + if x != nil { + return x.ScopeKey + } + return "" +} + +func (x *SourceCacheLookupAsk_Query) SetRowKind(v string) { + x.RowKind = v +} + +func (x *SourceCacheLookupAsk_Query) SetScopeKey(v string) { + x.ScopeKey = v +} + +type SourceCacheLookupAsk_Query_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + // One of the sourcecache.RowKind values: "resources", + // "entitlements", "grants". + RowKind string + ScopeKey string +} + +func (b0 SourceCacheLookupAsk_Query_builder) Build() *SourceCacheLookupAsk_Query { + m0 := &SourceCacheLookupAsk_Query{} + b, x := &b0, m0 + _, _ = b, x + x.RowKind = b.RowKind + x.ScopeKey = b.ScopeKey + return m0 +} + +type SourceCacheLookupAnswers_Answer struct { + state protoimpl.MessageState `protogen:"hybrid.v1"` + RowKind string `protobuf:"bytes,1,opt,name=row_kind,json=rowKind,proto3" json:"row_kind,omitempty"` + ScopeKey string `protobuf:"bytes,2,opt,name=scope_key,json=scopeKey,proto3" json:"scope_key,omitempty"` + Found bool `protobuf:"varint,3,opt,name=found,proto3" json:"found,omitempty"` + // The previous sync's validator; empty when found is false. Cap + // matches the lookup RPC (Graph delta tokens run long). + CacheValidator string `protobuf:"bytes,4,opt,name=cache_validator,json=cacheValidator,proto3" json:"cache_validator,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SourceCacheLookupAnswers_Answer) Reset() { + *x = SourceCacheLookupAnswers_Answer{} + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SourceCacheLookupAnswers_Answer) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SourceCacheLookupAnswers_Answer) ProtoMessage() {} + +func (x *SourceCacheLookupAnswers_Answer) ProtoReflect() protoreflect.Message { + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[7] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *SourceCacheLookupAnswers_Answer) GetRowKind() string { + if x != nil { + return x.RowKind + } + return "" +} + +func (x *SourceCacheLookupAnswers_Answer) GetScopeKey() string { + if x != nil { + return x.ScopeKey + } + return "" +} + +func (x *SourceCacheLookupAnswers_Answer) GetFound() bool { + if x != nil { + return x.Found + } + return false +} + +func (x *SourceCacheLookupAnswers_Answer) GetCacheValidator() string { + if x != nil { + return x.CacheValidator + } + return "" +} + +func (x *SourceCacheLookupAnswers_Answer) SetRowKind(v string) { + x.RowKind = v +} + +func (x *SourceCacheLookupAnswers_Answer) SetScopeKey(v string) { + x.ScopeKey = v +} + +func (x *SourceCacheLookupAnswers_Answer) SetFound(v bool) { + x.Found = v +} + +func (x *SourceCacheLookupAnswers_Answer) SetCacheValidator(v string) { + x.CacheValidator = v +} + +type SourceCacheLookupAnswers_Answer_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + RowKind string + ScopeKey string + Found bool + // The previous sync's validator; empty when found is false. Cap + // matches the lookup RPC (Graph delta tokens run long). + CacheValidator string +} + +func (b0 SourceCacheLookupAnswers_Answer_builder) Build() *SourceCacheLookupAnswers_Answer { + m0 := &SourceCacheLookupAnswers_Answer{} + b, x := &b0, m0 + _, _ = b, x + x.RowKind = b.RowKind + x.ScopeKey = b.ScopeKey + x.Found = b.Found + x.CacheValidator = b.CacheValidator + return m0 +} + +var File_c1_connector_v2_annotation_source_cache_proto protoreflect.FileDescriptor + +const file_c1_connector_v2_annotation_source_cache_proto_rawDesc = "" + + "\n" + + "-c1/connector/v2/annotation_source_cache.proto\x12\x0fc1.connector.v2\x1a\x17validate/validate.proto\"\x92\x02\n" + + "\x15SourceCacheCapability\x12?\n" + + "\x04mode\x18\x01 \x01(\x0e2+.c1.connector.v2.SourceCacheCapability.ModeR\x04mode\x126\n" + + "\x10cache_generation\x18\x02 \x01(\tB\v\xfaB\br\x06(\x80\x02\xd0\x01\x01R\x0fcacheGeneration\x12:\n" + + "\x12config_fingerprint\x18\x03 \x01(\tB\v\xfaB\br\x06(\x80\x02\xd0\x01\x01R\x11configFingerprint\"D\n" + + "\x04Mode\x12\x14\n" + + "\x10MODE_UNSPECIFIED\x10\x00\x12\x11\n" + + "\rMODE_DISABLED\x10\x01\x12\x13\n" + + "\x0fMODE_READ_WRITE\x10\x02\"\xae\x01\n" + + "\x11SourceCacheRecord\x12\x1b\n" + + "\tscope_key\x18\x01 \x01(\tR\bscopeKey\x12'\n" + + "\x0fcache_validator\x18\x02 \x01(\tR\x0ecacheValidator\x12\x1f\n" + + "\vdeleted_ids\x18\x03 \x03(\tR\n" + + "deletedIds\x122\n" + + "\x15deleted_principal_ids\x18\x04 \x03(\tR\x13deletedPrincipalIds\"\xc8\x01\n" + + "\x11SourceCacheReplay\x12\x1b\n" + + "\tscope_key\x18\x01 \x01(\tR\bscopeKey\x12'\n" + + "\x0fcache_validator\x18\x02 \x01(\tR\x0ecacheValidator\x12\x18\n" + + "\aoverlay\x18\x03 \x01(\bR\aoverlay\x12\x1f\n" + + "\vdeleted_ids\x18\x04 \x03(\tR\n" + + "deletedIds\x122\n" + + "\x15deleted_principal_ids\x18\x05 \x03(\tR\x13deletedPrincipalIds\"\x18\n" + + "\x16SourceCacheLookupOffer\"\xc2\x01\n" + + "\x14SourceCacheLookupAsk\x12R\n" + + "\aqueries\x18\x01 \x03(\v2+.c1.connector.v2.SourceCacheLookupAsk.QueryB\v\xfaB\b\x92\x01\x05\b\x01\x10\x80 R\aqueries\x1aV\n" + + "\x05Query\x12$\n" + + "\brow_kind\x18\x01 \x01(\tB\t\xfaB\x06r\x04 \x01(@R\arowKind\x12'\n" + + "\tscope_key\x18\x02 \x01(\tB\n" + + "\xfaB\ar\x05 \x01(\x80\x02R\bscopeKey\"\x99\x02\n" + + "\x18SourceCacheLookupAnswers\x12V\n" + + "\aanswers\x18\x01 \x03(\v20.c1.connector.v2.SourceCacheLookupAnswers.AnswerB\n" + + "\xfaB\a\x92\x01\x04\x10\x80\x80\x01R\aanswers\x1a\xa4\x01\n" + + "\x06Answer\x12$\n" + + "\brow_kind\x18\x01 \x01(\tB\t\xfaB\x06r\x04 \x01(@R\arowKind\x12'\n" + + "\tscope_key\x18\x02 \x01(\tB\n" + + "\xfaB\ar\x05 \x01(\x80\x02R\bscopeKey\x12\x14\n" + + "\x05found\x18\x03 \x01(\bR\x05found\x125\n" + + "\x0fcache_validator\x18\x04 \x01(\tB\f\xfaB\tr\a(\x80\x80\x04\xd0\x01\x01R\x0ecacheValidatorB6Z4github.com/conductorone/baton-sdk/pb/c1/connector/v2b\x06proto3" + +var file_c1_connector_v2_annotation_source_cache_proto_enumTypes = make([]protoimpl.EnumInfo, 1) +var file_c1_connector_v2_annotation_source_cache_proto_msgTypes = make([]protoimpl.MessageInfo, 8) +var file_c1_connector_v2_annotation_source_cache_proto_goTypes = []any{ + (SourceCacheCapability_Mode)(0), // 0: c1.connector.v2.SourceCacheCapability.Mode + (*SourceCacheCapability)(nil), // 1: c1.connector.v2.SourceCacheCapability + (*SourceCacheRecord)(nil), // 2: c1.connector.v2.SourceCacheRecord + (*SourceCacheReplay)(nil), // 3: c1.connector.v2.SourceCacheReplay + (*SourceCacheLookupOffer)(nil), // 4: c1.connector.v2.SourceCacheLookupOffer + (*SourceCacheLookupAsk)(nil), // 5: c1.connector.v2.SourceCacheLookupAsk + (*SourceCacheLookupAnswers)(nil), // 6: c1.connector.v2.SourceCacheLookupAnswers + (*SourceCacheLookupAsk_Query)(nil), // 7: c1.connector.v2.SourceCacheLookupAsk.Query + (*SourceCacheLookupAnswers_Answer)(nil), // 8: c1.connector.v2.SourceCacheLookupAnswers.Answer +} +var file_c1_connector_v2_annotation_source_cache_proto_depIdxs = []int32{ + 0, // 0: c1.connector.v2.SourceCacheCapability.mode:type_name -> c1.connector.v2.SourceCacheCapability.Mode + 7, // 1: c1.connector.v2.SourceCacheLookupAsk.queries:type_name -> c1.connector.v2.SourceCacheLookupAsk.Query + 8, // 2: c1.connector.v2.SourceCacheLookupAnswers.answers:type_name -> c1.connector.v2.SourceCacheLookupAnswers.Answer + 3, // [3:3] is the sub-list for method output_type + 3, // [3:3] is the sub-list for method input_type + 3, // [3:3] is the sub-list for extension type_name + 3, // [3:3] is the sub-list for extension extendee + 0, // [0:3] is the sub-list for field type_name +} + +func init() { file_c1_connector_v2_annotation_source_cache_proto_init() } +func file_c1_connector_v2_annotation_source_cache_proto_init() { + if File_c1_connector_v2_annotation_source_cache_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_c1_connector_v2_annotation_source_cache_proto_rawDesc), len(file_c1_connector_v2_annotation_source_cache_proto_rawDesc)), + NumEnums: 1, + NumMessages: 8, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_c1_connector_v2_annotation_source_cache_proto_goTypes, + DependencyIndexes: file_c1_connector_v2_annotation_source_cache_proto_depIdxs, + EnumInfos: file_c1_connector_v2_annotation_source_cache_proto_enumTypes, + MessageInfos: file_c1_connector_v2_annotation_source_cache_proto_msgTypes, + }.Build() + File_c1_connector_v2_annotation_source_cache_proto = out.File + file_c1_connector_v2_annotation_source_cache_proto_goTypes = nil + file_c1_connector_v2_annotation_source_cache_proto_depIdxs = nil +} diff --git a/pb/c1/connector/v2/annotation_source_cache.pb.validate.go b/pb/c1/connector/v2/annotation_source_cache.pb.validate.go new file mode 100644 index 000000000..80fb38aa0 --- /dev/null +++ b/pb/c1/connector/v2/annotation_source_cache.pb.validate.go @@ -0,0 +1,1046 @@ +// Code generated by protoc-gen-validate. DO NOT EDIT. +// source: c1/connector/v2/annotation_source_cache.proto + +package v2 + +import ( + "bytes" + "errors" + "fmt" + "net" + "net/mail" + "net/url" + "regexp" + "sort" + "strings" + "time" + "unicode/utf8" + + "google.golang.org/protobuf/types/known/anypb" +) + +// ensure the imports are used +var ( + _ = bytes.MinRead + _ = errors.New("") + _ = fmt.Print + _ = utf8.UTFMax + _ = (*regexp.Regexp)(nil) + _ = (*strings.Reader)(nil) + _ = net.IPv4len + _ = time.Duration(0) + _ = (*url.URL)(nil) + _ = (*mail.Address)(nil) + _ = anypb.Any{} + _ = sort.Sort +) + +// Validate checks the field values on SourceCacheCapability with the rules +// defined in the proto definition for this message. If any rules are +// violated, the first error encountered is returned, or nil if there are no violations. +func (m *SourceCacheCapability) Validate() error { + return m.validate(false) +} + +// ValidateAll checks the field values on SourceCacheCapability with the rules +// defined in the proto definition for this message. If any rules are +// violated, the result is a list of violation errors wrapped in +// SourceCacheCapabilityMultiError, or nil if none found. +func (m *SourceCacheCapability) ValidateAll() error { + return m.validate(true) +} + +func (m *SourceCacheCapability) validate(all bool) error { + if m == nil { + return nil + } + + var errors []error + + // no validation rules for Mode + + if m.GetCacheGeneration() != "" { + + if len(m.GetCacheGeneration()) > 256 { + err := SourceCacheCapabilityValidationError{ + field: "CacheGeneration", + reason: "value length must be at most 256 bytes", + } + if !all { + return err + } + errors = append(errors, err) + } + + } + + if m.GetConfigFingerprint() != "" { + + if len(m.GetConfigFingerprint()) > 256 { + err := SourceCacheCapabilityValidationError{ + field: "ConfigFingerprint", + reason: "value length must be at most 256 bytes", + } + if !all { + return err + } + errors = append(errors, err) + } + + } + + if len(errors) > 0 { + return SourceCacheCapabilityMultiError(errors) + } + + return nil +} + +// SourceCacheCapabilityMultiError is an error wrapping multiple validation +// errors returned by SourceCacheCapability.ValidateAll() if the designated +// constraints aren't met. +type SourceCacheCapabilityMultiError []error + +// Error returns a concatenation of all the error messages it wraps. +func (m SourceCacheCapabilityMultiError) Error() string { + msgs := make([]string, 0, len(m)) + for _, err := range m { + msgs = append(msgs, err.Error()) + } + return strings.Join(msgs, "; ") +} + +// AllErrors returns a list of validation violation errors. +func (m SourceCacheCapabilityMultiError) AllErrors() []error { return m } + +// SourceCacheCapabilityValidationError is the validation error returned by +// SourceCacheCapability.Validate if the designated constraints aren't met. +type SourceCacheCapabilityValidationError struct { + field string + reason string + cause error + key bool +} + +// Field function returns field value. +func (e SourceCacheCapabilityValidationError) Field() string { return e.field } + +// Reason function returns reason value. +func (e SourceCacheCapabilityValidationError) Reason() string { return e.reason } + +// Cause function returns cause value. +func (e SourceCacheCapabilityValidationError) Cause() error { return e.cause } + +// Key function returns key value. +func (e SourceCacheCapabilityValidationError) Key() bool { return e.key } + +// ErrorName returns error name. +func (e SourceCacheCapabilityValidationError) ErrorName() string { + return "SourceCacheCapabilityValidationError" +} + +// Error satisfies the builtin error interface +func (e SourceCacheCapabilityValidationError) Error() string { + cause := "" + if e.cause != nil { + cause = fmt.Sprintf(" | caused by: %v", e.cause) + } + + key := "" + if e.key { + key = "key for " + } + + return fmt.Sprintf( + "invalid %sSourceCacheCapability.%s: %s%s", + key, + e.field, + e.reason, + cause) +} + +var _ error = SourceCacheCapabilityValidationError{} + +var _ interface { + Field() string + Reason() string + Key() bool + Cause() error + ErrorName() string +} = SourceCacheCapabilityValidationError{} + +// Validate checks the field values on SourceCacheRecord with the rules defined +// in the proto definition for this message. If any rules are violated, the +// first error encountered is returned, or nil if there are no violations. +func (m *SourceCacheRecord) Validate() error { + return m.validate(false) +} + +// ValidateAll checks the field values on SourceCacheRecord with the rules +// defined in the proto definition for this message. If any rules are +// violated, the result is a list of violation errors wrapped in +// SourceCacheRecordMultiError, or nil if none found. +func (m *SourceCacheRecord) ValidateAll() error { + return m.validate(true) +} + +func (m *SourceCacheRecord) validate(all bool) error { + if m == nil { + return nil + } + + var errors []error + + // no validation rules for ScopeKey + + // no validation rules for CacheValidator + + if len(errors) > 0 { + return SourceCacheRecordMultiError(errors) + } + + return nil +} + +// SourceCacheRecordMultiError is an error wrapping multiple validation errors +// returned by SourceCacheRecord.ValidateAll() if the designated constraints +// aren't met. +type SourceCacheRecordMultiError []error + +// Error returns a concatenation of all the error messages it wraps. +func (m SourceCacheRecordMultiError) Error() string { + msgs := make([]string, 0, len(m)) + for _, err := range m { + msgs = append(msgs, err.Error()) + } + return strings.Join(msgs, "; ") +} + +// AllErrors returns a list of validation violation errors. +func (m SourceCacheRecordMultiError) AllErrors() []error { return m } + +// SourceCacheRecordValidationError is the validation error returned by +// SourceCacheRecord.Validate if the designated constraints aren't met. +type SourceCacheRecordValidationError struct { + field string + reason string + cause error + key bool +} + +// Field function returns field value. +func (e SourceCacheRecordValidationError) Field() string { return e.field } + +// Reason function returns reason value. +func (e SourceCacheRecordValidationError) Reason() string { return e.reason } + +// Cause function returns cause value. +func (e SourceCacheRecordValidationError) Cause() error { return e.cause } + +// Key function returns key value. +func (e SourceCacheRecordValidationError) Key() bool { return e.key } + +// ErrorName returns error name. +func (e SourceCacheRecordValidationError) ErrorName() string { + return "SourceCacheRecordValidationError" +} + +// Error satisfies the builtin error interface +func (e SourceCacheRecordValidationError) Error() string { + cause := "" + if e.cause != nil { + cause = fmt.Sprintf(" | caused by: %v", e.cause) + } + + key := "" + if e.key { + key = "key for " + } + + return fmt.Sprintf( + "invalid %sSourceCacheRecord.%s: %s%s", + key, + e.field, + e.reason, + cause) +} + +var _ error = SourceCacheRecordValidationError{} + +var _ interface { + Field() string + Reason() string + Key() bool + Cause() error + ErrorName() string +} = SourceCacheRecordValidationError{} + +// Validate checks the field values on SourceCacheReplay with the rules defined +// in the proto definition for this message. If any rules are violated, the +// first error encountered is returned, or nil if there are no violations. +func (m *SourceCacheReplay) Validate() error { + return m.validate(false) +} + +// ValidateAll checks the field values on SourceCacheReplay with the rules +// defined in the proto definition for this message. If any rules are +// violated, the result is a list of violation errors wrapped in +// SourceCacheReplayMultiError, or nil if none found. +func (m *SourceCacheReplay) ValidateAll() error { + return m.validate(true) +} + +func (m *SourceCacheReplay) validate(all bool) error { + if m == nil { + return nil + } + + var errors []error + + // no validation rules for ScopeKey + + // no validation rules for CacheValidator + + // no validation rules for Overlay + + if len(errors) > 0 { + return SourceCacheReplayMultiError(errors) + } + + return nil +} + +// SourceCacheReplayMultiError is an error wrapping multiple validation errors +// returned by SourceCacheReplay.ValidateAll() if the designated constraints +// aren't met. +type SourceCacheReplayMultiError []error + +// Error returns a concatenation of all the error messages it wraps. +func (m SourceCacheReplayMultiError) Error() string { + msgs := make([]string, 0, len(m)) + for _, err := range m { + msgs = append(msgs, err.Error()) + } + return strings.Join(msgs, "; ") +} + +// AllErrors returns a list of validation violation errors. +func (m SourceCacheReplayMultiError) AllErrors() []error { return m } + +// SourceCacheReplayValidationError is the validation error returned by +// SourceCacheReplay.Validate if the designated constraints aren't met. +type SourceCacheReplayValidationError struct { + field string + reason string + cause error + key bool +} + +// Field function returns field value. +func (e SourceCacheReplayValidationError) Field() string { return e.field } + +// Reason function returns reason value. +func (e SourceCacheReplayValidationError) Reason() string { return e.reason } + +// Cause function returns cause value. +func (e SourceCacheReplayValidationError) Cause() error { return e.cause } + +// Key function returns key value. +func (e SourceCacheReplayValidationError) Key() bool { return e.key } + +// ErrorName returns error name. +func (e SourceCacheReplayValidationError) ErrorName() string { + return "SourceCacheReplayValidationError" +} + +// Error satisfies the builtin error interface +func (e SourceCacheReplayValidationError) Error() string { + cause := "" + if e.cause != nil { + cause = fmt.Sprintf(" | caused by: %v", e.cause) + } + + key := "" + if e.key { + key = "key for " + } + + return fmt.Sprintf( + "invalid %sSourceCacheReplay.%s: %s%s", + key, + e.field, + e.reason, + cause) +} + +var _ error = SourceCacheReplayValidationError{} + +var _ interface { + Field() string + Reason() string + Key() bool + Cause() error + ErrorName() string +} = SourceCacheReplayValidationError{} + +// Validate checks the field values on SourceCacheLookupOffer with the rules +// defined in the proto definition for this message. If any rules are +// violated, the first error encountered is returned, or nil if there are no violations. +func (m *SourceCacheLookupOffer) Validate() error { + return m.validate(false) +} + +// ValidateAll checks the field values on SourceCacheLookupOffer with the rules +// defined in the proto definition for this message. If any rules are +// violated, the result is a list of violation errors wrapped in +// SourceCacheLookupOfferMultiError, or nil if none found. +func (m *SourceCacheLookupOffer) ValidateAll() error { + return m.validate(true) +} + +func (m *SourceCacheLookupOffer) validate(all bool) error { + if m == nil { + return nil + } + + var errors []error + + if len(errors) > 0 { + return SourceCacheLookupOfferMultiError(errors) + } + + return nil +} + +// SourceCacheLookupOfferMultiError is an error wrapping multiple validation +// errors returned by SourceCacheLookupOffer.ValidateAll() if the designated +// constraints aren't met. +type SourceCacheLookupOfferMultiError []error + +// Error returns a concatenation of all the error messages it wraps. +func (m SourceCacheLookupOfferMultiError) Error() string { + msgs := make([]string, 0, len(m)) + for _, err := range m { + msgs = append(msgs, err.Error()) + } + return strings.Join(msgs, "; ") +} + +// AllErrors returns a list of validation violation errors. +func (m SourceCacheLookupOfferMultiError) AllErrors() []error { return m } + +// SourceCacheLookupOfferValidationError is the validation error returned by +// SourceCacheLookupOffer.Validate if the designated constraints aren't met. +type SourceCacheLookupOfferValidationError struct { + field string + reason string + cause error + key bool +} + +// Field function returns field value. +func (e SourceCacheLookupOfferValidationError) Field() string { return e.field } + +// Reason function returns reason value. +func (e SourceCacheLookupOfferValidationError) Reason() string { return e.reason } + +// Cause function returns cause value. +func (e SourceCacheLookupOfferValidationError) Cause() error { return e.cause } + +// Key function returns key value. +func (e SourceCacheLookupOfferValidationError) Key() bool { return e.key } + +// ErrorName returns error name. +func (e SourceCacheLookupOfferValidationError) ErrorName() string { + return "SourceCacheLookupOfferValidationError" +} + +// Error satisfies the builtin error interface +func (e SourceCacheLookupOfferValidationError) Error() string { + cause := "" + if e.cause != nil { + cause = fmt.Sprintf(" | caused by: %v", e.cause) + } + + key := "" + if e.key { + key = "key for " + } + + return fmt.Sprintf( + "invalid %sSourceCacheLookupOffer.%s: %s%s", + key, + e.field, + e.reason, + cause) +} + +var _ error = SourceCacheLookupOfferValidationError{} + +var _ interface { + Field() string + Reason() string + Key() bool + Cause() error + ErrorName() string +} = SourceCacheLookupOfferValidationError{} + +// Validate checks the field values on SourceCacheLookupAsk with the rules +// defined in the proto definition for this message. If any rules are +// violated, the first error encountered is returned, or nil if there are no violations. +func (m *SourceCacheLookupAsk) Validate() error { + return m.validate(false) +} + +// ValidateAll checks the field values on SourceCacheLookupAsk with the rules +// defined in the proto definition for this message. If any rules are +// violated, the result is a list of violation errors wrapped in +// SourceCacheLookupAskMultiError, or nil if none found. +func (m *SourceCacheLookupAsk) ValidateAll() error { + return m.validate(true) +} + +func (m *SourceCacheLookupAsk) validate(all bool) error { + if m == nil { + return nil + } + + var errors []error + + if l := len(m.GetQueries()); l < 1 || l > 4096 { + err := SourceCacheLookupAskValidationError{ + field: "Queries", + reason: "value must contain between 1 and 4096 items, inclusive", + } + if !all { + return err + } + errors = append(errors, err) + } + + for idx, item := range m.GetQueries() { + _, _ = idx, item + + if all { + switch v := interface{}(item).(type) { + case interface{ ValidateAll() error }: + if err := v.ValidateAll(); err != nil { + errors = append(errors, SourceCacheLookupAskValidationError{ + field: fmt.Sprintf("Queries[%v]", idx), + reason: "embedded message failed validation", + cause: err, + }) + } + case interface{ Validate() error }: + if err := v.Validate(); err != nil { + errors = append(errors, SourceCacheLookupAskValidationError{ + field: fmt.Sprintf("Queries[%v]", idx), + reason: "embedded message failed validation", + cause: err, + }) + } + } + } else if v, ok := interface{}(item).(interface{ Validate() error }); ok { + if err := v.Validate(); err != nil { + return SourceCacheLookupAskValidationError{ + field: fmt.Sprintf("Queries[%v]", idx), + reason: "embedded message failed validation", + cause: err, + } + } + } + + } + + if len(errors) > 0 { + return SourceCacheLookupAskMultiError(errors) + } + + return nil +} + +// SourceCacheLookupAskMultiError is an error wrapping multiple validation +// errors returned by SourceCacheLookupAsk.ValidateAll() if the designated +// constraints aren't met. +type SourceCacheLookupAskMultiError []error + +// Error returns a concatenation of all the error messages it wraps. +func (m SourceCacheLookupAskMultiError) Error() string { + msgs := make([]string, 0, len(m)) + for _, err := range m { + msgs = append(msgs, err.Error()) + } + return strings.Join(msgs, "; ") +} + +// AllErrors returns a list of validation violation errors. +func (m SourceCacheLookupAskMultiError) AllErrors() []error { return m } + +// SourceCacheLookupAskValidationError is the validation error returned by +// SourceCacheLookupAsk.Validate if the designated constraints aren't met. +type SourceCacheLookupAskValidationError struct { + field string + reason string + cause error + key bool +} + +// Field function returns field value. +func (e SourceCacheLookupAskValidationError) Field() string { return e.field } + +// Reason function returns reason value. +func (e SourceCacheLookupAskValidationError) Reason() string { return e.reason } + +// Cause function returns cause value. +func (e SourceCacheLookupAskValidationError) Cause() error { return e.cause } + +// Key function returns key value. +func (e SourceCacheLookupAskValidationError) Key() bool { return e.key } + +// ErrorName returns error name. +func (e SourceCacheLookupAskValidationError) ErrorName() string { + return "SourceCacheLookupAskValidationError" +} + +// Error satisfies the builtin error interface +func (e SourceCacheLookupAskValidationError) Error() string { + cause := "" + if e.cause != nil { + cause = fmt.Sprintf(" | caused by: %v", e.cause) + } + + key := "" + if e.key { + key = "key for " + } + + return fmt.Sprintf( + "invalid %sSourceCacheLookupAsk.%s: %s%s", + key, + e.field, + e.reason, + cause) +} + +var _ error = SourceCacheLookupAskValidationError{} + +var _ interface { + Field() string + Reason() string + Key() bool + Cause() error + ErrorName() string +} = SourceCacheLookupAskValidationError{} + +// Validate checks the field values on SourceCacheLookupAnswers with the rules +// defined in the proto definition for this message. If any rules are +// violated, the first error encountered is returned, or nil if there are no violations. +func (m *SourceCacheLookupAnswers) Validate() error { + return m.validate(false) +} + +// ValidateAll checks the field values on SourceCacheLookupAnswers with the +// rules defined in the proto definition for this message. If any rules are +// violated, the result is a list of violation errors wrapped in +// SourceCacheLookupAnswersMultiError, or nil if none found. +func (m *SourceCacheLookupAnswers) ValidateAll() error { + return m.validate(true) +} + +func (m *SourceCacheLookupAnswers) validate(all bool) error { + if m == nil { + return nil + } + + var errors []error + + if len(m.GetAnswers()) > 16384 { + err := SourceCacheLookupAnswersValidationError{ + field: "Answers", + reason: "value must contain no more than 16384 item(s)", + } + if !all { + return err + } + errors = append(errors, err) + } + + for idx, item := range m.GetAnswers() { + _, _ = idx, item + + if all { + switch v := interface{}(item).(type) { + case interface{ ValidateAll() error }: + if err := v.ValidateAll(); err != nil { + errors = append(errors, SourceCacheLookupAnswersValidationError{ + field: fmt.Sprintf("Answers[%v]", idx), + reason: "embedded message failed validation", + cause: err, + }) + } + case interface{ Validate() error }: + if err := v.Validate(); err != nil { + errors = append(errors, SourceCacheLookupAnswersValidationError{ + field: fmt.Sprintf("Answers[%v]", idx), + reason: "embedded message failed validation", + cause: err, + }) + } + } + } else if v, ok := interface{}(item).(interface{ Validate() error }); ok { + if err := v.Validate(); err != nil { + return SourceCacheLookupAnswersValidationError{ + field: fmt.Sprintf("Answers[%v]", idx), + reason: "embedded message failed validation", + cause: err, + } + } + } + + } + + if len(errors) > 0 { + return SourceCacheLookupAnswersMultiError(errors) + } + + return nil +} + +// SourceCacheLookupAnswersMultiError is an error wrapping multiple validation +// errors returned by SourceCacheLookupAnswers.ValidateAll() if the designated +// constraints aren't met. +type SourceCacheLookupAnswersMultiError []error + +// Error returns a concatenation of all the error messages it wraps. +func (m SourceCacheLookupAnswersMultiError) Error() string { + msgs := make([]string, 0, len(m)) + for _, err := range m { + msgs = append(msgs, err.Error()) + } + return strings.Join(msgs, "; ") +} + +// AllErrors returns a list of validation violation errors. +func (m SourceCacheLookupAnswersMultiError) AllErrors() []error { return m } + +// SourceCacheLookupAnswersValidationError is the validation error returned by +// SourceCacheLookupAnswers.Validate if the designated constraints aren't met. +type SourceCacheLookupAnswersValidationError struct { + field string + reason string + cause error + key bool +} + +// Field function returns field value. +func (e SourceCacheLookupAnswersValidationError) Field() string { return e.field } + +// Reason function returns reason value. +func (e SourceCacheLookupAnswersValidationError) Reason() string { return e.reason } + +// Cause function returns cause value. +func (e SourceCacheLookupAnswersValidationError) Cause() error { return e.cause } + +// Key function returns key value. +func (e SourceCacheLookupAnswersValidationError) Key() bool { return e.key } + +// ErrorName returns error name. +func (e SourceCacheLookupAnswersValidationError) ErrorName() string { + return "SourceCacheLookupAnswersValidationError" +} + +// Error satisfies the builtin error interface +func (e SourceCacheLookupAnswersValidationError) Error() string { + cause := "" + if e.cause != nil { + cause = fmt.Sprintf(" | caused by: %v", e.cause) + } + + key := "" + if e.key { + key = "key for " + } + + return fmt.Sprintf( + "invalid %sSourceCacheLookupAnswers.%s: %s%s", + key, + e.field, + e.reason, + cause) +} + +var _ error = SourceCacheLookupAnswersValidationError{} + +var _ interface { + Field() string + Reason() string + Key() bool + Cause() error + ErrorName() string +} = SourceCacheLookupAnswersValidationError{} + +// Validate checks the field values on SourceCacheLookupAsk_Query with the +// rules defined in the proto definition for this message. If any rules are +// violated, the first error encountered is returned, or nil if there are no violations. +func (m *SourceCacheLookupAsk_Query) Validate() error { + return m.validate(false) +} + +// ValidateAll checks the field values on SourceCacheLookupAsk_Query with the +// rules defined in the proto definition for this message. If any rules are +// violated, the result is a list of violation errors wrapped in +// SourceCacheLookupAsk_QueryMultiError, or nil if none found. +func (m *SourceCacheLookupAsk_Query) ValidateAll() error { + return m.validate(true) +} + +func (m *SourceCacheLookupAsk_Query) validate(all bool) error { + if m == nil { + return nil + } + + var errors []error + + if l := len(m.GetRowKind()); l < 1 || l > 64 { + err := SourceCacheLookupAsk_QueryValidationError{ + field: "RowKind", + reason: "value length must be between 1 and 64 bytes, inclusive", + } + if !all { + return err + } + errors = append(errors, err) + } + + if l := len(m.GetScopeKey()); l < 1 || l > 256 { + err := SourceCacheLookupAsk_QueryValidationError{ + field: "ScopeKey", + reason: "value length must be between 1 and 256 bytes, inclusive", + } + if !all { + return err + } + errors = append(errors, err) + } + + if len(errors) > 0 { + return SourceCacheLookupAsk_QueryMultiError(errors) + } + + return nil +} + +// SourceCacheLookupAsk_QueryMultiError is an error wrapping multiple +// validation errors returned by SourceCacheLookupAsk_Query.ValidateAll() if +// the designated constraints aren't met. +type SourceCacheLookupAsk_QueryMultiError []error + +// Error returns a concatenation of all the error messages it wraps. +func (m SourceCacheLookupAsk_QueryMultiError) Error() string { + msgs := make([]string, 0, len(m)) + for _, err := range m { + msgs = append(msgs, err.Error()) + } + return strings.Join(msgs, "; ") +} + +// AllErrors returns a list of validation violation errors. +func (m SourceCacheLookupAsk_QueryMultiError) AllErrors() []error { return m } + +// SourceCacheLookupAsk_QueryValidationError is the validation error returned +// by SourceCacheLookupAsk_Query.Validate if the designated constraints aren't met. +type SourceCacheLookupAsk_QueryValidationError struct { + field string + reason string + cause error + key bool +} + +// Field function returns field value. +func (e SourceCacheLookupAsk_QueryValidationError) Field() string { return e.field } + +// Reason function returns reason value. +func (e SourceCacheLookupAsk_QueryValidationError) Reason() string { return e.reason } + +// Cause function returns cause value. +func (e SourceCacheLookupAsk_QueryValidationError) Cause() error { return e.cause } + +// Key function returns key value. +func (e SourceCacheLookupAsk_QueryValidationError) Key() bool { return e.key } + +// ErrorName returns error name. +func (e SourceCacheLookupAsk_QueryValidationError) ErrorName() string { + return "SourceCacheLookupAsk_QueryValidationError" +} + +// Error satisfies the builtin error interface +func (e SourceCacheLookupAsk_QueryValidationError) Error() string { + cause := "" + if e.cause != nil { + cause = fmt.Sprintf(" | caused by: %v", e.cause) + } + + key := "" + if e.key { + key = "key for " + } + + return fmt.Sprintf( + "invalid %sSourceCacheLookupAsk_Query.%s: %s%s", + key, + e.field, + e.reason, + cause) +} + +var _ error = SourceCacheLookupAsk_QueryValidationError{} + +var _ interface { + Field() string + Reason() string + Key() bool + Cause() error + ErrorName() string +} = SourceCacheLookupAsk_QueryValidationError{} + +// Validate checks the field values on SourceCacheLookupAnswers_Answer with the +// rules defined in the proto definition for this message. If any rules are +// violated, the first error encountered is returned, or nil if there are no violations. +func (m *SourceCacheLookupAnswers_Answer) Validate() error { + return m.validate(false) +} + +// ValidateAll checks the field values on SourceCacheLookupAnswers_Answer with +// the rules defined in the proto definition for this message. If any rules +// are violated, the result is a list of violation errors wrapped in +// SourceCacheLookupAnswers_AnswerMultiError, or nil if none found. +func (m *SourceCacheLookupAnswers_Answer) ValidateAll() error { + return m.validate(true) +} + +func (m *SourceCacheLookupAnswers_Answer) validate(all bool) error { + if m == nil { + return nil + } + + var errors []error + + if l := len(m.GetRowKind()); l < 1 || l > 64 { + err := SourceCacheLookupAnswers_AnswerValidationError{ + field: "RowKind", + reason: "value length must be between 1 and 64 bytes, inclusive", + } + if !all { + return err + } + errors = append(errors, err) + } + + if l := len(m.GetScopeKey()); l < 1 || l > 256 { + err := SourceCacheLookupAnswers_AnswerValidationError{ + field: "ScopeKey", + reason: "value length must be between 1 and 256 bytes, inclusive", + } + if !all { + return err + } + errors = append(errors, err) + } + + // no validation rules for Found + + if m.GetCacheValidator() != "" { + + if len(m.GetCacheValidator()) > 65536 { + err := SourceCacheLookupAnswers_AnswerValidationError{ + field: "CacheValidator", + reason: "value length must be at most 65536 bytes", + } + if !all { + return err + } + errors = append(errors, err) + } + + } + + if len(errors) > 0 { + return SourceCacheLookupAnswers_AnswerMultiError(errors) + } + + return nil +} + +// SourceCacheLookupAnswers_AnswerMultiError is an error wrapping multiple +// validation errors returned by SourceCacheLookupAnswers_Answer.ValidateAll() +// if the designated constraints aren't met. +type SourceCacheLookupAnswers_AnswerMultiError []error + +// Error returns a concatenation of all the error messages it wraps. +func (m SourceCacheLookupAnswers_AnswerMultiError) Error() string { + msgs := make([]string, 0, len(m)) + for _, err := range m { + msgs = append(msgs, err.Error()) + } + return strings.Join(msgs, "; ") +} + +// AllErrors returns a list of validation violation errors. +func (m SourceCacheLookupAnswers_AnswerMultiError) AllErrors() []error { return m } + +// SourceCacheLookupAnswers_AnswerValidationError is the validation error +// returned by SourceCacheLookupAnswers_Answer.Validate if the designated +// constraints aren't met. +type SourceCacheLookupAnswers_AnswerValidationError struct { + field string + reason string + cause error + key bool +} + +// Field function returns field value. +func (e SourceCacheLookupAnswers_AnswerValidationError) Field() string { return e.field } + +// Reason function returns reason value. +func (e SourceCacheLookupAnswers_AnswerValidationError) Reason() string { return e.reason } + +// Cause function returns cause value. +func (e SourceCacheLookupAnswers_AnswerValidationError) Cause() error { return e.cause } + +// Key function returns key value. +func (e SourceCacheLookupAnswers_AnswerValidationError) Key() bool { return e.key } + +// ErrorName returns error name. +func (e SourceCacheLookupAnswers_AnswerValidationError) ErrorName() string { + return "SourceCacheLookupAnswers_AnswerValidationError" +} + +// Error satisfies the builtin error interface +func (e SourceCacheLookupAnswers_AnswerValidationError) Error() string { + cause := "" + if e.cause != nil { + cause = fmt.Sprintf(" | caused by: %v", e.cause) + } + + key := "" + if e.key { + key = "key for " + } + + return fmt.Sprintf( + "invalid %sSourceCacheLookupAnswers_Answer.%s: %s%s", + key, + e.field, + e.reason, + cause) +} + +var _ error = SourceCacheLookupAnswers_AnswerValidationError{} + +var _ interface { + Field() string + Reason() string + Key() bool + Cause() error + ErrorName() string +} = SourceCacheLookupAnswers_AnswerValidationError{} diff --git a/pb/c1/connector/v2/annotation_source_cache_protoopaque.pb.go b/pb/c1/connector/v2/annotation_source_cache_protoopaque.pb.go new file mode 100644 index 000000000..74c055264 --- /dev/null +++ b/pb/c1/connector/v2/annotation_source_cache_protoopaque.pb.go @@ -0,0 +1,923 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.10 +// protoc (unknown) +// source: c1/connector/v2/annotation_source_cache.proto + +//go:build protoopaque + +package v2 + +import ( + _ "github.com/envoyproxy/protoc-gen-validate/validate" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type SourceCacheCapability_Mode int32 + +const ( + SourceCacheCapability_MODE_UNSPECIFIED SourceCacheCapability_Mode = 0 + SourceCacheCapability_MODE_DISABLED SourceCacheCapability_Mode = 1 + SourceCacheCapability_MODE_READ_WRITE SourceCacheCapability_Mode = 2 +) + +// Enum value maps for SourceCacheCapability_Mode. +var ( + SourceCacheCapability_Mode_name = map[int32]string{ + 0: "MODE_UNSPECIFIED", + 1: "MODE_DISABLED", + 2: "MODE_READ_WRITE", + } + SourceCacheCapability_Mode_value = map[string]int32{ + "MODE_UNSPECIFIED": 0, + "MODE_DISABLED": 1, + "MODE_READ_WRITE": 2, + } +) + +func (x SourceCacheCapability_Mode) Enum() *SourceCacheCapability_Mode { + p := new(SourceCacheCapability_Mode) + *p = x + return p +} + +func (x SourceCacheCapability_Mode) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (SourceCacheCapability_Mode) Descriptor() protoreflect.EnumDescriptor { + return file_c1_connector_v2_annotation_source_cache_proto_enumTypes[0].Descriptor() +} + +func (SourceCacheCapability_Mode) Type() protoreflect.EnumType { + return &file_c1_connector_v2_annotation_source_cache_proto_enumTypes[0] +} + +func (x SourceCacheCapability_Mode) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// SourceCacheCapability is attached to ConnectorServiceValidateResponse +// annotations to opt in to source-cache replay. Absent or any mode other +// than MODE_READ_WRITE means all source-cache annotations are ignored. +type SourceCacheCapability struct { + state protoimpl.MessageState `protogen:"opaque.v1"` + xxx_hidden_Mode SourceCacheCapability_Mode `protobuf:"varint,1,opt,name=mode,proto3,enum=c1.connector.v2.SourceCacheCapability_Mode"` + xxx_hidden_CacheGeneration string `protobuf:"bytes,2,opt,name=cache_generation,json=cacheGeneration,proto3"` + xxx_hidden_ConfigFingerprint string `protobuf:"bytes,3,opt,name=config_fingerprint,json=configFingerprint,proto3"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SourceCacheCapability) Reset() { + *x = SourceCacheCapability{} + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SourceCacheCapability) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SourceCacheCapability) ProtoMessage() {} + +func (x *SourceCacheCapability) ProtoReflect() protoreflect.Message { + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *SourceCacheCapability) GetMode() SourceCacheCapability_Mode { + if x != nil { + return x.xxx_hidden_Mode + } + return SourceCacheCapability_MODE_UNSPECIFIED +} + +func (x *SourceCacheCapability) GetCacheGeneration() string { + if x != nil { + return x.xxx_hidden_CacheGeneration + } + return "" +} + +func (x *SourceCacheCapability) GetConfigFingerprint() string { + if x != nil { + return x.xxx_hidden_ConfigFingerprint + } + return "" +} + +func (x *SourceCacheCapability) SetMode(v SourceCacheCapability_Mode) { + x.xxx_hidden_Mode = v +} + +func (x *SourceCacheCapability) SetCacheGeneration(v string) { + x.xxx_hidden_CacheGeneration = v +} + +func (x *SourceCacheCapability) SetConfigFingerprint(v string) { + x.xxx_hidden_ConfigFingerprint = v +} + +type SourceCacheCapability_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + Mode SourceCacheCapability_Mode + // cache_generation is the connector's replay-compatibility generation: + // bump it whenever the connector changes how it computes scopes, + // validators, or rows in a way that makes a PRIOR artifact's cached + // rows unsafe to replay (schema of emitted rows, scope partitioning, + // id construction). The SDK never interprets the value — replay is + // permitted only when the previous artifact recorded a byte-identical + // generation (any mismatch runs the sync cold). Never derive this from + // a semver: compatibility is an explicit declaration, not an + // inference. Empty is a valid (constant) generation. + CacheGeneration string + // config_fingerprint is an opaque connector-computed digest of every + // configuration and permission input that changes what upstream data + // the connector CAN see (credentials scope, tenant/org selection, + // API-permission grants, include/exclude config). Replay is permitted + // only when the previous artifact recorded a byte-identical + // fingerprint: a config change means cached scopes may cover a + // different universe, and replaying them would resurrect rows the new + // configuration can no longer observe (or hide rows it now can). + // Empty means "connector declares no config sensitivity" and matches + // only empty. + ConfigFingerprint string +} + +func (b0 SourceCacheCapability_builder) Build() *SourceCacheCapability { + m0 := &SourceCacheCapability{} + b, x := &b0, m0 + _, _ = b, x + x.xxx_hidden_Mode = b.Mode + x.xxx_hidden_CacheGeneration = b.CacheGeneration + x.xxx_hidden_ConfigFingerprint = b.ConfigFingerprint + return m0 +} + +// SourceCacheRecord is attached to a list-response page whose rows were +// freshly fetched from upstream. The SDK stamps the page's rows with +// scope_key so a future sync can replay them as a unit. +type SourceCacheRecord struct { + state protoimpl.MessageState `protogen:"opaque.v1"` + xxx_hidden_ScopeKey string `protobuf:"bytes,1,opt,name=scope_key,json=scopeKey,proto3"` + xxx_hidden_CacheValidator string `protobuf:"bytes,2,opt,name=cache_validator,json=cacheValidator,proto3"` + xxx_hidden_DeletedIds []string `protobuf:"bytes,3,rep,name=deleted_ids,json=deletedIds,proto3"` + xxx_hidden_DeletedPrincipalIds []string `protobuf:"bytes,4,rep,name=deleted_principal_ids,json=deletedPrincipalIds,proto3"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SourceCacheRecord) Reset() { + *x = SourceCacheRecord{} + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SourceCacheRecord) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SourceCacheRecord) ProtoMessage() {} + +func (x *SourceCacheRecord) ProtoReflect() protoreflect.Message { + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *SourceCacheRecord) GetScopeKey() string { + if x != nil { + return x.xxx_hidden_ScopeKey + } + return "" +} + +func (x *SourceCacheRecord) GetCacheValidator() string { + if x != nil { + return x.xxx_hidden_CacheValidator + } + return "" +} + +func (x *SourceCacheRecord) GetDeletedIds() []string { + if x != nil { + return x.xxx_hidden_DeletedIds + } + return nil +} + +func (x *SourceCacheRecord) GetDeletedPrincipalIds() []string { + if x != nil { + return x.xxx_hidden_DeletedPrincipalIds + } + return nil +} + +func (x *SourceCacheRecord) SetScopeKey(v string) { + x.xxx_hidden_ScopeKey = v +} + +func (x *SourceCacheRecord) SetCacheValidator(v string) { + x.xxx_hidden_CacheValidator = v +} + +func (x *SourceCacheRecord) SetDeletedIds(v []string) { + x.xxx_hidden_DeletedIds = v +} + +func (x *SourceCacheRecord) SetDeletedPrincipalIds(v []string) { + x.xxx_hidden_DeletedPrincipalIds = v +} + +type SourceCacheRecord_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + // Connector-computed stable identifier for the canonical scope. Must be + // byte-stable across syncs for the same logical scope. Prefer an ORDERED + // natural identifier (e.g. the request URL, or "groups/{id}/members") + // over a random hash: scope-index writes lead with this value, so + // identifiers that correlate with fetch order keep index writes nearly + // append-ordered at scale. sourcecache.HashScope is the fallback when no + // compact natural form exists. + ScopeKey string + // Opaque validator to persist for this scope. May be empty on interim + // pages of a multi-page scope (e.g. Graph @odata.nextLink pages); the + // SDK writes the scope's manifest entry when a non-empty cache_validator arrives. + // A 200 response with zero rows still persists the entry. + CacheValidator string + // Tombstones applied after this page's rows commit. Lets every page of + // a multi-page delta round carry its own deletions as the provider + // delivers them, instead of buffering a whole round onto the first + // (replay-annotated) page. Same formats as SourceCacheReplay. + // + // PRECONDITION for tombstones anywhere in a round: the provider's delta + // must be coalesced — at most one add-or-tombstone per object per round + // (Microsoft Graph guarantees this by returning final object state). + // With interleaved add/remove events for one object, per-page ordering + // is deterministic (a page's rows upsert before its deletions apply) + // but cross-page re-adds after a tombstone are the connector's + // responsibility to order. + DeletedIds []string + // Principal-scoped grant tombstones: for RowKindGrants pages, each + // entry deletes EVERY grant row stamped with this scope whose principal + // id equals the entry — no principal resource type and no canonical + // grant-id reconstruction required (delta tombstones usually carry only + // a bare object id, and the object may no longer exist to look up). + // + // PRECONDITION: the scope must be partitioned so that "principal + // removed from scope" means every grant they have in the scope is gone + // — one scope per navigation with independent removal semantics (e.g. + // members and owners of a group are separate scopes, or membership + // removal would take the owner grant with it). + // + // For RowKindResources pages, each entry deletes the resource row(s) + // stamped with this scope whose resource id equals the entry, any + // resource type. + DeletedPrincipalIds []string +} + +func (b0 SourceCacheRecord_builder) Build() *SourceCacheRecord { + m0 := &SourceCacheRecord{} + b, x := &b0, m0 + _, _ = b, x + x.xxx_hidden_ScopeKey = b.ScopeKey + x.xxx_hidden_CacheValidator = b.CacheValidator + x.xxx_hidden_DeletedIds = b.DeletedIds + x.xxx_hidden_DeletedPrincipalIds = b.DeletedPrincipalIds + return m0 +} + +// SourceCacheReplay is attached to a list-response page to tell the SDK to +// copy the previous sync's rows for scope_key into the current sync. +// +// The row kind (resources, entitlements, grants) is determined by which +// RPC the annotation arrived on, never by the annotation itself. +// +// The connector must only emit this for a scope whose validator it received +// from the SDK's source-cache lookup during this same sync. A replay for +// an unknown scope fails the sync: the connector has already skipped row +// generation, so there is nothing to fall back to. +type SourceCacheReplay struct { + state protoimpl.MessageState `protogen:"opaque.v1"` + xxx_hidden_ScopeKey string `protobuf:"bytes,1,opt,name=scope_key,json=scopeKey,proto3"` + xxx_hidden_CacheValidator string `protobuf:"bytes,2,opt,name=cache_validator,json=cacheValidator,proto3"` + xxx_hidden_Overlay bool `protobuf:"varint,3,opt,name=overlay,proto3"` + xxx_hidden_DeletedIds []string `protobuf:"bytes,4,rep,name=deleted_ids,json=deletedIds,proto3"` + xxx_hidden_DeletedPrincipalIds []string `protobuf:"bytes,5,rep,name=deleted_principal_ids,json=deletedPrincipalIds,proto3"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SourceCacheReplay) Reset() { + *x = SourceCacheReplay{} + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SourceCacheReplay) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SourceCacheReplay) ProtoMessage() {} + +func (x *SourceCacheReplay) ProtoReflect() protoreflect.Message { + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *SourceCacheReplay) GetScopeKey() string { + if x != nil { + return x.xxx_hidden_ScopeKey + } + return "" +} + +func (x *SourceCacheReplay) GetCacheValidator() string { + if x != nil { + return x.xxx_hidden_CacheValidator + } + return "" +} + +func (x *SourceCacheReplay) GetOverlay() bool { + if x != nil { + return x.xxx_hidden_Overlay + } + return false +} + +func (x *SourceCacheReplay) GetDeletedIds() []string { + if x != nil { + return x.xxx_hidden_DeletedIds + } + return nil +} + +func (x *SourceCacheReplay) GetDeletedPrincipalIds() []string { + if x != nil { + return x.xxx_hidden_DeletedPrincipalIds + } + return nil +} + +func (x *SourceCacheReplay) SetScopeKey(v string) { + x.xxx_hidden_ScopeKey = v +} + +func (x *SourceCacheReplay) SetCacheValidator(v string) { + x.xxx_hidden_CacheValidator = v +} + +func (x *SourceCacheReplay) SetOverlay(v bool) { + x.xxx_hidden_Overlay = v +} + +func (x *SourceCacheReplay) SetDeletedIds(v []string) { + x.xxx_hidden_DeletedIds = v +} + +func (x *SourceCacheReplay) SetDeletedPrincipalIds(v []string) { + x.xxx_hidden_DeletedPrincipalIds = v +} + +type SourceCacheReplay_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + ScopeKey string + // Validator to persist for this scope in the current sync. For an HTTP + // 304 this is the unchanged validator. For a delta query this is the NEW + // token; it may instead be supplied by the final overlay page's + // SourceCacheRecord.cache_validator, in which case this field may be left empty. + CacheValidator string + // When true, the response (and subsequent pages carrying + // SourceCacheRecord with the same scope_key) contains changed rows to + // upsert on top of the replayed base. When false the response must + // contain no rows for this scope. + // + // TRANSITIONAL: the SDK currently tolerates rows on a non-overlay + // replay page — it warns and upserts them with overlay semantics — + // while the model is validated against real providers. Do not rely on + // this: the tolerance will become a hard error, because a connector + // that fetched fresh rows and still attached a replay annotation keeps + // resurrecting the replayed base every sync (upstream deletions never + // propagate). + Overlay bool + // Public canonical IDs (grant/entitlement IDs, or resource BIDs for + // RowKindResources) to delete from the current sync after the replay + // copy and this page's upserts. Used for delta-query tombstones (e.g. + // Microsoft Graph @removed entries). Subsequent pages of the round + // carry their tombstones on SourceCacheRecord.deleted_ids. + DeletedIds []string + // Principal-scoped tombstones for this page; see + // SourceCacheRecord.deleted_principal_ids for semantics and + // preconditions. + DeletedPrincipalIds []string +} + +func (b0 SourceCacheReplay_builder) Build() *SourceCacheReplay { + m0 := &SourceCacheReplay{} + b, x := &b0, m0 + _, _ = b, x + x.xxx_hidden_ScopeKey = b.ScopeKey + x.xxx_hidden_CacheValidator = b.CacheValidator + x.xxx_hidden_Overlay = b.Overlay + x.xxx_hidden_DeletedIds = b.DeletedIds + x.xxx_hidden_DeletedPrincipalIds = b.DeletedPrincipalIds + return m0 +} + +// SourceCacheLookupOffer is attached by the SDK to list REQUESTS when the +// syncer can answer source-cache lookup asks: the connector declared +// SourceCacheCapability and a warm previous-sync lookup is installed. +// Its presence is the connector's permission to answer with +// SourceCacheLookupAsk instead of rows. Connectors with a direct lookup +// (in-process, subprocess) never need to defer and may ignore it. +type SourceCacheLookupOffer struct { + state protoimpl.MessageState `protogen:"opaque.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SourceCacheLookupOffer) Reset() { + *x = SourceCacheLookupOffer{} + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SourceCacheLookupOffer) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SourceCacheLookupOffer) ProtoMessage() {} + +func (x *SourceCacheLookupOffer) ProtoReflect() protoreflect.Message { + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +type SourceCacheLookupOffer_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + +} + +func (b0 SourceCacheLookupOffer_builder) Build() *SourceCacheLookupOffer { + m0 := &SourceCacheLookupOffer{} + b, x := &b0, m0 + _, _ = b, x + return m0 +} + +// SourceCacheLookupAsk is attached to a list RESPONSE in place of rows: +// the connector needs previous-sync validators before it can serve the +// page. An ask response must carry NO rows, NO next page token, and no +// other source-cache annotations; the syncer consumes it (it never +// reaches page handling), resolves every query, and re-invokes the same +// request with SourceCacheLookupAnswers attached. +// +// Only legal on responses to requests that carried +// SourceCacheLookupOffer. Bounces are capped per REQUEST (a page-token +// advance is a new request and resets the cap, so a multi-page action +// may legally ask once per page); a connector that keeps asking without +// progressing past the cap fails the sync loudly. +type SourceCacheLookupAsk struct { + state protoimpl.MessageState `protogen:"opaque.v1"` + xxx_hidden_Queries *[]*SourceCacheLookupAsk_Query `protobuf:"bytes,1,rep,name=queries,proto3"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SourceCacheLookupAsk) Reset() { + *x = SourceCacheLookupAsk{} + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SourceCacheLookupAsk) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SourceCacheLookupAsk) ProtoMessage() {} + +func (x *SourceCacheLookupAsk) ProtoReflect() protoreflect.Message { + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[4] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *SourceCacheLookupAsk) GetQueries() []*SourceCacheLookupAsk_Query { + if x != nil { + if x.xxx_hidden_Queries != nil { + return *x.xxx_hidden_Queries + } + } + return nil +} + +func (x *SourceCacheLookupAsk) SetQueries(v []*SourceCacheLookupAsk_Query) { + x.xxx_hidden_Queries = &v +} + +type SourceCacheLookupAsk_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + Queries []*SourceCacheLookupAsk_Query +} + +func (b0 SourceCacheLookupAsk_builder) Build() *SourceCacheLookupAsk { + m0 := &SourceCacheLookupAsk{} + b, x := &b0, m0 + _, _ = b, x + x.xxx_hidden_Queries = &b.Queries + return m0 +} + +// SourceCacheLookupAnswers is attached by the SDK to the re-invoked +// REQUEST, carrying the resolution of a prior ask's queries. +// +// Every query of the prior ask is answered — found=true with the +// previous sync's validator, or found=false meaning fetch fresh. A found +// answer whose validator does not fit the per-request answer size budget is +// delivered as found=false: the scope degrades to a cold fetch +// (correct, just slower) instead of staying unresolved, because answers +// accumulate across bounces and a validator that did not fit once can never +// fit a later re-invoke. An ABSENT query (a phase-2 lookup for a scope +// the connector did not ask before) means unresolved: the connector may +// ask again, subject to the bounce cap. +type SourceCacheLookupAnswers struct { + state protoimpl.MessageState `protogen:"opaque.v1"` + xxx_hidden_Answers *[]*SourceCacheLookupAnswers_Answer `protobuf:"bytes,1,rep,name=answers,proto3"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SourceCacheLookupAnswers) Reset() { + *x = SourceCacheLookupAnswers{} + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SourceCacheLookupAnswers) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SourceCacheLookupAnswers) ProtoMessage() {} + +func (x *SourceCacheLookupAnswers) ProtoReflect() protoreflect.Message { + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[5] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *SourceCacheLookupAnswers) GetAnswers() []*SourceCacheLookupAnswers_Answer { + if x != nil { + if x.xxx_hidden_Answers != nil { + return *x.xxx_hidden_Answers + } + } + return nil +} + +func (x *SourceCacheLookupAnswers) SetAnswers(v []*SourceCacheLookupAnswers_Answer) { + x.xxx_hidden_Answers = &v +} + +type SourceCacheLookupAnswers_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + // Cap = ask max_items (4096) × the per-request bounce cap (4): answers + // accumulate across bounces, so a request may legally carry every + // answer its asks produced. Keep in sync with + // sourcecache.MaxLookupBouncesPerRequest / maxAnswersPerMessage. + Answers []*SourceCacheLookupAnswers_Answer +} + +func (b0 SourceCacheLookupAnswers_builder) Build() *SourceCacheLookupAnswers { + m0 := &SourceCacheLookupAnswers{} + b, x := &b0, m0 + _, _ = b, x + x.xxx_hidden_Answers = &b.Answers + return m0 +} + +type SourceCacheLookupAsk_Query struct { + state protoimpl.MessageState `protogen:"opaque.v1"` + xxx_hidden_RowKind string `protobuf:"bytes,1,opt,name=row_kind,json=rowKind,proto3"` + xxx_hidden_ScopeKey string `protobuf:"bytes,2,opt,name=scope_key,json=scopeKey,proto3"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SourceCacheLookupAsk_Query) Reset() { + *x = SourceCacheLookupAsk_Query{} + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SourceCacheLookupAsk_Query) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SourceCacheLookupAsk_Query) ProtoMessage() {} + +func (x *SourceCacheLookupAsk_Query) ProtoReflect() protoreflect.Message { + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[6] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *SourceCacheLookupAsk_Query) GetRowKind() string { + if x != nil { + return x.xxx_hidden_RowKind + } + return "" +} + +func (x *SourceCacheLookupAsk_Query) GetScopeKey() string { + if x != nil { + return x.xxx_hidden_ScopeKey + } + return "" +} + +func (x *SourceCacheLookupAsk_Query) SetRowKind(v string) { + x.xxx_hidden_RowKind = v +} + +func (x *SourceCacheLookupAsk_Query) SetScopeKey(v string) { + x.xxx_hidden_ScopeKey = v +} + +type SourceCacheLookupAsk_Query_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + // One of the sourcecache.RowKind values: "resources", + // "entitlements", "grants". + RowKind string + ScopeKey string +} + +func (b0 SourceCacheLookupAsk_Query_builder) Build() *SourceCacheLookupAsk_Query { + m0 := &SourceCacheLookupAsk_Query{} + b, x := &b0, m0 + _, _ = b, x + x.xxx_hidden_RowKind = b.RowKind + x.xxx_hidden_ScopeKey = b.ScopeKey + return m0 +} + +type SourceCacheLookupAnswers_Answer struct { + state protoimpl.MessageState `protogen:"opaque.v1"` + xxx_hidden_RowKind string `protobuf:"bytes,1,opt,name=row_kind,json=rowKind,proto3"` + xxx_hidden_ScopeKey string `protobuf:"bytes,2,opt,name=scope_key,json=scopeKey,proto3"` + xxx_hidden_Found bool `protobuf:"varint,3,opt,name=found,proto3"` + xxx_hidden_CacheValidator string `protobuf:"bytes,4,opt,name=cache_validator,json=cacheValidator,proto3"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SourceCacheLookupAnswers_Answer) Reset() { + *x = SourceCacheLookupAnswers_Answer{} + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SourceCacheLookupAnswers_Answer) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SourceCacheLookupAnswers_Answer) ProtoMessage() {} + +func (x *SourceCacheLookupAnswers_Answer) ProtoReflect() protoreflect.Message { + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[7] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *SourceCacheLookupAnswers_Answer) GetRowKind() string { + if x != nil { + return x.xxx_hidden_RowKind + } + return "" +} + +func (x *SourceCacheLookupAnswers_Answer) GetScopeKey() string { + if x != nil { + return x.xxx_hidden_ScopeKey + } + return "" +} + +func (x *SourceCacheLookupAnswers_Answer) GetFound() bool { + if x != nil { + return x.xxx_hidden_Found + } + return false +} + +func (x *SourceCacheLookupAnswers_Answer) GetCacheValidator() string { + if x != nil { + return x.xxx_hidden_CacheValidator + } + return "" +} + +func (x *SourceCacheLookupAnswers_Answer) SetRowKind(v string) { + x.xxx_hidden_RowKind = v +} + +func (x *SourceCacheLookupAnswers_Answer) SetScopeKey(v string) { + x.xxx_hidden_ScopeKey = v +} + +func (x *SourceCacheLookupAnswers_Answer) SetFound(v bool) { + x.xxx_hidden_Found = v +} + +func (x *SourceCacheLookupAnswers_Answer) SetCacheValidator(v string) { + x.xxx_hidden_CacheValidator = v +} + +type SourceCacheLookupAnswers_Answer_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + RowKind string + ScopeKey string + Found bool + // The previous sync's validator; empty when found is false. Cap + // matches the lookup RPC (Graph delta tokens run long). + CacheValidator string +} + +func (b0 SourceCacheLookupAnswers_Answer_builder) Build() *SourceCacheLookupAnswers_Answer { + m0 := &SourceCacheLookupAnswers_Answer{} + b, x := &b0, m0 + _, _ = b, x + x.xxx_hidden_RowKind = b.RowKind + x.xxx_hidden_ScopeKey = b.ScopeKey + x.xxx_hidden_Found = b.Found + x.xxx_hidden_CacheValidator = b.CacheValidator + return m0 +} + +var File_c1_connector_v2_annotation_source_cache_proto protoreflect.FileDescriptor + +const file_c1_connector_v2_annotation_source_cache_proto_rawDesc = "" + + "\n" + + "-c1/connector/v2/annotation_source_cache.proto\x12\x0fc1.connector.v2\x1a\x17validate/validate.proto\"\x92\x02\n" + + "\x15SourceCacheCapability\x12?\n" + + "\x04mode\x18\x01 \x01(\x0e2+.c1.connector.v2.SourceCacheCapability.ModeR\x04mode\x126\n" + + "\x10cache_generation\x18\x02 \x01(\tB\v\xfaB\br\x06(\x80\x02\xd0\x01\x01R\x0fcacheGeneration\x12:\n" + + "\x12config_fingerprint\x18\x03 \x01(\tB\v\xfaB\br\x06(\x80\x02\xd0\x01\x01R\x11configFingerprint\"D\n" + + "\x04Mode\x12\x14\n" + + "\x10MODE_UNSPECIFIED\x10\x00\x12\x11\n" + + "\rMODE_DISABLED\x10\x01\x12\x13\n" + + "\x0fMODE_READ_WRITE\x10\x02\"\xae\x01\n" + + "\x11SourceCacheRecord\x12\x1b\n" + + "\tscope_key\x18\x01 \x01(\tR\bscopeKey\x12'\n" + + "\x0fcache_validator\x18\x02 \x01(\tR\x0ecacheValidator\x12\x1f\n" + + "\vdeleted_ids\x18\x03 \x03(\tR\n" + + "deletedIds\x122\n" + + "\x15deleted_principal_ids\x18\x04 \x03(\tR\x13deletedPrincipalIds\"\xc8\x01\n" + + "\x11SourceCacheReplay\x12\x1b\n" + + "\tscope_key\x18\x01 \x01(\tR\bscopeKey\x12'\n" + + "\x0fcache_validator\x18\x02 \x01(\tR\x0ecacheValidator\x12\x18\n" + + "\aoverlay\x18\x03 \x01(\bR\aoverlay\x12\x1f\n" + + "\vdeleted_ids\x18\x04 \x03(\tR\n" + + "deletedIds\x122\n" + + "\x15deleted_principal_ids\x18\x05 \x03(\tR\x13deletedPrincipalIds\"\x18\n" + + "\x16SourceCacheLookupOffer\"\xc2\x01\n" + + "\x14SourceCacheLookupAsk\x12R\n" + + "\aqueries\x18\x01 \x03(\v2+.c1.connector.v2.SourceCacheLookupAsk.QueryB\v\xfaB\b\x92\x01\x05\b\x01\x10\x80 R\aqueries\x1aV\n" + + "\x05Query\x12$\n" + + "\brow_kind\x18\x01 \x01(\tB\t\xfaB\x06r\x04 \x01(@R\arowKind\x12'\n" + + "\tscope_key\x18\x02 \x01(\tB\n" + + "\xfaB\ar\x05 \x01(\x80\x02R\bscopeKey\"\x99\x02\n" + + "\x18SourceCacheLookupAnswers\x12V\n" + + "\aanswers\x18\x01 \x03(\v20.c1.connector.v2.SourceCacheLookupAnswers.AnswerB\n" + + "\xfaB\a\x92\x01\x04\x10\x80\x80\x01R\aanswers\x1a\xa4\x01\n" + + "\x06Answer\x12$\n" + + "\brow_kind\x18\x01 \x01(\tB\t\xfaB\x06r\x04 \x01(@R\arowKind\x12'\n" + + "\tscope_key\x18\x02 \x01(\tB\n" + + "\xfaB\ar\x05 \x01(\x80\x02R\bscopeKey\x12\x14\n" + + "\x05found\x18\x03 \x01(\bR\x05found\x125\n" + + "\x0fcache_validator\x18\x04 \x01(\tB\f\xfaB\tr\a(\x80\x80\x04\xd0\x01\x01R\x0ecacheValidatorB6Z4github.com/conductorone/baton-sdk/pb/c1/connector/v2b\x06proto3" + +var file_c1_connector_v2_annotation_source_cache_proto_enumTypes = make([]protoimpl.EnumInfo, 1) +var file_c1_connector_v2_annotation_source_cache_proto_msgTypes = make([]protoimpl.MessageInfo, 8) +var file_c1_connector_v2_annotation_source_cache_proto_goTypes = []any{ + (SourceCacheCapability_Mode)(0), // 0: c1.connector.v2.SourceCacheCapability.Mode + (*SourceCacheCapability)(nil), // 1: c1.connector.v2.SourceCacheCapability + (*SourceCacheRecord)(nil), // 2: c1.connector.v2.SourceCacheRecord + (*SourceCacheReplay)(nil), // 3: c1.connector.v2.SourceCacheReplay + (*SourceCacheLookupOffer)(nil), // 4: c1.connector.v2.SourceCacheLookupOffer + (*SourceCacheLookupAsk)(nil), // 5: c1.connector.v2.SourceCacheLookupAsk + (*SourceCacheLookupAnswers)(nil), // 6: c1.connector.v2.SourceCacheLookupAnswers + (*SourceCacheLookupAsk_Query)(nil), // 7: c1.connector.v2.SourceCacheLookupAsk.Query + (*SourceCacheLookupAnswers_Answer)(nil), // 8: c1.connector.v2.SourceCacheLookupAnswers.Answer +} +var file_c1_connector_v2_annotation_source_cache_proto_depIdxs = []int32{ + 0, // 0: c1.connector.v2.SourceCacheCapability.mode:type_name -> c1.connector.v2.SourceCacheCapability.Mode + 7, // 1: c1.connector.v2.SourceCacheLookupAsk.queries:type_name -> c1.connector.v2.SourceCacheLookupAsk.Query + 8, // 2: c1.connector.v2.SourceCacheLookupAnswers.answers:type_name -> c1.connector.v2.SourceCacheLookupAnswers.Answer + 3, // [3:3] is the sub-list for method output_type + 3, // [3:3] is the sub-list for method input_type + 3, // [3:3] is the sub-list for extension type_name + 3, // [3:3] is the sub-list for extension extendee + 0, // [0:3] is the sub-list for field type_name +} + +func init() { file_c1_connector_v2_annotation_source_cache_proto_init() } +func file_c1_connector_v2_annotation_source_cache_proto_init() { + if File_c1_connector_v2_annotation_source_cache_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_c1_connector_v2_annotation_source_cache_proto_rawDesc), len(file_c1_connector_v2_annotation_source_cache_proto_rawDesc)), + NumEnums: 1, + NumMessages: 8, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_c1_connector_v2_annotation_source_cache_proto_goTypes, + DependencyIndexes: file_c1_connector_v2_annotation_source_cache_proto_depIdxs, + EnumInfos: file_c1_connector_v2_annotation_source_cache_proto_enumTypes, + MessageInfos: file_c1_connector_v2_annotation_source_cache_proto_msgTypes, + }.Build() + File_c1_connector_v2_annotation_source_cache_proto = out.File + file_c1_connector_v2_annotation_source_cache_proto_goTypes = nil + file_c1_connector_v2_annotation_source_cache_proto_depIdxs = nil +} diff --git a/pb/c1/connector/v2/annotation_type_scoped_entitlements.pb.go b/pb/c1/connector/v2/annotation_type_scoped_entitlements.pb.go new file mode 100644 index 000000000..e5f9a8b45 --- /dev/null +++ b/pb/c1/connector/v2/annotation_type_scoped_entitlements.pb.go @@ -0,0 +1,163 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.10 +// protoc (unknown) +// source: c1/connector/v2/annotation_type_scoped_entitlements.proto + +//go:build !protoopaque + +package v2 + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// Type-scoped entitlement ingestion. +// +// Some providers expose change/enumeration APIs whose natural unit is a +// whole collection (or a connector-defined shard of one), not a single +// resource — e.g. Microsoft Graph delta queries, where one stream yields +// entitlement definitions for up to 50 groups. Forcing that shape through +// the per-resource ListEntitlements fan-out costs one request per resource +// even when nothing changed. +// +// A resource type annotated with TypeScopedEntitlements is EXCLUDED from +// the per-resource entitlements fan-out. Instead the syncer issues +// ListEntitlements calls carrying this annotation on the REQUEST as the +// routing marker; the request's resource is a self-referential stub +// ({type, type} — wire validation requires a non-empty resource id and +// the stub identifies the type, never a real resource). The connector +// answers with entitlements for the whole type, across as many paginated +// cursors as it chooses to spawn (see EnqueuePageTokens in +// annotation_type_scoped_grants.proto). +// +// This is the entitlements-phase analogue of TypeScopedGrants: the +// connector takes over enumeration for the type, and every downstream +// behavior (storage, source-cache scopes/replay/tombstones, exclusion- +// group validation, rate limiting, page-token checkpointing) is unchanged +// because entitlements are self-describing rows. +// +// COMPLETENESS CONTRACT: a full sync of an annotated type must emit (or +// replay, via source-cache annotations) EVERY entitlement of that type. +// The syncer no longer visits each resource, so completeness rests +// entirely on the connector's enumeration. +// +// SKIP ANNOTATIONS: for an annotated type the syncer does not consult +// shouldSkipEntitlements on the type-scoped path. The connector owns skip +// semantics for resources of that type (omit rows; do not rely on +// per-resource SkipEntitlements* annotations to suppress the type-scoped +// call). +// +// ZERO-RESOURCE TYPES: the planner still enqueues the type-scoped action +// when the store holds no resources of the type. The connector must +// return an empty page (or EnqueuePageTokens that resolve empty), not an +// error — an empty type is a valid outcome. +// +// TARGETED SYNC: SyncTargetedResource must not enqueue a per-resource +// SyncEntitlementsOp for a type-scoped type. The type-scoped planner +// action is the only legal entitlements path; a targeted sync of one +// resource of an annotated type syncs the resource (and children) but +// leaves type-scoped entitlement enumeration to a full entitlements +// phase. +// +// ROUTING MARKER: TypeScopedEntitlements routes the ListEntitlements +// call; it implies no repair machinery of its own. It IS listed in the +// ingestion-invariant side-effect coverage map, covered by invariant I7 +// (entitlement→resource referential integrity): type-granularity scopes +// can carry entitlement rows for resources that vanished between +// enumerations, and the post-collection check is what catches that. +type TypeScopedEntitlements struct { + state protoimpl.MessageState `protogen:"hybrid.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *TypeScopedEntitlements) Reset() { + *x = TypeScopedEntitlements{} + mi := &file_c1_connector_v2_annotation_type_scoped_entitlements_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *TypeScopedEntitlements) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*TypeScopedEntitlements) ProtoMessage() {} + +func (x *TypeScopedEntitlements) ProtoReflect() protoreflect.Message { + mi := &file_c1_connector_v2_annotation_type_scoped_entitlements_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +type TypeScopedEntitlements_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + +} + +func (b0 TypeScopedEntitlements_builder) Build() *TypeScopedEntitlements { + m0 := &TypeScopedEntitlements{} + b, x := &b0, m0 + _, _ = b, x + return m0 +} + +var File_c1_connector_v2_annotation_type_scoped_entitlements_proto protoreflect.FileDescriptor + +const file_c1_connector_v2_annotation_type_scoped_entitlements_proto_rawDesc = "" + + "\n" + + "9c1/connector/v2/annotation_type_scoped_entitlements.proto\x12\x0fc1.connector.v2\"\x18\n" + + "\x16TypeScopedEntitlementsB6Z4github.com/conductorone/baton-sdk/pb/c1/connector/v2b\x06proto3" + +var file_c1_connector_v2_annotation_type_scoped_entitlements_proto_msgTypes = make([]protoimpl.MessageInfo, 1) +var file_c1_connector_v2_annotation_type_scoped_entitlements_proto_goTypes = []any{ + (*TypeScopedEntitlements)(nil), // 0: c1.connector.v2.TypeScopedEntitlements +} +var file_c1_connector_v2_annotation_type_scoped_entitlements_proto_depIdxs = []int32{ + 0, // [0:0] is the sub-list for method output_type + 0, // [0:0] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_c1_connector_v2_annotation_type_scoped_entitlements_proto_init() } +func file_c1_connector_v2_annotation_type_scoped_entitlements_proto_init() { + if File_c1_connector_v2_annotation_type_scoped_entitlements_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_c1_connector_v2_annotation_type_scoped_entitlements_proto_rawDesc), len(file_c1_connector_v2_annotation_type_scoped_entitlements_proto_rawDesc)), + NumEnums: 0, + NumMessages: 1, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_c1_connector_v2_annotation_type_scoped_entitlements_proto_goTypes, + DependencyIndexes: file_c1_connector_v2_annotation_type_scoped_entitlements_proto_depIdxs, + MessageInfos: file_c1_connector_v2_annotation_type_scoped_entitlements_proto_msgTypes, + }.Build() + File_c1_connector_v2_annotation_type_scoped_entitlements_proto = out.File + file_c1_connector_v2_annotation_type_scoped_entitlements_proto_goTypes = nil + file_c1_connector_v2_annotation_type_scoped_entitlements_proto_depIdxs = nil +} diff --git a/pb/c1/connector/v2/annotation_type_scoped_entitlements.pb.validate.go b/pb/c1/connector/v2/annotation_type_scoped_entitlements.pb.validate.go new file mode 100644 index 000000000..cbd3b3169 --- /dev/null +++ b/pb/c1/connector/v2/annotation_type_scoped_entitlements.pb.validate.go @@ -0,0 +1,138 @@ +// Code generated by protoc-gen-validate. DO NOT EDIT. +// source: c1/connector/v2/annotation_type_scoped_entitlements.proto + +package v2 + +import ( + "bytes" + "errors" + "fmt" + "net" + "net/mail" + "net/url" + "regexp" + "sort" + "strings" + "time" + "unicode/utf8" + + "google.golang.org/protobuf/types/known/anypb" +) + +// ensure the imports are used +var ( + _ = bytes.MinRead + _ = errors.New("") + _ = fmt.Print + _ = utf8.UTFMax + _ = (*regexp.Regexp)(nil) + _ = (*strings.Reader)(nil) + _ = net.IPv4len + _ = time.Duration(0) + _ = (*url.URL)(nil) + _ = (*mail.Address)(nil) + _ = anypb.Any{} + _ = sort.Sort +) + +// Validate checks the field values on TypeScopedEntitlements with the rules +// defined in the proto definition for this message. If any rules are +// violated, the first error encountered is returned, or nil if there are no violations. +func (m *TypeScopedEntitlements) Validate() error { + return m.validate(false) +} + +// ValidateAll checks the field values on TypeScopedEntitlements with the rules +// defined in the proto definition for this message. If any rules are +// violated, the result is a list of violation errors wrapped in +// TypeScopedEntitlementsMultiError, or nil if none found. +func (m *TypeScopedEntitlements) ValidateAll() error { + return m.validate(true) +} + +func (m *TypeScopedEntitlements) validate(all bool) error { + if m == nil { + return nil + } + + var errors []error + + if len(errors) > 0 { + return TypeScopedEntitlementsMultiError(errors) + } + + return nil +} + +// TypeScopedEntitlementsMultiError is an error wrapping multiple validation +// errors returned by TypeScopedEntitlements.ValidateAll() if the designated +// constraints aren't met. +type TypeScopedEntitlementsMultiError []error + +// Error returns a concatenation of all the error messages it wraps. +func (m TypeScopedEntitlementsMultiError) Error() string { + msgs := make([]string, 0, len(m)) + for _, err := range m { + msgs = append(msgs, err.Error()) + } + return strings.Join(msgs, "; ") +} + +// AllErrors returns a list of validation violation errors. +func (m TypeScopedEntitlementsMultiError) AllErrors() []error { return m } + +// TypeScopedEntitlementsValidationError is the validation error returned by +// TypeScopedEntitlements.Validate if the designated constraints aren't met. +type TypeScopedEntitlementsValidationError struct { + field string + reason string + cause error + key bool +} + +// Field function returns field value. +func (e TypeScopedEntitlementsValidationError) Field() string { return e.field } + +// Reason function returns reason value. +func (e TypeScopedEntitlementsValidationError) Reason() string { return e.reason } + +// Cause function returns cause value. +func (e TypeScopedEntitlementsValidationError) Cause() error { return e.cause } + +// Key function returns key value. +func (e TypeScopedEntitlementsValidationError) Key() bool { return e.key } + +// ErrorName returns error name. +func (e TypeScopedEntitlementsValidationError) ErrorName() string { + return "TypeScopedEntitlementsValidationError" +} + +// Error satisfies the builtin error interface +func (e TypeScopedEntitlementsValidationError) Error() string { + cause := "" + if e.cause != nil { + cause = fmt.Sprintf(" | caused by: %v", e.cause) + } + + key := "" + if e.key { + key = "key for " + } + + return fmt.Sprintf( + "invalid %sTypeScopedEntitlements.%s: %s%s", + key, + e.field, + e.reason, + cause) +} + +var _ error = TypeScopedEntitlementsValidationError{} + +var _ interface { + Field() string + Reason() string + Key() bool + Cause() error + ErrorName() string +} = TypeScopedEntitlementsValidationError{} diff --git a/pb/c1/connector/v2/annotation_type_scoped_entitlements_protoopaque.pb.go b/pb/c1/connector/v2/annotation_type_scoped_entitlements_protoopaque.pb.go new file mode 100644 index 000000000..c5773acd6 --- /dev/null +++ b/pb/c1/connector/v2/annotation_type_scoped_entitlements_protoopaque.pb.go @@ -0,0 +1,163 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.10 +// protoc (unknown) +// source: c1/connector/v2/annotation_type_scoped_entitlements.proto + +//go:build protoopaque + +package v2 + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// Type-scoped entitlement ingestion. +// +// Some providers expose change/enumeration APIs whose natural unit is a +// whole collection (or a connector-defined shard of one), not a single +// resource — e.g. Microsoft Graph delta queries, where one stream yields +// entitlement definitions for up to 50 groups. Forcing that shape through +// the per-resource ListEntitlements fan-out costs one request per resource +// even when nothing changed. +// +// A resource type annotated with TypeScopedEntitlements is EXCLUDED from +// the per-resource entitlements fan-out. Instead the syncer issues +// ListEntitlements calls carrying this annotation on the REQUEST as the +// routing marker; the request's resource is a self-referential stub +// ({type, type} — wire validation requires a non-empty resource id and +// the stub identifies the type, never a real resource). The connector +// answers with entitlements for the whole type, across as many paginated +// cursors as it chooses to spawn (see EnqueuePageTokens in +// annotation_type_scoped_grants.proto). +// +// This is the entitlements-phase analogue of TypeScopedGrants: the +// connector takes over enumeration for the type, and every downstream +// behavior (storage, source-cache scopes/replay/tombstones, exclusion- +// group validation, rate limiting, page-token checkpointing) is unchanged +// because entitlements are self-describing rows. +// +// COMPLETENESS CONTRACT: a full sync of an annotated type must emit (or +// replay, via source-cache annotations) EVERY entitlement of that type. +// The syncer no longer visits each resource, so completeness rests +// entirely on the connector's enumeration. +// +// SKIP ANNOTATIONS: for an annotated type the syncer does not consult +// shouldSkipEntitlements on the type-scoped path. The connector owns skip +// semantics for resources of that type (omit rows; do not rely on +// per-resource SkipEntitlements* annotations to suppress the type-scoped +// call). +// +// ZERO-RESOURCE TYPES: the planner still enqueues the type-scoped action +// when the store holds no resources of the type. The connector must +// return an empty page (or EnqueuePageTokens that resolve empty), not an +// error — an empty type is a valid outcome. +// +// TARGETED SYNC: SyncTargetedResource must not enqueue a per-resource +// SyncEntitlementsOp for a type-scoped type. The type-scoped planner +// action is the only legal entitlements path; a targeted sync of one +// resource of an annotated type syncs the resource (and children) but +// leaves type-scoped entitlement enumeration to a full entitlements +// phase. +// +// ROUTING MARKER: TypeScopedEntitlements routes the ListEntitlements +// call; it implies no repair machinery of its own. It IS listed in the +// ingestion-invariant side-effect coverage map, covered by invariant I7 +// (entitlement→resource referential integrity): type-granularity scopes +// can carry entitlement rows for resources that vanished between +// enumerations, and the post-collection check is what catches that. +type TypeScopedEntitlements struct { + state protoimpl.MessageState `protogen:"opaque.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *TypeScopedEntitlements) Reset() { + *x = TypeScopedEntitlements{} + mi := &file_c1_connector_v2_annotation_type_scoped_entitlements_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *TypeScopedEntitlements) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*TypeScopedEntitlements) ProtoMessage() {} + +func (x *TypeScopedEntitlements) ProtoReflect() protoreflect.Message { + mi := &file_c1_connector_v2_annotation_type_scoped_entitlements_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +type TypeScopedEntitlements_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + +} + +func (b0 TypeScopedEntitlements_builder) Build() *TypeScopedEntitlements { + m0 := &TypeScopedEntitlements{} + b, x := &b0, m0 + _, _ = b, x + return m0 +} + +var File_c1_connector_v2_annotation_type_scoped_entitlements_proto protoreflect.FileDescriptor + +const file_c1_connector_v2_annotation_type_scoped_entitlements_proto_rawDesc = "" + + "\n" + + "9c1/connector/v2/annotation_type_scoped_entitlements.proto\x12\x0fc1.connector.v2\"\x18\n" + + "\x16TypeScopedEntitlementsB6Z4github.com/conductorone/baton-sdk/pb/c1/connector/v2b\x06proto3" + +var file_c1_connector_v2_annotation_type_scoped_entitlements_proto_msgTypes = make([]protoimpl.MessageInfo, 1) +var file_c1_connector_v2_annotation_type_scoped_entitlements_proto_goTypes = []any{ + (*TypeScopedEntitlements)(nil), // 0: c1.connector.v2.TypeScopedEntitlements +} +var file_c1_connector_v2_annotation_type_scoped_entitlements_proto_depIdxs = []int32{ + 0, // [0:0] is the sub-list for method output_type + 0, // [0:0] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_c1_connector_v2_annotation_type_scoped_entitlements_proto_init() } +func file_c1_connector_v2_annotation_type_scoped_entitlements_proto_init() { + if File_c1_connector_v2_annotation_type_scoped_entitlements_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_c1_connector_v2_annotation_type_scoped_entitlements_proto_rawDesc), len(file_c1_connector_v2_annotation_type_scoped_entitlements_proto_rawDesc)), + NumEnums: 0, + NumMessages: 1, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_c1_connector_v2_annotation_type_scoped_entitlements_proto_goTypes, + DependencyIndexes: file_c1_connector_v2_annotation_type_scoped_entitlements_proto_depIdxs, + MessageInfos: file_c1_connector_v2_annotation_type_scoped_entitlements_proto_msgTypes, + }.Build() + File_c1_connector_v2_annotation_type_scoped_entitlements_proto = out.File + file_c1_connector_v2_annotation_type_scoped_entitlements_proto_goTypes = nil + file_c1_connector_v2_annotation_type_scoped_entitlements_proto_depIdxs = nil +} diff --git a/pb/c1/connector/v2/annotation_type_scoped_grants.pb.go b/pb/c1/connector/v2/annotation_type_scoped_grants.pb.go new file mode 100644 index 000000000..c5e1bf688 --- /dev/null +++ b/pb/c1/connector/v2/annotation_type_scoped_grants.pb.go @@ -0,0 +1,279 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.10 +// protoc (unknown) +// source: c1/connector/v2/annotation_type_scoped_grants.proto + +//go:build !protoopaque + +package v2 + +import ( + _ "github.com/envoyproxy/protoc-gen-validate/validate" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// Type-scoped grant ingestion. +// +// Some providers expose change/enumeration APIs whose natural unit is a +// whole collection (or a connector-defined shard of one), not a single +// resource — e.g. Microsoft Graph delta queries, where one stream yields +// membership changes for up to 50 groups. Forcing that shape through the +// per-resource ListGrants fan-out costs one request per resource even when +// nothing changed. +// +// A resource type annotated with TypeScopedGrants is EXCLUDED from the +// per-resource grants fan-out. Instead the syncer issues ListGrants calls +// carrying this annotation on the REQUEST as the routing marker; the +// request's resource is a self-referential stub ({type, type} — wire +// validation requires a non-empty resource id and the stub identifies +// the type, never a real resource). The connector answers with grants +// for the whole type, across as many paginated cursors as it chooses to +// spawn (see EnqueuePageTokens). +// +// The connector takes over enumeration for the type (the same contract +// as TypeScopedEntitlements in +// annotation_type_scoped_entitlements.proto), and every downstream behavior +// (storage, source-cache scopes/replay/tombstones, grant expansion, rate +// limiting, page-token checkpointing) is unchanged because grants are +// self-describing rows. +// +// COMPLETENESS CONTRACT: a full sync of an annotated type must emit (or +// replay, via source-cache annotations) EVERY grant of that type. The +// syncer no longer visits each resource, so completeness rests entirely on +// the connector's enumeration. +// +// SKIP ANNOTATIONS: for an annotated type the syncer does not consult +// shouldSkipGrants on the type-scoped path. The connector owns skip +// semantics for resources of that type. +// +// ZERO-RESOURCE TYPES: the planner still enqueues the type-scoped action +// when the store holds no resources of the type. The connector must +// return an empty page (or EnqueuePageTokens that resolve empty), not an error. +// +// TARGETED SYNC: SyncTargetedResource must not enqueue a per-resource +// SyncGrantsOp for a type-scoped type. Type-scoped grant enumeration is +// left to a full grants phase. +// +// ROUTING MARKER: TypeScopedGrants routes ListGrants; it implies no +// repair machinery of its own. It IS listed in the ingestion-invariant +// side-effect coverage map, covered by invariant I8 (grant→entitlement +// referential integrity): independently-fresh type scopes can strand +// grant references against a refreshed entitlement set, and the +// post-collection check is what catches that. +type TypeScopedGrants struct { + state protoimpl.MessageState `protogen:"hybrid.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *TypeScopedGrants) Reset() { + *x = TypeScopedGrants{} + mi := &file_c1_connector_v2_annotation_type_scoped_grants_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *TypeScopedGrants) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*TypeScopedGrants) ProtoMessage() {} + +func (x *TypeScopedGrants) ProtoReflect() protoreflect.Message { + mi := &file_c1_connector_v2_annotation_type_scoped_grants_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +type TypeScopedGrants_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + +} + +func (b0 TypeScopedGrants_builder) Build() *TypeScopedGrants { + m0 := &TypeScopedGrants{} + b, x := &b0, m0 + _, _ = b, x + return m0 +} + +// EnqueuePageTokens is attached to a ListGrants or ListEntitlements response to +// enqueue additional independent sibling cursors. Each token is delivered +// back to the connector as the page token of its own action — scheduled by +// the syncer's worker pool, rate-limited, and checkpointed like any other +// pagination. Honored on BOTH type-scoped and per-resource ListGrants / +// ListEntitlements responses; the spawned actions inherit the response's +// resource identity (type only for type-scoped calls, type+resource for +// per-resource calls). +// +// Typical uses: +// +// - Type-scoped planning: the first call computes shard assignments — +// e.g. one 50-id delta filter chunk per cursor — returns no rows, and +// spawns one cursor per shard. Cold enumeration then parallelizes +// across shards instead of serializing through one stream. +// - Per-resource parallel warm revalidation (page-numbered APIs): on the +// first page of a collection the connector already knows every other +// page's URL and stored validator, so it answers page one and spawns +// pages 2..N; the worker pool revalidates them concurrently instead +// of paying one round trip per page serially. +// +// Spawned cursors are ORDINARY pages that happen to be enqueued eagerly. +// The SDK assumes nothing about how they resolve: a spawned page may hit +// its source-cache lookup and replay, miss and fetch cold (e.g. a page +// boundary shifted since the last sync), and may continue a chain via its +// response's next page token. Any page of a fan-out may itself spawn. +// +// Progress accounting counts a resource as covered when its ORIGIN +// action's chain ends; spawned siblings never double-count it. +// +// Tokens are opaque to the SDK. They must be self-contained: a cursor may +// execute after a suspend/resume, on a different worker, so anything the +// connector needs to serve the cursor must be inside the token (or +// re-derivable from it) — for per-resource spawns the resource identity +// rides the action, so tokens only need the page coordinate. +type EnqueuePageTokens struct { + state protoimpl.MessageState `protogen:"hybrid.v1"` + // Page tokens for the sibling cursors to enqueue, one action each. + // Capped per response (spawned actions persist in the checkpointed + // state token); chain additional spawns across pages to fan out wider. + PageTokens []string `protobuf:"bytes,1,rep,name=page_tokens,json=pageTokens,proto3" json:"page_tokens,omitempty"` + // Optional connector estimate of total rows across all cursors of this + // type, for progress reporting. Zero means unknown. + EstimatedTotal int64 `protobuf:"varint,2,opt,name=estimated_total,json=estimatedTotal,proto3" json:"estimated_total,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *EnqueuePageTokens) Reset() { + *x = EnqueuePageTokens{} + mi := &file_c1_connector_v2_annotation_type_scoped_grants_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *EnqueuePageTokens) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*EnqueuePageTokens) ProtoMessage() {} + +func (x *EnqueuePageTokens) ProtoReflect() protoreflect.Message { + mi := &file_c1_connector_v2_annotation_type_scoped_grants_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *EnqueuePageTokens) GetPageTokens() []string { + if x != nil { + return x.PageTokens + } + return nil +} + +func (x *EnqueuePageTokens) GetEstimatedTotal() int64 { + if x != nil { + return x.EstimatedTotal + } + return 0 +} + +func (x *EnqueuePageTokens) SetPageTokens(v []string) { + x.PageTokens = v +} + +func (x *EnqueuePageTokens) SetEstimatedTotal(v int64) { + x.EstimatedTotal = v +} + +type EnqueuePageTokens_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + // Page tokens for the sibling cursors to enqueue, one action each. + // Capped per response (spawned actions persist in the checkpointed + // state token); chain additional spawns across pages to fan out wider. + PageTokens []string + // Optional connector estimate of total rows across all cursors of this + // type, for progress reporting. Zero means unknown. + EstimatedTotal int64 +} + +func (b0 EnqueuePageTokens_builder) Build() *EnqueuePageTokens { + m0 := &EnqueuePageTokens{} + b, x := &b0, m0 + _, _ = b, x + x.PageTokens = b.PageTokens + x.EstimatedTotal = b.EstimatedTotal + return m0 +} + +var File_c1_connector_v2_annotation_type_scoped_grants_proto protoreflect.FileDescriptor + +const file_c1_connector_v2_annotation_type_scoped_grants_proto_rawDesc = "" + + "\n" + + "3c1/connector/v2/annotation_type_scoped_grants.proto\x12\x0fc1.connector.v2\x1a\x17validate/validate.proto\"\x12\n" + + "\x10TypeScopedGrants\"r\n" + + "\x11EnqueuePageTokens\x124\n" + + "\vpage_tokens\x18\x01 \x03(\tB\x13\xfaB\x10\x92\x01\r\x10\x80\b\"\br\x06\x10\x01\x18\x80\x80@R\n" + + "pageTokens\x12'\n" + + "\x0festimated_total\x18\x02 \x01(\x03R\x0eestimatedTotalB6Z4github.com/conductorone/baton-sdk/pb/c1/connector/v2b\x06proto3" + +var file_c1_connector_v2_annotation_type_scoped_grants_proto_msgTypes = make([]protoimpl.MessageInfo, 2) +var file_c1_connector_v2_annotation_type_scoped_grants_proto_goTypes = []any{ + (*TypeScopedGrants)(nil), // 0: c1.connector.v2.TypeScopedGrants + (*EnqueuePageTokens)(nil), // 1: c1.connector.v2.EnqueuePageTokens +} +var file_c1_connector_v2_annotation_type_scoped_grants_proto_depIdxs = []int32{ + 0, // [0:0] is the sub-list for method output_type + 0, // [0:0] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_c1_connector_v2_annotation_type_scoped_grants_proto_init() } +func file_c1_connector_v2_annotation_type_scoped_grants_proto_init() { + if File_c1_connector_v2_annotation_type_scoped_grants_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_c1_connector_v2_annotation_type_scoped_grants_proto_rawDesc), len(file_c1_connector_v2_annotation_type_scoped_grants_proto_rawDesc)), + NumEnums: 0, + NumMessages: 2, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_c1_connector_v2_annotation_type_scoped_grants_proto_goTypes, + DependencyIndexes: file_c1_connector_v2_annotation_type_scoped_grants_proto_depIdxs, + MessageInfos: file_c1_connector_v2_annotation_type_scoped_grants_proto_msgTypes, + }.Build() + File_c1_connector_v2_annotation_type_scoped_grants_proto = out.File + file_c1_connector_v2_annotation_type_scoped_grants_proto_goTypes = nil + file_c1_connector_v2_annotation_type_scoped_grants_proto_depIdxs = nil +} diff --git a/pb/c1/connector/v2/annotation_type_scoped_grants.pb.validate.go b/pb/c1/connector/v2/annotation_type_scoped_grants.pb.validate.go new file mode 100644 index 000000000..ee529553a --- /dev/null +++ b/pb/c1/connector/v2/annotation_type_scoped_grants.pb.validate.go @@ -0,0 +1,267 @@ +// Code generated by protoc-gen-validate. DO NOT EDIT. +// source: c1/connector/v2/annotation_type_scoped_grants.proto + +package v2 + +import ( + "bytes" + "errors" + "fmt" + "net" + "net/mail" + "net/url" + "regexp" + "sort" + "strings" + "time" + "unicode/utf8" + + "google.golang.org/protobuf/types/known/anypb" +) + +// ensure the imports are used +var ( + _ = bytes.MinRead + _ = errors.New("") + _ = fmt.Print + _ = utf8.UTFMax + _ = (*regexp.Regexp)(nil) + _ = (*strings.Reader)(nil) + _ = net.IPv4len + _ = time.Duration(0) + _ = (*url.URL)(nil) + _ = (*mail.Address)(nil) + _ = anypb.Any{} + _ = sort.Sort +) + +// Validate checks the field values on TypeScopedGrants with the rules defined +// in the proto definition for this message. If any rules are violated, the +// first error encountered is returned, or nil if there are no violations. +func (m *TypeScopedGrants) Validate() error { + return m.validate(false) +} + +// ValidateAll checks the field values on TypeScopedGrants with the rules +// defined in the proto definition for this message. If any rules are +// violated, the result is a list of violation errors wrapped in +// TypeScopedGrantsMultiError, or nil if none found. +func (m *TypeScopedGrants) ValidateAll() error { + return m.validate(true) +} + +func (m *TypeScopedGrants) validate(all bool) error { + if m == nil { + return nil + } + + var errors []error + + if len(errors) > 0 { + return TypeScopedGrantsMultiError(errors) + } + + return nil +} + +// TypeScopedGrantsMultiError is an error wrapping multiple validation errors +// returned by TypeScopedGrants.ValidateAll() if the designated constraints +// aren't met. +type TypeScopedGrantsMultiError []error + +// Error returns a concatenation of all the error messages it wraps. +func (m TypeScopedGrantsMultiError) Error() string { + msgs := make([]string, 0, len(m)) + for _, err := range m { + msgs = append(msgs, err.Error()) + } + return strings.Join(msgs, "; ") +} + +// AllErrors returns a list of validation violation errors. +func (m TypeScopedGrantsMultiError) AllErrors() []error { return m } + +// TypeScopedGrantsValidationError is the validation error returned by +// TypeScopedGrants.Validate if the designated constraints aren't met. +type TypeScopedGrantsValidationError struct { + field string + reason string + cause error + key bool +} + +// Field function returns field value. +func (e TypeScopedGrantsValidationError) Field() string { return e.field } + +// Reason function returns reason value. +func (e TypeScopedGrantsValidationError) Reason() string { return e.reason } + +// Cause function returns cause value. +func (e TypeScopedGrantsValidationError) Cause() error { return e.cause } + +// Key function returns key value. +func (e TypeScopedGrantsValidationError) Key() bool { return e.key } + +// ErrorName returns error name. +func (e TypeScopedGrantsValidationError) ErrorName() string { return "TypeScopedGrantsValidationError" } + +// Error satisfies the builtin error interface +func (e TypeScopedGrantsValidationError) Error() string { + cause := "" + if e.cause != nil { + cause = fmt.Sprintf(" | caused by: %v", e.cause) + } + + key := "" + if e.key { + key = "key for " + } + + return fmt.Sprintf( + "invalid %sTypeScopedGrants.%s: %s%s", + key, + e.field, + e.reason, + cause) +} + +var _ error = TypeScopedGrantsValidationError{} + +var _ interface { + Field() string + Reason() string + Key() bool + Cause() error + ErrorName() string +} = TypeScopedGrantsValidationError{} + +// Validate checks the field values on EnqueuePageTokens with the rules defined +// in the proto definition for this message. If any rules are violated, the +// first error encountered is returned, or nil if there are no violations. +func (m *EnqueuePageTokens) Validate() error { + return m.validate(false) +} + +// ValidateAll checks the field values on EnqueuePageTokens with the rules +// defined in the proto definition for this message. If any rules are +// violated, the result is a list of violation errors wrapped in +// EnqueuePageTokensMultiError, or nil if none found. +func (m *EnqueuePageTokens) ValidateAll() error { + return m.validate(true) +} + +func (m *EnqueuePageTokens) validate(all bool) error { + if m == nil { + return nil + } + + var errors []error + + if len(m.GetPageTokens()) > 1024 { + err := EnqueuePageTokensValidationError{ + field: "PageTokens", + reason: "value must contain no more than 1024 item(s)", + } + if !all { + return err + } + errors = append(errors, err) + } + + for idx, item := range m.GetPageTokens() { + _, _ = idx, item + + if l := utf8.RuneCountInString(item); l < 1 || l > 1048576 { + err := EnqueuePageTokensValidationError{ + field: fmt.Sprintf("PageTokens[%v]", idx), + reason: "value length must be between 1 and 1048576 runes, inclusive", + } + if !all { + return err + } + errors = append(errors, err) + } + + } + + // no validation rules for EstimatedTotal + + if len(errors) > 0 { + return EnqueuePageTokensMultiError(errors) + } + + return nil +} + +// EnqueuePageTokensMultiError is an error wrapping multiple validation errors +// returned by EnqueuePageTokens.ValidateAll() if the designated constraints +// aren't met. +type EnqueuePageTokensMultiError []error + +// Error returns a concatenation of all the error messages it wraps. +func (m EnqueuePageTokensMultiError) Error() string { + msgs := make([]string, 0, len(m)) + for _, err := range m { + msgs = append(msgs, err.Error()) + } + return strings.Join(msgs, "; ") +} + +// AllErrors returns a list of validation violation errors. +func (m EnqueuePageTokensMultiError) AllErrors() []error { return m } + +// EnqueuePageTokensValidationError is the validation error returned by +// EnqueuePageTokens.Validate if the designated constraints aren't met. +type EnqueuePageTokensValidationError struct { + field string + reason string + cause error + key bool +} + +// Field function returns field value. +func (e EnqueuePageTokensValidationError) Field() string { return e.field } + +// Reason function returns reason value. +func (e EnqueuePageTokensValidationError) Reason() string { return e.reason } + +// Cause function returns cause value. +func (e EnqueuePageTokensValidationError) Cause() error { return e.cause } + +// Key function returns key value. +func (e EnqueuePageTokensValidationError) Key() bool { return e.key } + +// ErrorName returns error name. +func (e EnqueuePageTokensValidationError) ErrorName() string { + return "EnqueuePageTokensValidationError" +} + +// Error satisfies the builtin error interface +func (e EnqueuePageTokensValidationError) Error() string { + cause := "" + if e.cause != nil { + cause = fmt.Sprintf(" | caused by: %v", e.cause) + } + + key := "" + if e.key { + key = "key for " + } + + return fmt.Sprintf( + "invalid %sEnqueuePageTokens.%s: %s%s", + key, + e.field, + e.reason, + cause) +} + +var _ error = EnqueuePageTokensValidationError{} + +var _ interface { + Field() string + Reason() string + Key() bool + Cause() error + ErrorName() string +} = EnqueuePageTokensValidationError{} diff --git a/pb/c1/connector/v2/annotation_type_scoped_grants_protoopaque.pb.go b/pb/c1/connector/v2/annotation_type_scoped_grants_protoopaque.pb.go new file mode 100644 index 000000000..4a30650a3 --- /dev/null +++ b/pb/c1/connector/v2/annotation_type_scoped_grants_protoopaque.pb.go @@ -0,0 +1,274 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.10 +// protoc (unknown) +// source: c1/connector/v2/annotation_type_scoped_grants.proto + +//go:build protoopaque + +package v2 + +import ( + _ "github.com/envoyproxy/protoc-gen-validate/validate" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// Type-scoped grant ingestion. +// +// Some providers expose change/enumeration APIs whose natural unit is a +// whole collection (or a connector-defined shard of one), not a single +// resource — e.g. Microsoft Graph delta queries, where one stream yields +// membership changes for up to 50 groups. Forcing that shape through the +// per-resource ListGrants fan-out costs one request per resource even when +// nothing changed. +// +// A resource type annotated with TypeScopedGrants is EXCLUDED from the +// per-resource grants fan-out. Instead the syncer issues ListGrants calls +// carrying this annotation on the REQUEST as the routing marker; the +// request's resource is a self-referential stub ({type, type} — wire +// validation requires a non-empty resource id and the stub identifies +// the type, never a real resource). The connector answers with grants +// for the whole type, across as many paginated cursors as it chooses to +// spawn (see EnqueuePageTokens). +// +// The connector takes over enumeration for the type (the same contract +// as TypeScopedEntitlements in +// annotation_type_scoped_entitlements.proto), and every downstream behavior +// (storage, source-cache scopes/replay/tombstones, grant expansion, rate +// limiting, page-token checkpointing) is unchanged because grants are +// self-describing rows. +// +// COMPLETENESS CONTRACT: a full sync of an annotated type must emit (or +// replay, via source-cache annotations) EVERY grant of that type. The +// syncer no longer visits each resource, so completeness rests entirely on +// the connector's enumeration. +// +// SKIP ANNOTATIONS: for an annotated type the syncer does not consult +// shouldSkipGrants on the type-scoped path. The connector owns skip +// semantics for resources of that type. +// +// ZERO-RESOURCE TYPES: the planner still enqueues the type-scoped action +// when the store holds no resources of the type. The connector must +// return an empty page (or EnqueuePageTokens that resolve empty), not an error. +// +// TARGETED SYNC: SyncTargetedResource must not enqueue a per-resource +// SyncGrantsOp for a type-scoped type. Type-scoped grant enumeration is +// left to a full grants phase. +// +// ROUTING MARKER: TypeScopedGrants routes ListGrants; it implies no +// repair machinery of its own. It IS listed in the ingestion-invariant +// side-effect coverage map, covered by invariant I8 (grant→entitlement +// referential integrity): independently-fresh type scopes can strand +// grant references against a refreshed entitlement set, and the +// post-collection check is what catches that. +type TypeScopedGrants struct { + state protoimpl.MessageState `protogen:"opaque.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *TypeScopedGrants) Reset() { + *x = TypeScopedGrants{} + mi := &file_c1_connector_v2_annotation_type_scoped_grants_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *TypeScopedGrants) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*TypeScopedGrants) ProtoMessage() {} + +func (x *TypeScopedGrants) ProtoReflect() protoreflect.Message { + mi := &file_c1_connector_v2_annotation_type_scoped_grants_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +type TypeScopedGrants_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + +} + +func (b0 TypeScopedGrants_builder) Build() *TypeScopedGrants { + m0 := &TypeScopedGrants{} + b, x := &b0, m0 + _, _ = b, x + return m0 +} + +// EnqueuePageTokens is attached to a ListGrants or ListEntitlements response to +// enqueue additional independent sibling cursors. Each token is delivered +// back to the connector as the page token of its own action — scheduled by +// the syncer's worker pool, rate-limited, and checkpointed like any other +// pagination. Honored on BOTH type-scoped and per-resource ListGrants / +// ListEntitlements responses; the spawned actions inherit the response's +// resource identity (type only for type-scoped calls, type+resource for +// per-resource calls). +// +// Typical uses: +// +// - Type-scoped planning: the first call computes shard assignments — +// e.g. one 50-id delta filter chunk per cursor — returns no rows, and +// spawns one cursor per shard. Cold enumeration then parallelizes +// across shards instead of serializing through one stream. +// - Per-resource parallel warm revalidation (page-numbered APIs): on the +// first page of a collection the connector already knows every other +// page's URL and stored validator, so it answers page one and spawns +// pages 2..N; the worker pool revalidates them concurrently instead +// of paying one round trip per page serially. +// +// Spawned cursors are ORDINARY pages that happen to be enqueued eagerly. +// The SDK assumes nothing about how they resolve: a spawned page may hit +// its source-cache lookup and replay, miss and fetch cold (e.g. a page +// boundary shifted since the last sync), and may continue a chain via its +// response's next page token. Any page of a fan-out may itself spawn. +// +// Progress accounting counts a resource as covered when its ORIGIN +// action's chain ends; spawned siblings never double-count it. +// +// Tokens are opaque to the SDK. They must be self-contained: a cursor may +// execute after a suspend/resume, on a different worker, so anything the +// connector needs to serve the cursor must be inside the token (or +// re-derivable from it) — for per-resource spawns the resource identity +// rides the action, so tokens only need the page coordinate. +type EnqueuePageTokens struct { + state protoimpl.MessageState `protogen:"opaque.v1"` + xxx_hidden_PageTokens []string `protobuf:"bytes,1,rep,name=page_tokens,json=pageTokens,proto3"` + xxx_hidden_EstimatedTotal int64 `protobuf:"varint,2,opt,name=estimated_total,json=estimatedTotal,proto3"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *EnqueuePageTokens) Reset() { + *x = EnqueuePageTokens{} + mi := &file_c1_connector_v2_annotation_type_scoped_grants_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *EnqueuePageTokens) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*EnqueuePageTokens) ProtoMessage() {} + +func (x *EnqueuePageTokens) ProtoReflect() protoreflect.Message { + mi := &file_c1_connector_v2_annotation_type_scoped_grants_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *EnqueuePageTokens) GetPageTokens() []string { + if x != nil { + return x.xxx_hidden_PageTokens + } + return nil +} + +func (x *EnqueuePageTokens) GetEstimatedTotal() int64 { + if x != nil { + return x.xxx_hidden_EstimatedTotal + } + return 0 +} + +func (x *EnqueuePageTokens) SetPageTokens(v []string) { + x.xxx_hidden_PageTokens = v +} + +func (x *EnqueuePageTokens) SetEstimatedTotal(v int64) { + x.xxx_hidden_EstimatedTotal = v +} + +type EnqueuePageTokens_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + // Page tokens for the sibling cursors to enqueue, one action each. + // Capped per response (spawned actions persist in the checkpointed + // state token); chain additional spawns across pages to fan out wider. + PageTokens []string + // Optional connector estimate of total rows across all cursors of this + // type, for progress reporting. Zero means unknown. + EstimatedTotal int64 +} + +func (b0 EnqueuePageTokens_builder) Build() *EnqueuePageTokens { + m0 := &EnqueuePageTokens{} + b, x := &b0, m0 + _, _ = b, x + x.xxx_hidden_PageTokens = b.PageTokens + x.xxx_hidden_EstimatedTotal = b.EstimatedTotal + return m0 +} + +var File_c1_connector_v2_annotation_type_scoped_grants_proto protoreflect.FileDescriptor + +const file_c1_connector_v2_annotation_type_scoped_grants_proto_rawDesc = "" + + "\n" + + "3c1/connector/v2/annotation_type_scoped_grants.proto\x12\x0fc1.connector.v2\x1a\x17validate/validate.proto\"\x12\n" + + "\x10TypeScopedGrants\"r\n" + + "\x11EnqueuePageTokens\x124\n" + + "\vpage_tokens\x18\x01 \x03(\tB\x13\xfaB\x10\x92\x01\r\x10\x80\b\"\br\x06\x10\x01\x18\x80\x80@R\n" + + "pageTokens\x12'\n" + + "\x0festimated_total\x18\x02 \x01(\x03R\x0eestimatedTotalB6Z4github.com/conductorone/baton-sdk/pb/c1/connector/v2b\x06proto3" + +var file_c1_connector_v2_annotation_type_scoped_grants_proto_msgTypes = make([]protoimpl.MessageInfo, 2) +var file_c1_connector_v2_annotation_type_scoped_grants_proto_goTypes = []any{ + (*TypeScopedGrants)(nil), // 0: c1.connector.v2.TypeScopedGrants + (*EnqueuePageTokens)(nil), // 1: c1.connector.v2.EnqueuePageTokens +} +var file_c1_connector_v2_annotation_type_scoped_grants_proto_depIdxs = []int32{ + 0, // [0:0] is the sub-list for method output_type + 0, // [0:0] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_c1_connector_v2_annotation_type_scoped_grants_proto_init() } +func file_c1_connector_v2_annotation_type_scoped_grants_proto_init() { + if File_c1_connector_v2_annotation_type_scoped_grants_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_c1_connector_v2_annotation_type_scoped_grants_proto_rawDesc), len(file_c1_connector_v2_annotation_type_scoped_grants_proto_rawDesc)), + NumEnums: 0, + NumMessages: 2, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_c1_connector_v2_annotation_type_scoped_grants_proto_goTypes, + DependencyIndexes: file_c1_connector_v2_annotation_type_scoped_grants_proto_depIdxs, + MessageInfos: file_c1_connector_v2_annotation_type_scoped_grants_proto_msgTypes, + }.Build() + File_c1_connector_v2_annotation_type_scoped_grants_proto = out.File + file_c1_connector_v2_annotation_type_scoped_grants_proto_goTypes = nil + file_c1_connector_v2_annotation_type_scoped_grants_proto_depIdxs = nil +} diff --git a/pb/c1/connectorapi/baton/v1/source_cache.pb.go b/pb/c1/connectorapi/baton/v1/source_cache.pb.go new file mode 100644 index 000000000..0b2bc4823 --- /dev/null +++ b/pb/c1/connectorapi/baton/v1/source_cache.pb.go @@ -0,0 +1,248 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.10 +// protoc (unknown) +// source: c1/connectorapi/baton/v1/source_cache.proto + +//go:build !protoopaque + +package v1 + +import ( + _ "github.com/envoyproxy/protoc-gen-validate/validate" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type LookupRequest struct { + state protoimpl.MessageState `protogen:"hybrid.v1"` + // Row kind: resources / entitlements / grants + // (pkg/sourcecache.RowKind values). Entries are partitioned by row + // kind, so one scope key can carry a different validator per kind. + RowKind string `protobuf:"bytes,1,opt,name=row_kind,json=rowKind,proto3" json:"row_kind,omitempty"` + // Connector-defined stable scope identifier (a stable identifier for the canonical scope + // (often a hex hash; see pkg/sourcecache.HashScope). Opaque to the + // parent; matched verbatim against the previous sync's source-cache + // entries. + ScopeKey string `protobuf:"bytes,2,opt,name=scope_key,json=scopeKey,proto3" json:"scope_key,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *LookupRequest) Reset() { + *x = LookupRequest{} + mi := &file_c1_connectorapi_baton_v1_source_cache_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *LookupRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*LookupRequest) ProtoMessage() {} + +func (x *LookupRequest) ProtoReflect() protoreflect.Message { + mi := &file_c1_connectorapi_baton_v1_source_cache_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *LookupRequest) GetRowKind() string { + if x != nil { + return x.RowKind + } + return "" +} + +func (x *LookupRequest) GetScopeKey() string { + if x != nil { + return x.ScopeKey + } + return "" +} + +func (x *LookupRequest) SetRowKind(v string) { + x.RowKind = v +} + +func (x *LookupRequest) SetScopeKey(v string) { + x.ScopeKey = v +} + +type LookupRequest_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + // Row kind: resources / entitlements / grants + // (pkg/sourcecache.RowKind values). Entries are partitioned by row + // kind, so one scope key can carry a different validator per kind. + RowKind string + // Connector-defined stable scope identifier (a stable identifier for the canonical scope + // (often a hex hash; see pkg/sourcecache.HashScope). Opaque to the + // parent; matched verbatim against the previous sync's source-cache + // entries. + ScopeKey string +} + +func (b0 LookupRequest_builder) Build() *LookupRequest { + m0 := &LookupRequest{} + b, x := &b0, m0 + _, _ = b, x + x.RowKind = b.RowKind + x.ScopeKey = b.ScopeKey + return m0 +} + +type LookupResponse struct { + state protoimpl.MessageState `protogen:"hybrid.v1"` + // False means no prior entry exists for (row_kind, scope_key): the + // connector must fetch fresh and must not emit SourceCacheReplay for + // this scope. + Found bool `protobuf:"varint,1,opt,name=found,proto3" json:"found,omitempty"` + // The opaque validator the previous sync recorded for this scope (HTTP + // ETag, delta token, ...). Empty when found is false. The cap is a + // sanity bound sized for Microsoft Graph delta tokens, which are known + // to run to thousands of characters; storage imposes no limit. + CacheValidator string `protobuf:"bytes,2,opt,name=cache_validator,json=cacheValidator,proto3" json:"cache_validator,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *LookupResponse) Reset() { + *x = LookupResponse{} + mi := &file_c1_connectorapi_baton_v1_source_cache_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *LookupResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*LookupResponse) ProtoMessage() {} + +func (x *LookupResponse) ProtoReflect() protoreflect.Message { + mi := &file_c1_connectorapi_baton_v1_source_cache_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *LookupResponse) GetFound() bool { + if x != nil { + return x.Found + } + return false +} + +func (x *LookupResponse) GetCacheValidator() string { + if x != nil { + return x.CacheValidator + } + return "" +} + +func (x *LookupResponse) SetFound(v bool) { + x.Found = v +} + +func (x *LookupResponse) SetCacheValidator(v string) { + x.CacheValidator = v +} + +type LookupResponse_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + // False means no prior entry exists for (row_kind, scope_key): the + // connector must fetch fresh and must not emit SourceCacheReplay for + // this scope. + Found bool + // The opaque validator the previous sync recorded for this scope (HTTP + // ETag, delta token, ...). Empty when found is false. The cap is a + // sanity bound sized for Microsoft Graph delta tokens, which are known + // to run to thousands of characters; storage imposes no limit. + CacheValidator string +} + +func (b0 LookupResponse_builder) Build() *LookupResponse { + m0 := &LookupResponse{} + b, x := &b0, m0 + _, _ = b, x + x.Found = b.Found + x.CacheValidator = b.CacheValidator + return m0 +} + +var File_c1_connectorapi_baton_v1_source_cache_proto protoreflect.FileDescriptor + +const file_c1_connectorapi_baton_v1_source_cache_proto_rawDesc = "" + + "\n" + + "+c1/connectorapi/baton/v1/source_cache.proto\x12\x18c1.connectorapi.baton.v1\x1a\x17validate/validate.proto\"^\n" + + "\rLookupRequest\x12$\n" + + "\brow_kind\x18\x01 \x01(\tB\t\xfaB\x06r\x04\x10\x01\x18@R\arowKind\x12'\n" + + "\tscope_key\x18\x02 \x01(\tB\n" + + "\xfaB\ar\x05\x10\x01\x18\x80\x02R\bscopeKey\"Z\n" + + "\x0eLookupResponse\x12\x14\n" + + "\x05found\x18\x01 \x01(\bR\x05found\x122\n" + + "\x0fcache_validator\x18\x02 \x01(\tB\t\xfaB\x06r\x04\x18\x80\x80\x04R\x0ecacheValidator2x\n" + + "\x17BatonSourceCacheService\x12]\n" + + "\x06Lookup\x12'.c1.connectorapi.baton.v1.LookupRequest\x1a(.c1.connectorapi.baton.v1.LookupResponse\"\x00B7Z5gitlab.com/ductone/c1/pkg/pb/c1/connectorapi/baton/v1b\x06proto3" + +var file_c1_connectorapi_baton_v1_source_cache_proto_msgTypes = make([]protoimpl.MessageInfo, 2) +var file_c1_connectorapi_baton_v1_source_cache_proto_goTypes = []any{ + (*LookupRequest)(nil), // 0: c1.connectorapi.baton.v1.LookupRequest + (*LookupResponse)(nil), // 1: c1.connectorapi.baton.v1.LookupResponse +} +var file_c1_connectorapi_baton_v1_source_cache_proto_depIdxs = []int32{ + 0, // 0: c1.connectorapi.baton.v1.BatonSourceCacheService.Lookup:input_type -> c1.connectorapi.baton.v1.LookupRequest + 1, // 1: c1.connectorapi.baton.v1.BatonSourceCacheService.Lookup:output_type -> c1.connectorapi.baton.v1.LookupResponse + 1, // [1:2] is the sub-list for method output_type + 0, // [0:1] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_c1_connectorapi_baton_v1_source_cache_proto_init() } +func file_c1_connectorapi_baton_v1_source_cache_proto_init() { + if File_c1_connectorapi_baton_v1_source_cache_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_c1_connectorapi_baton_v1_source_cache_proto_rawDesc), len(file_c1_connectorapi_baton_v1_source_cache_proto_rawDesc)), + NumEnums: 0, + NumMessages: 2, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_c1_connectorapi_baton_v1_source_cache_proto_goTypes, + DependencyIndexes: file_c1_connectorapi_baton_v1_source_cache_proto_depIdxs, + MessageInfos: file_c1_connectorapi_baton_v1_source_cache_proto_msgTypes, + }.Build() + File_c1_connectorapi_baton_v1_source_cache_proto = out.File + file_c1_connectorapi_baton_v1_source_cache_proto_goTypes = nil + file_c1_connectorapi_baton_v1_source_cache_proto_depIdxs = nil +} diff --git a/pb/c1/connectorapi/baton/v1/source_cache.pb.validate.go b/pb/c1/connectorapi/baton/v1/source_cache.pb.validate.go new file mode 100644 index 000000000..55912ddab --- /dev/null +++ b/pb/c1/connectorapi/baton/v1/source_cache.pb.validate.go @@ -0,0 +1,271 @@ +// Code generated by protoc-gen-validate. DO NOT EDIT. +// source: c1/connectorapi/baton/v1/source_cache.proto + +package v1 + +import ( + "bytes" + "errors" + "fmt" + "net" + "net/mail" + "net/url" + "regexp" + "sort" + "strings" + "time" + "unicode/utf8" + + "google.golang.org/protobuf/types/known/anypb" +) + +// ensure the imports are used +var ( + _ = bytes.MinRead + _ = errors.New("") + _ = fmt.Print + _ = utf8.UTFMax + _ = (*regexp.Regexp)(nil) + _ = (*strings.Reader)(nil) + _ = net.IPv4len + _ = time.Duration(0) + _ = (*url.URL)(nil) + _ = (*mail.Address)(nil) + _ = anypb.Any{} + _ = sort.Sort +) + +// Validate checks the field values on LookupRequest with the rules defined in +// the proto definition for this message. If any rules are violated, the first +// error encountered is returned, or nil if there are no violations. +func (m *LookupRequest) Validate() error { + return m.validate(false) +} + +// ValidateAll checks the field values on LookupRequest with the rules defined +// in the proto definition for this message. If any rules are violated, the +// result is a list of violation errors wrapped in LookupRequestMultiError, or +// nil if none found. +func (m *LookupRequest) ValidateAll() error { + return m.validate(true) +} + +func (m *LookupRequest) validate(all bool) error { + if m == nil { + return nil + } + + var errors []error + + if l := utf8.RuneCountInString(m.GetRowKind()); l < 1 || l > 64 { + err := LookupRequestValidationError{ + field: "RowKind", + reason: "value length must be between 1 and 64 runes, inclusive", + } + if !all { + return err + } + errors = append(errors, err) + } + + if l := utf8.RuneCountInString(m.GetScopeKey()); l < 1 || l > 256 { + err := LookupRequestValidationError{ + field: "ScopeKey", + reason: "value length must be between 1 and 256 runes, inclusive", + } + if !all { + return err + } + errors = append(errors, err) + } + + if len(errors) > 0 { + return LookupRequestMultiError(errors) + } + + return nil +} + +// LookupRequestMultiError is an error wrapping multiple validation errors +// returned by LookupRequest.ValidateAll() if the designated constraints +// aren't met. +type LookupRequestMultiError []error + +// Error returns a concatenation of all the error messages it wraps. +func (m LookupRequestMultiError) Error() string { + msgs := make([]string, 0, len(m)) + for _, err := range m { + msgs = append(msgs, err.Error()) + } + return strings.Join(msgs, "; ") +} + +// AllErrors returns a list of validation violation errors. +func (m LookupRequestMultiError) AllErrors() []error { return m } + +// LookupRequestValidationError is the validation error returned by +// LookupRequest.Validate if the designated constraints aren't met. +type LookupRequestValidationError struct { + field string + reason string + cause error + key bool +} + +// Field function returns field value. +func (e LookupRequestValidationError) Field() string { return e.field } + +// Reason function returns reason value. +func (e LookupRequestValidationError) Reason() string { return e.reason } + +// Cause function returns cause value. +func (e LookupRequestValidationError) Cause() error { return e.cause } + +// Key function returns key value. +func (e LookupRequestValidationError) Key() bool { return e.key } + +// ErrorName returns error name. +func (e LookupRequestValidationError) ErrorName() string { return "LookupRequestValidationError" } + +// Error satisfies the builtin error interface +func (e LookupRequestValidationError) Error() string { + cause := "" + if e.cause != nil { + cause = fmt.Sprintf(" | caused by: %v", e.cause) + } + + key := "" + if e.key { + key = "key for " + } + + return fmt.Sprintf( + "invalid %sLookupRequest.%s: %s%s", + key, + e.field, + e.reason, + cause) +} + +var _ error = LookupRequestValidationError{} + +var _ interface { + Field() string + Reason() string + Key() bool + Cause() error + ErrorName() string +} = LookupRequestValidationError{} + +// Validate checks the field values on LookupResponse with the rules defined in +// the proto definition for this message. If any rules are violated, the first +// error encountered is returned, or nil if there are no violations. +func (m *LookupResponse) Validate() error { + return m.validate(false) +} + +// ValidateAll checks the field values on LookupResponse with the rules defined +// in the proto definition for this message. If any rules are violated, the +// result is a list of violation errors wrapped in LookupResponseMultiError, +// or nil if none found. +func (m *LookupResponse) ValidateAll() error { + return m.validate(true) +} + +func (m *LookupResponse) validate(all bool) error { + if m == nil { + return nil + } + + var errors []error + + // no validation rules for Found + + if utf8.RuneCountInString(m.GetCacheValidator()) > 65536 { + err := LookupResponseValidationError{ + field: "CacheValidator", + reason: "value length must be at most 65536 runes", + } + if !all { + return err + } + errors = append(errors, err) + } + + if len(errors) > 0 { + return LookupResponseMultiError(errors) + } + + return nil +} + +// LookupResponseMultiError is an error wrapping multiple validation errors +// returned by LookupResponse.ValidateAll() if the designated constraints +// aren't met. +type LookupResponseMultiError []error + +// Error returns a concatenation of all the error messages it wraps. +func (m LookupResponseMultiError) Error() string { + msgs := make([]string, 0, len(m)) + for _, err := range m { + msgs = append(msgs, err.Error()) + } + return strings.Join(msgs, "; ") +} + +// AllErrors returns a list of validation violation errors. +func (m LookupResponseMultiError) AllErrors() []error { return m } + +// LookupResponseValidationError is the validation error returned by +// LookupResponse.Validate if the designated constraints aren't met. +type LookupResponseValidationError struct { + field string + reason string + cause error + key bool +} + +// Field function returns field value. +func (e LookupResponseValidationError) Field() string { return e.field } + +// Reason function returns reason value. +func (e LookupResponseValidationError) Reason() string { return e.reason } + +// Cause function returns cause value. +func (e LookupResponseValidationError) Cause() error { return e.cause } + +// Key function returns key value. +func (e LookupResponseValidationError) Key() bool { return e.key } + +// ErrorName returns error name. +func (e LookupResponseValidationError) ErrorName() string { return "LookupResponseValidationError" } + +// Error satisfies the builtin error interface +func (e LookupResponseValidationError) Error() string { + cause := "" + if e.cause != nil { + cause = fmt.Sprintf(" | caused by: %v", e.cause) + } + + key := "" + if e.key { + key = "key for " + } + + return fmt.Sprintf( + "invalid %sLookupResponse.%s: %s%s", + key, + e.field, + e.reason, + cause) +} + +var _ error = LookupResponseValidationError{} + +var _ interface { + Field() string + Reason() string + Key() bool + Cause() error + ErrorName() string +} = LookupResponseValidationError{} diff --git a/pb/c1/connectorapi/baton/v1/source_cache_grpc.pb.go b/pb/c1/connectorapi/baton/v1/source_cache_grpc.pb.go new file mode 100644 index 000000000..6c5bd6a71 --- /dev/null +++ b/pb/c1/connectorapi/baton/v1/source_cache_grpc.pb.go @@ -0,0 +1,145 @@ +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.5.1 +// - protoc (unknown) +// source: c1/connectorapi/baton/v1/source_cache.proto + +package v1 + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.64.0 or later. +const _ = grpc.SupportPackageIsVersion9 + +const ( + BatonSourceCacheService_Lookup_FullMethodName = "/c1.connectorapi.baton.v1.BatonSourceCacheService/Lookup" +) + +// BatonSourceCacheServiceClient is the client API for BatonSourceCacheService service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +// +// BatonSourceCacheService is the dedicated parent-side RPC the connector +// subprocess uses to ask "do I have a previous validator (HTTP ETag / delta +// token) for this scope?" before revalidating upstream. +// +// This is intentionally NOT routed through the session-store gRPC service: +// session data goes through the connector's local MemorySessionCache +// (otter), which would subject sync-scoped validator state to generic +// TTL/eviction policies and burn bounded cache weight on it. A dedicated +// service keeps the message shape explicit and the path uncached. +// +// The parent has exactly one active lookup registered at a time (set +// per-sync, cleared at sync end), so no sync_id travels on the wire. +type BatonSourceCacheServiceClient interface { + Lookup(ctx context.Context, in *LookupRequest, opts ...grpc.CallOption) (*LookupResponse, error) +} + +type batonSourceCacheServiceClient struct { + cc grpc.ClientConnInterface +} + +func NewBatonSourceCacheServiceClient(cc grpc.ClientConnInterface) BatonSourceCacheServiceClient { + return &batonSourceCacheServiceClient{cc} +} + +func (c *batonSourceCacheServiceClient) Lookup(ctx context.Context, in *LookupRequest, opts ...grpc.CallOption) (*LookupResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(LookupResponse) + err := c.cc.Invoke(ctx, BatonSourceCacheService_Lookup_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +// BatonSourceCacheServiceServer is the server API for BatonSourceCacheService service. +// All implementations should embed UnimplementedBatonSourceCacheServiceServer +// for forward compatibility. +// +// BatonSourceCacheService is the dedicated parent-side RPC the connector +// subprocess uses to ask "do I have a previous validator (HTTP ETag / delta +// token) for this scope?" before revalidating upstream. +// +// This is intentionally NOT routed through the session-store gRPC service: +// session data goes through the connector's local MemorySessionCache +// (otter), which would subject sync-scoped validator state to generic +// TTL/eviction policies and burn bounded cache weight on it. A dedicated +// service keeps the message shape explicit and the path uncached. +// +// The parent has exactly one active lookup registered at a time (set +// per-sync, cleared at sync end), so no sync_id travels on the wire. +type BatonSourceCacheServiceServer interface { + Lookup(context.Context, *LookupRequest) (*LookupResponse, error) +} + +// UnimplementedBatonSourceCacheServiceServer should be embedded to have +// forward compatible implementations. +// +// NOTE: this should be embedded by value instead of pointer to avoid a nil +// pointer dereference when methods are called. +type UnimplementedBatonSourceCacheServiceServer struct{} + +func (UnimplementedBatonSourceCacheServiceServer) Lookup(context.Context, *LookupRequest) (*LookupResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method Lookup not implemented") +} +func (UnimplementedBatonSourceCacheServiceServer) testEmbeddedByValue() {} + +// UnsafeBatonSourceCacheServiceServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to BatonSourceCacheServiceServer will +// result in compilation errors. +type UnsafeBatonSourceCacheServiceServer interface { + mustEmbedUnimplementedBatonSourceCacheServiceServer() +} + +func RegisterBatonSourceCacheServiceServer(s grpc.ServiceRegistrar, srv BatonSourceCacheServiceServer) { + // If the following call pancis, it indicates UnimplementedBatonSourceCacheServiceServer was + // embedded by pointer and is nil. This will cause panics if an + // unimplemented method is ever invoked, so we test this at initialization + // time to prevent it from happening at runtime later due to I/O. + if t, ok := srv.(interface{ testEmbeddedByValue() }); ok { + t.testEmbeddedByValue() + } + s.RegisterService(&BatonSourceCacheService_ServiceDesc, srv) +} + +func _BatonSourceCacheService_Lookup_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(LookupRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(BatonSourceCacheServiceServer).Lookup(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: BatonSourceCacheService_Lookup_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(BatonSourceCacheServiceServer).Lookup(ctx, req.(*LookupRequest)) + } + return interceptor(ctx, in, info, handler) +} + +// BatonSourceCacheService_ServiceDesc is the grpc.ServiceDesc for BatonSourceCacheService service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var BatonSourceCacheService_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "c1.connectorapi.baton.v1.BatonSourceCacheService", + HandlerType: (*BatonSourceCacheServiceServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "Lookup", + Handler: _BatonSourceCacheService_Lookup_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "c1/connectorapi/baton/v1/source_cache.proto", +} diff --git a/pb/c1/connectorapi/baton/v1/source_cache_protoopaque.pb.go b/pb/c1/connectorapi/baton/v1/source_cache_protoopaque.pb.go new file mode 100644 index 000000000..a645a377a --- /dev/null +++ b/pb/c1/connectorapi/baton/v1/source_cache_protoopaque.pb.go @@ -0,0 +1,234 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.10 +// protoc (unknown) +// source: c1/connectorapi/baton/v1/source_cache.proto + +//go:build protoopaque + +package v1 + +import ( + _ "github.com/envoyproxy/protoc-gen-validate/validate" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type LookupRequest struct { + state protoimpl.MessageState `protogen:"opaque.v1"` + xxx_hidden_RowKind string `protobuf:"bytes,1,opt,name=row_kind,json=rowKind,proto3"` + xxx_hidden_ScopeKey string `protobuf:"bytes,2,opt,name=scope_key,json=scopeKey,proto3"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *LookupRequest) Reset() { + *x = LookupRequest{} + mi := &file_c1_connectorapi_baton_v1_source_cache_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *LookupRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*LookupRequest) ProtoMessage() {} + +func (x *LookupRequest) ProtoReflect() protoreflect.Message { + mi := &file_c1_connectorapi_baton_v1_source_cache_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *LookupRequest) GetRowKind() string { + if x != nil { + return x.xxx_hidden_RowKind + } + return "" +} + +func (x *LookupRequest) GetScopeKey() string { + if x != nil { + return x.xxx_hidden_ScopeKey + } + return "" +} + +func (x *LookupRequest) SetRowKind(v string) { + x.xxx_hidden_RowKind = v +} + +func (x *LookupRequest) SetScopeKey(v string) { + x.xxx_hidden_ScopeKey = v +} + +type LookupRequest_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + // Row kind: resources / entitlements / grants + // (pkg/sourcecache.RowKind values). Entries are partitioned by row + // kind, so one scope key can carry a different validator per kind. + RowKind string + // Connector-defined stable scope identifier (a stable identifier for the canonical scope + // (often a hex hash; see pkg/sourcecache.HashScope). Opaque to the + // parent; matched verbatim against the previous sync's source-cache + // entries. + ScopeKey string +} + +func (b0 LookupRequest_builder) Build() *LookupRequest { + m0 := &LookupRequest{} + b, x := &b0, m0 + _, _ = b, x + x.xxx_hidden_RowKind = b.RowKind + x.xxx_hidden_ScopeKey = b.ScopeKey + return m0 +} + +type LookupResponse struct { + state protoimpl.MessageState `protogen:"opaque.v1"` + xxx_hidden_Found bool `protobuf:"varint,1,opt,name=found,proto3"` + xxx_hidden_CacheValidator string `protobuf:"bytes,2,opt,name=cache_validator,json=cacheValidator,proto3"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *LookupResponse) Reset() { + *x = LookupResponse{} + mi := &file_c1_connectorapi_baton_v1_source_cache_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *LookupResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*LookupResponse) ProtoMessage() {} + +func (x *LookupResponse) ProtoReflect() protoreflect.Message { + mi := &file_c1_connectorapi_baton_v1_source_cache_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *LookupResponse) GetFound() bool { + if x != nil { + return x.xxx_hidden_Found + } + return false +} + +func (x *LookupResponse) GetCacheValidator() string { + if x != nil { + return x.xxx_hidden_CacheValidator + } + return "" +} + +func (x *LookupResponse) SetFound(v bool) { + x.xxx_hidden_Found = v +} + +func (x *LookupResponse) SetCacheValidator(v string) { + x.xxx_hidden_CacheValidator = v +} + +type LookupResponse_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + // False means no prior entry exists for (row_kind, scope_key): the + // connector must fetch fresh and must not emit SourceCacheReplay for + // this scope. + Found bool + // The opaque validator the previous sync recorded for this scope (HTTP + // ETag, delta token, ...). Empty when found is false. The cap is a + // sanity bound sized for Microsoft Graph delta tokens, which are known + // to run to thousands of characters; storage imposes no limit. + CacheValidator string +} + +func (b0 LookupResponse_builder) Build() *LookupResponse { + m0 := &LookupResponse{} + b, x := &b0, m0 + _, _ = b, x + x.xxx_hidden_Found = b.Found + x.xxx_hidden_CacheValidator = b.CacheValidator + return m0 +} + +var File_c1_connectorapi_baton_v1_source_cache_proto protoreflect.FileDescriptor + +const file_c1_connectorapi_baton_v1_source_cache_proto_rawDesc = "" + + "\n" + + "+c1/connectorapi/baton/v1/source_cache.proto\x12\x18c1.connectorapi.baton.v1\x1a\x17validate/validate.proto\"^\n" + + "\rLookupRequest\x12$\n" + + "\brow_kind\x18\x01 \x01(\tB\t\xfaB\x06r\x04\x10\x01\x18@R\arowKind\x12'\n" + + "\tscope_key\x18\x02 \x01(\tB\n" + + "\xfaB\ar\x05\x10\x01\x18\x80\x02R\bscopeKey\"Z\n" + + "\x0eLookupResponse\x12\x14\n" + + "\x05found\x18\x01 \x01(\bR\x05found\x122\n" + + "\x0fcache_validator\x18\x02 \x01(\tB\t\xfaB\x06r\x04\x18\x80\x80\x04R\x0ecacheValidator2x\n" + + "\x17BatonSourceCacheService\x12]\n" + + "\x06Lookup\x12'.c1.connectorapi.baton.v1.LookupRequest\x1a(.c1.connectorapi.baton.v1.LookupResponse\"\x00B7Z5gitlab.com/ductone/c1/pkg/pb/c1/connectorapi/baton/v1b\x06proto3" + +var file_c1_connectorapi_baton_v1_source_cache_proto_msgTypes = make([]protoimpl.MessageInfo, 2) +var file_c1_connectorapi_baton_v1_source_cache_proto_goTypes = []any{ + (*LookupRequest)(nil), // 0: c1.connectorapi.baton.v1.LookupRequest + (*LookupResponse)(nil), // 1: c1.connectorapi.baton.v1.LookupResponse +} +var file_c1_connectorapi_baton_v1_source_cache_proto_depIdxs = []int32{ + 0, // 0: c1.connectorapi.baton.v1.BatonSourceCacheService.Lookup:input_type -> c1.connectorapi.baton.v1.LookupRequest + 1, // 1: c1.connectorapi.baton.v1.BatonSourceCacheService.Lookup:output_type -> c1.connectorapi.baton.v1.LookupResponse + 1, // [1:2] is the sub-list for method output_type + 0, // [0:1] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_c1_connectorapi_baton_v1_source_cache_proto_init() } +func file_c1_connectorapi_baton_v1_source_cache_proto_init() { + if File_c1_connectorapi_baton_v1_source_cache_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_c1_connectorapi_baton_v1_source_cache_proto_rawDesc), len(file_c1_connectorapi_baton_v1_source_cache_proto_rawDesc)), + NumEnums: 0, + NumMessages: 2, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_c1_connectorapi_baton_v1_source_cache_proto_goTypes, + DependencyIndexes: file_c1_connectorapi_baton_v1_source_cache_proto_depIdxs, + MessageInfos: file_c1_connectorapi_baton_v1_source_cache_proto_msgTypes, + }.Build() + File_c1_connectorapi_baton_v1_source_cache_proto = out.File + file_c1_connectorapi_baton_v1_source_cache_proto_goTypes = nil + file_c1_connectorapi_baton_v1_source_cache_proto_depIdxs = nil +} diff --git a/pb/c1/storage/v3/records.pb.go b/pb/c1/storage/v3/records.pb.go index 312f32a0e..7f80444f8 100644 --- a/pb/c1/storage/v3/records.pb.go +++ b/pb/c1/storage/v3/records.pb.go @@ -455,6 +455,9 @@ type ResourceRecord struct { Parent *ResourceRef `protobuf:"bytes,6,opt,name=parent,proto3" json:"parent,omitempty"` Annotations []*anypb.Any `protobuf:"bytes,7,rep,name=annotations,proto3" json:"annotations,omitempty"` DiscoveredAt *timestamppb.Timestamp `protobuf:"bytes,8,opt,name=discovered_at,json=discoveredAt,proto3" json:"discovered_at,omitempty"` + // Source-cache scope stamp (see annotation_source_cache.proto). + // Empty for rows not produced under a source-cache scope. + SourceScopeKey string `protobuf:"bytes,9,opt,name=source_scope_key,json=sourceScopeKey,proto3" json:"source_scope_key,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -533,6 +536,13 @@ func (x *ResourceRecord) GetDiscoveredAt() *timestamppb.Timestamp { return nil } +func (x *ResourceRecord) GetSourceScopeKey() string { + if x != nil { + return x.SourceScopeKey + } + return "" +} + func (x *ResourceRecord) SetResourceTypeId(v string) { x.ResourceTypeId = v } @@ -561,6 +571,10 @@ func (x *ResourceRecord) SetDiscoveredAt(v *timestamppb.Timestamp) { x.DiscoveredAt = v } +func (x *ResourceRecord) SetSourceScopeKey(v string) { + x.SourceScopeKey = v +} + func (x *ResourceRecord) HasParent() bool { if x == nil { return false @@ -593,6 +607,9 @@ type ResourceRecord_builder struct { Parent *ResourceRef Annotations []*anypb.Any DiscoveredAt *timestamppb.Timestamp + // Source-cache scope stamp (see annotation_source_cache.proto). + // Empty for rows not produced under a source-cache scope. + SourceScopeKey string } func (b0 ResourceRecord_builder) Build() *ResourceRecord { @@ -606,6 +623,7 @@ func (b0 ResourceRecord_builder) Build() *ResourceRecord { x.Parent = b.Parent x.Annotations = b.Annotations x.DiscoveredAt = b.DiscoveredAt + x.SourceScopeKey = b.SourceScopeKey return m0 } @@ -632,8 +650,11 @@ type EntitlementRecord struct { // resource_types table when needed. Consumed by // pkg/sync/syncer.go's principal-type narrowing. GrantableToResourceTypeIds []string `protobuf:"bytes,10,rep,name=grantable_to_resource_type_ids,json=grantableToResourceTypeIds,proto3" json:"grantable_to_resource_type_ids,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // Source-cache scope stamp (see annotation_source_cache.proto). + // Empty for rows not produced under a source-cache scope. + SourceScopeKey string `protobuf:"bytes,11,opt,name=source_scope_key,json=sourceScopeKey,proto3" json:"source_scope_key,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *EntitlementRecord) Reset() { @@ -724,6 +745,13 @@ func (x *EntitlementRecord) GetGrantableToResourceTypeIds() []string { return nil } +func (x *EntitlementRecord) GetSourceScopeKey() string { + if x != nil { + return x.SourceScopeKey + } + return "" +} + func (x *EntitlementRecord) SetExternalId(v string) { x.ExternalId = v } @@ -760,6 +788,10 @@ func (x *EntitlementRecord) SetGrantableToResourceTypeIds(v []string) { x.GrantableToResourceTypeIds = v } +func (x *EntitlementRecord) SetSourceScopeKey(v string) { + x.SourceScopeKey = v +} + func (x *EntitlementRecord) HasResource() bool { if x == nil { return false @@ -806,6 +838,9 @@ type EntitlementRecord_builder struct { // resource_types table when needed. Consumed by // pkg/sync/syncer.go's principal-type narrowing. GrantableToResourceTypeIds []string + // Source-cache scope stamp (see annotation_source_cache.proto). + // Empty for rows not produced under a source-cache scope. + SourceScopeKey string } func (b0 EntitlementRecord_builder) Build() *EntitlementRecord { @@ -821,6 +856,7 @@ func (b0 EntitlementRecord_builder) Build() *EntitlementRecord { x.DiscoveredAt = b.DiscoveredAt x.Slug = b.Slug x.GrantableToResourceTypeIds = b.GrantableToResourceTypeIds + x.SourceScopeKey = b.SourceScopeKey return m0 } @@ -846,9 +882,15 @@ type GrantRecord struct { Annotations []*anypb.Any `protobuf:"bytes,8,rep,name=annotations,proto3" json:"annotations,omitempty"` // map — same wire shape as // c1.connector.v2.GrantSources.sources. - Sources map[string]*GrantSourceRecord `protobuf:"bytes,9,rep,name=sources,proto3" json:"sources,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + Sources map[string]*GrantSourceRecord `protobuf:"bytes,9,rep,name=sources,proto3" json:"sources,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + // Source-cache scope stamp (see annotation_source_cache.proto). + // Empty for rows not produced under a source-cache scope — notably + // expander-derived grants, which are recreated by expansion each sync + // and never replayed. StoreExpandedGrants preserves this field on + // rewrites of existing rows, exactly like expansion/needs_expansion. + SourceScopeKey string `protobuf:"bytes,10,opt,name=source_scope_key,json=sourceScopeKey,proto3" json:"source_scope_key,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *GrantRecord) Reset() { @@ -932,6 +974,13 @@ func (x *GrantRecord) GetSources() map[string]*GrantSourceRecord { return nil } +func (x *GrantRecord) GetSourceScopeKey() string { + if x != nil { + return x.SourceScopeKey + } + return "" +} + func (x *GrantRecord) SetExternalId(v string) { x.ExternalId = v } @@ -964,6 +1013,10 @@ func (x *GrantRecord) SetSources(v map[string]*GrantSourceRecord) { x.Sources = v } +func (x *GrantRecord) SetSourceScopeKey(v string) { + x.SourceScopeKey = v +} + func (x *GrantRecord) HasEntitlement() bool { if x == nil { return false @@ -1032,6 +1085,12 @@ type GrantRecord_builder struct { // map — same wire shape as // c1.connector.v2.GrantSources.sources. Sources map[string]*GrantSourceRecord + // Source-cache scope stamp (see annotation_source_cache.proto). + // Empty for rows not produced under a source-cache scope — notably + // expander-derived grants, which are recreated by expansion each sync + // and never replayed. StoreExpandedGrants preserves this field on + // rewrites of existing rows, exactly like expansion/needs_expansion. + SourceScopeKey string } func (b0 GrantRecord_builder) Build() *GrantRecord { @@ -1046,6 +1105,7 @@ func (b0 GrantRecord_builder) Build() *GrantRecord { x.NeedsExpansion = b.NeedsExpansion x.Annotations = b.Annotations x.Sources = b.Sources + x.SourceScopeKey = b.SourceScopeKey return m0 } @@ -1177,15 +1237,29 @@ func (b0 AssetRecord_builder) Build() *AssetRecord { } type SyncRunRecord struct { - state protoimpl.MessageState `protogen:"hybrid.v1"` - SyncId string `protobuf:"bytes,1,opt,name=sync_id,json=syncId,proto3" json:"sync_id,omitempty"` - Type SyncType `protobuf:"varint,2,opt,name=type,proto3,enum=c1.storage.v3.SyncType" json:"type,omitempty"` - ParentSyncId string `protobuf:"bytes,3,opt,name=parent_sync_id,json=parentSyncId,proto3" json:"parent_sync_id,omitempty"` - StartedAt *timestamppb.Timestamp `protobuf:"bytes,4,opt,name=started_at,json=startedAt,proto3" json:"started_at,omitempty"` - EndedAt *timestamppb.Timestamp `protobuf:"bytes,5,opt,name=ended_at,json=endedAt,proto3" json:"ended_at,omitempty"` - SyncToken string `protobuf:"bytes,6,opt,name=sync_token,json=syncToken,proto3" json:"sync_token,omitempty"` - SupportsDiff bool `protobuf:"varint,7,opt,name=supports_diff,json=supportsDiff,proto3" json:"supports_diff,omitempty"` - LinkedSyncId string `protobuf:"bytes,8,opt,name=linked_sync_id,json=linkedSyncId,proto3" json:"linked_sync_id,omitempty"` + state protoimpl.MessageState `protogen:"hybrid.v1"` + SyncId string `protobuf:"bytes,1,opt,name=sync_id,json=syncId,proto3" json:"sync_id,omitempty"` + Type SyncType `protobuf:"varint,2,opt,name=type,proto3,enum=c1.storage.v3.SyncType" json:"type,omitempty"` + ParentSyncId string `protobuf:"bytes,3,opt,name=parent_sync_id,json=parentSyncId,proto3" json:"parent_sync_id,omitempty"` + StartedAt *timestamppb.Timestamp `protobuf:"bytes,4,opt,name=started_at,json=startedAt,proto3" json:"started_at,omitempty"` + EndedAt *timestamppb.Timestamp `protobuf:"bytes,5,opt,name=ended_at,json=endedAt,proto3" json:"ended_at,omitempty"` + SyncToken string `protobuf:"bytes,6,opt,name=sync_token,json=syncToken,proto3" json:"sync_token,omitempty"` + SupportsDiff bool `protobuf:"varint,7,opt,name=supports_diff,json=supportsDiff,proto3" json:"supports_diff,omitempty"` + LinkedSyncId string `protobuf:"bytes,8,opt,name=linked_sync_id,json=linkedSyncId,proto3" json:"linked_sync_id,omitempty"` + // compacted marks a sync produced by compaction (fold or rebuild) + // rather than by a real connector run. Compacted artifacts are + // keep-newer UPSERT merges — base rows a newer input deleted survive — + // so no input sync's upstream validators (source-cache validators) describe + // their contents. + // + // "Can this sync be used as a source-cache replay source?" is the + // predicate c1zstore.SyncRun.UsableAsReplaySource: type == FULL and + // !compacted. The syncer enforces it — feeding a compacted or + // partial/derived sync to WithPreviousSyncC1ZPath degrades to a cold + // (full-fetch) sync. Orchestrators deciding which artifact to + // materialize as the previous sync should apply the same predicate + // instead of guessing from provenance. + Compacted bool `protobuf:"varint,9,opt,name=compacted,proto3" json:"compacted,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -1271,6 +1345,13 @@ func (x *SyncRunRecord) GetLinkedSyncId() string { return "" } +func (x *SyncRunRecord) GetCompacted() bool { + if x != nil { + return x.Compacted + } + return false +} + func (x *SyncRunRecord) SetSyncId(v string) { x.SyncId = v } @@ -1303,6 +1384,10 @@ func (x *SyncRunRecord) SetLinkedSyncId(v string) { x.LinkedSyncId = v } +func (x *SyncRunRecord) SetCompacted(v bool) { + x.Compacted = v +} + func (x *SyncRunRecord) HasStartedAt() bool { if x == nil { return false @@ -1336,6 +1421,20 @@ type SyncRunRecord_builder struct { SyncToken string SupportsDiff bool LinkedSyncId string + // compacted marks a sync produced by compaction (fold or rebuild) + // rather than by a real connector run. Compacted artifacts are + // keep-newer UPSERT merges — base rows a newer input deleted survive — + // so no input sync's upstream validators (source-cache validators) describe + // their contents. + // + // "Can this sync be used as a source-cache replay source?" is the + // predicate c1zstore.SyncRun.UsableAsReplaySource: type == FULL and + // !compacted. The syncer enforces it — feeding a compacted or + // partial/derived sync to WithPreviousSyncC1ZPath degrades to a cold + // (full-fetch) sync. Orchestrators deciding which artifact to + // materialize as the previous sync should apply the same predicate + // instead of guessing from provenance. + Compacted bool } func (b0 SyncRunRecord_builder) Build() *SyncRunRecord { @@ -1350,6 +1449,7 @@ func (b0 SyncRunRecord_builder) Build() *SyncRunRecord { x.SyncToken = b.SyncToken x.SupportsDiff = b.SupportsDiff x.LinkedSyncId = b.LinkedSyncId + x.Compacted = b.Compacted return m0 } @@ -1653,6 +1753,306 @@ func (b0 SessionRecord_builder) Build() *SessionRecord { return m0 } +// SourceCacheEntryRecord is the per-scope manifest entry for source-cache +// replay (see c1/connector/v2/annotation_source_cache.proto). One entry +// per (row_kind, scope_key), written for every freshly fetched scope — +// including zero-row responses — and rewritten on replay with the scope's +// current validator. The previous sync's entries (read from the previous +// c1z) are the lookup surface connectors revalidate against. +type SourceCacheEntryRecord struct { + state protoimpl.MessageState `protogen:"hybrid.v1"` + // Row kind partition: "resources", "entitlements", or "grants" + // (pkg/sourcecache.RowKind values). + RowKind string `protobuf:"bytes,1,opt,name=row_kind,json=rowKind,proto3" json:"row_kind,omitempty"` + // Connector-computed stable scope identifier (conventionally lowercase + // hex when hashed via sourcecache.HashScope; any byte-stable string). + ScopeKey string `protobuf:"bytes,2,opt,name=scope_key,json=scopeKey,proto3" json:"scope_key,omitempty"` + // Opaque upstream validator: HTTP ETag, delta token, etc. + CacheValidator string `protobuf:"bytes,3,opt,name=cache_validator,json=cacheValidator,proto3" json:"cache_validator,omitempty"` + DiscoveredAt *timestamppb.Timestamp `protobuf:"bytes,4,opt,name=discovered_at,json=discoveredAt,proto3" json:"discovered_at,omitempty"` + // Set when the dangling-reference drops (ingestion invariants I7/I8/I9, + // pkg/sync/ingest_invariants.go) deleted rows stamped with this scope: + // the artifact no longer holds the full row set the validator vouches + // for, so a future sync must not 304-replay it. Lookups treat an + // invalidated entry as a miss (the scope re-fetches cold and converges + // with a cold sync); the entry itself is kept so the scope's surviving + // stamped rows do not read as an I6 orphan (lost manifest write). + Invalidated bool `protobuf:"varint,5,opt,name=invalidated,proto3" json:"invalidated,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SourceCacheEntryRecord) Reset() { + *x = SourceCacheEntryRecord{} + mi := &file_c1_storage_v3_records_proto_msgTypes[10] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SourceCacheEntryRecord) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SourceCacheEntryRecord) ProtoMessage() {} + +func (x *SourceCacheEntryRecord) ProtoReflect() protoreflect.Message { + mi := &file_c1_storage_v3_records_proto_msgTypes[10] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *SourceCacheEntryRecord) GetRowKind() string { + if x != nil { + return x.RowKind + } + return "" +} + +func (x *SourceCacheEntryRecord) GetScopeKey() string { + if x != nil { + return x.ScopeKey + } + return "" +} + +func (x *SourceCacheEntryRecord) GetCacheValidator() string { + if x != nil { + return x.CacheValidator + } + return "" +} + +func (x *SourceCacheEntryRecord) GetDiscoveredAt() *timestamppb.Timestamp { + if x != nil { + return x.DiscoveredAt + } + return nil +} + +func (x *SourceCacheEntryRecord) GetInvalidated() bool { + if x != nil { + return x.Invalidated + } + return false +} + +func (x *SourceCacheEntryRecord) SetRowKind(v string) { + x.RowKind = v +} + +func (x *SourceCacheEntryRecord) SetScopeKey(v string) { + x.ScopeKey = v +} + +func (x *SourceCacheEntryRecord) SetCacheValidator(v string) { + x.CacheValidator = v +} + +func (x *SourceCacheEntryRecord) SetDiscoveredAt(v *timestamppb.Timestamp) { + x.DiscoveredAt = v +} + +func (x *SourceCacheEntryRecord) SetInvalidated(v bool) { + x.Invalidated = v +} + +func (x *SourceCacheEntryRecord) HasDiscoveredAt() bool { + if x == nil { + return false + } + return x.DiscoveredAt != nil +} + +func (x *SourceCacheEntryRecord) ClearDiscoveredAt() { + x.DiscoveredAt = nil +} + +type SourceCacheEntryRecord_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + // Row kind partition: "resources", "entitlements", or "grants" + // (pkg/sourcecache.RowKind values). + RowKind string + // Connector-computed stable scope identifier (conventionally lowercase + // hex when hashed via sourcecache.HashScope; any byte-stable string). + ScopeKey string + // Opaque upstream validator: HTTP ETag, delta token, etc. + CacheValidator string + DiscoveredAt *timestamppb.Timestamp + // Set when the dangling-reference drops (ingestion invariants I7/I8/I9, + // pkg/sync/ingest_invariants.go) deleted rows stamped with this scope: + // the artifact no longer holds the full row set the validator vouches + // for, so a future sync must not 304-replay it. Lookups treat an + // invalidated entry as a miss (the scope re-fetches cold and converges + // with a cold sync); the entry itself is kept so the scope's surviving + // stamped rows do not read as an I6 orphan (lost manifest write). + Invalidated bool +} + +func (b0 SourceCacheEntryRecord_builder) Build() *SourceCacheEntryRecord { + m0 := &SourceCacheEntryRecord{} + b, x := &b0, m0 + _, _ = b, x + x.RowKind = b.RowKind + x.ScopeKey = b.ScopeKey + x.CacheValidator = b.CacheValidator + x.DiscoveredAt = b.DiscoveredAt + x.Invalidated = b.Invalidated + return m0 +} + +// SourceCacheCompatRecord is the sync's replay-compatibility key: the +// exact conditions under which this artifact's source-cache manifest was +// recorded. A future sync may replay from this artifact ONLY when its +// own key matches byte-exactly on every component; an absent, unreadable, +// or mismatched record degrades that sync to cold (no-op lookup). +// Compatibility is never inferred from versions — each component is an +// explicit declaration by its owner. +// +// One record per sync, written when the write side of the source cache +// enables (before any manifest entry), cleared by compaction folds with +// the manifest. +type SourceCacheCompatRecord struct { + state protoimpl.MessageState `protogen:"hybrid.v1"` + // Fixed singleton id ("compat"); present so the record satisfies the + // table shape. + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + // Connector-declared components (SourceCacheCapability annotations): + // the connector's cache generation and configuration/permission + // fingerprint. + ConnectorCacheGeneration string `protobuf:"bytes,2,opt,name=connector_cache_generation,json=connectorCacheGeneration,proto3" json:"connector_cache_generation,omitempty"` + ConnectorConfigFingerprint string `protobuf:"bytes,3,opt,name=connector_config_fingerprint,json=connectorConfigFingerprint,proto3" json:"connector_config_fingerprint,omitempty"` + // SDK materialization-policy generation: bumped when the SDK changes + // how response rows or their side effects materialize into the store + // in a replay-visible way (sourcecache.MaterializationPolicyGeneration). + SdkMaterializationGeneration string `protobuf:"bytes,4,opt,name=sdk_materialization_generation,json=sdkMaterializationGeneration,proto3" json:"sdk_materialization_generation,omitempty"` + // SDK sync-selection fingerprint: digest of the sync-shaping inputs + // (enabled resource types, skip flags) — a selection change means the + // prior artifact's scopes cover a different row universe. + SyncSelectionFingerprint string `protobuf:"bytes,5,opt,name=sync_selection_fingerprint,json=syncSelectionFingerprint,proto3" json:"sync_selection_fingerprint,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SourceCacheCompatRecord) Reset() { + *x = SourceCacheCompatRecord{} + mi := &file_c1_storage_v3_records_proto_msgTypes[11] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SourceCacheCompatRecord) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SourceCacheCompatRecord) ProtoMessage() {} + +func (x *SourceCacheCompatRecord) ProtoReflect() protoreflect.Message { + mi := &file_c1_storage_v3_records_proto_msgTypes[11] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *SourceCacheCompatRecord) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +func (x *SourceCacheCompatRecord) GetConnectorCacheGeneration() string { + if x != nil { + return x.ConnectorCacheGeneration + } + return "" +} + +func (x *SourceCacheCompatRecord) GetConnectorConfigFingerprint() string { + if x != nil { + return x.ConnectorConfigFingerprint + } + return "" +} + +func (x *SourceCacheCompatRecord) GetSdkMaterializationGeneration() string { + if x != nil { + return x.SdkMaterializationGeneration + } + return "" +} + +func (x *SourceCacheCompatRecord) GetSyncSelectionFingerprint() string { + if x != nil { + return x.SyncSelectionFingerprint + } + return "" +} + +func (x *SourceCacheCompatRecord) SetId(v string) { + x.Id = v +} + +func (x *SourceCacheCompatRecord) SetConnectorCacheGeneration(v string) { + x.ConnectorCacheGeneration = v +} + +func (x *SourceCacheCompatRecord) SetConnectorConfigFingerprint(v string) { + x.ConnectorConfigFingerprint = v +} + +func (x *SourceCacheCompatRecord) SetSdkMaterializationGeneration(v string) { + x.SdkMaterializationGeneration = v +} + +func (x *SourceCacheCompatRecord) SetSyncSelectionFingerprint(v string) { + x.SyncSelectionFingerprint = v +} + +type SourceCacheCompatRecord_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + // Fixed singleton id ("compat"); present so the record satisfies the + // table shape. + Id string + // Connector-declared components (SourceCacheCapability annotations): + // the connector's cache generation and configuration/permission + // fingerprint. + ConnectorCacheGeneration string + ConnectorConfigFingerprint string + // SDK materialization-policy generation: bumped when the SDK changes + // how response rows or their side effects materialize into the store + // in a replay-visible way (sourcecache.MaterializationPolicyGeneration). + SdkMaterializationGeneration string + // SDK sync-selection fingerprint: digest of the sync-shaping inputs + // (enabled resource types, skip flags) — a selection change means the + // prior artifact's scopes cover a different row universe. + SyncSelectionFingerprint string +} + +func (b0 SourceCacheCompatRecord_builder) Build() *SourceCacheCompatRecord { + m0 := &SourceCacheCompatRecord{} + b, x := &b0, m0 + _, _ = b, x + x.Id = b.Id + x.ConnectorCacheGeneration = b.ConnectorCacheGeneration + x.ConnectorConfigFingerprint = b.ConnectorConfigFingerprint + x.SdkMaterializationGeneration = b.SdkMaterializationGeneration + x.SyncSelectionFingerprint = b.SyncSelectionFingerprint + return m0 +} + var File_c1_storage_v3_records_proto protoreflect.FileDescriptor const file_c1_storage_v3_records_proto_rawDesc = "" + @@ -1677,7 +2077,7 @@ const file_c1_storage_v3_records_proto_rawDesc = "" + "\rdiscovered_at\x18\x06 \x01(\v2\x1a.google.protobuf.TimestampR\fdiscoveredAt\x12 \n" + "\vdescription\x18\a \x01(\tR\vdescription\x12-\n" + "\x12sourced_externally\x18\b \x01(\bR\x11sourcedExternally:!\x82\xf9+\x1d\n" + - "\x0eresource_types\x12\vexternal_idJ\x04\b\x01\x10\x02R\async_id\"\xbc\x03\n" + + "\x0eresource_types\x12\vexternal_idJ\x04\b\x01\x10\x02R\async_id\"\x95\x04\n" + "\x0eResourceRecord\x12(\n" + "\x10resource_type_id\x18\x02 \x01(\tR\x0eresourceTypeId\x12\x1f\n" + "\vresource_id\x18\x03 \x01(\tR\n" + @@ -1687,8 +2087,10 @@ const file_c1_storage_v3_records_proto_rawDesc = "" + "\x06parent\x18\x06 \x01(\v2\x1a.c1.storage.v3.ResourceRefB.\x8a\xf9+*\n" + "\tby_parent\x1a\x10resource_type_id\x1a\vresource_idR\x06parent\x126\n" + "\vannotations\x18\a \x03(\v2\x14.google.protobuf.AnyR\vannotations\x12?\n" + - "\rdiscovered_at\x18\b \x01(\v2\x1a.google.protobuf.TimestampR\fdiscoveredAt:.\x82\xf9+*\n" + - "\tresources\x12\x10resource_type_id\x12\vresource_idJ\x04\b\x01\x10\x02R\async_id\"\xfe\x03\n" + + "\rdiscovered_at\x18\b \x01(\v2\x1a.google.protobuf.TimestampR\fdiscoveredAt\x12W\n" + + "\x10source_scope_key\x18\t \x01(\tB-\x8a\xf9+)\n" + + "\x0fby_source_scope\"\x16source_scope_key != ''R\x0esourceScopeKey:.\x82\xf9+*\n" + + "\tresources\x12\x10resource_type_id\x12\vresource_idJ\x04\b\x01\x10\x02R\async_id\"\xd7\x04\n" + "\x11EntitlementRecord\x12\x1f\n" + "\vexternal_id\x18\x02 \x01(\tR\n" + "externalId\x12h\n" + @@ -1701,8 +2103,10 @@ const file_c1_storage_v3_records_proto_rawDesc = "" + "\rdiscovered_at\x18\b \x01(\v2\x1a.google.protobuf.TimestampR\fdiscoveredAt\x12\x12\n" + "\x04slug\x18\t \x01(\tR\x04slug\x12B\n" + "\x1egrantable_to_resource_type_ids\x18\n" + - " \x03(\tR\x1agrantableToResourceTypeIds:\x1f\x82\xf9+\x1b\n" + - "\fentitlements\x12\vexternal_idJ\x04\b\x01\x10\x02R\async_id\"\x9a\x06\n" + + " \x03(\tR\x1agrantableToResourceTypeIds\x12W\n" + + "\x10source_scope_key\x18\v \x01(\tB-\x8a\xf9+)\n" + + "\x0fby_source_scope\"\x16source_scope_key != ''R\x0esourceScopeKey:\x1f\x82\xf9+\x1b\n" + + "\fentitlements\x12\vexternal_idJ\x04\b\x01\x10\x02R\async_id\"\xf3\x06\n" + "\vGrantRecord\x12\x1f\n" + "\vexternal_id\x18\x02 \x01(\tR\n" + "externalId\x12\x98\x01\n" + @@ -1715,7 +2119,10 @@ const file_c1_storage_v3_records_proto_rawDesc = "" + "\x0fneeds_expansion\x18\a \x01(\bB0\x8a\xf9+,\n" + "\x12by_needs_expansion\"\x16needs_expansion = trueR\x0eneedsExpansion\x126\n" + "\vannotations\x18\b \x03(\v2\x14.google.protobuf.AnyR\vannotations\x12A\n" + - "\asources\x18\t \x03(\v2'.c1.storage.v3.GrantRecord.SourcesEntryR\asources\x1a\\\n" + + "\asources\x18\t \x03(\v2'.c1.storage.v3.GrantRecord.SourcesEntryR\asources\x12W\n" + + "\x10source_scope_key\x18\n" + + " \x01(\tB-\x8a\xf9+)\n" + + "\x0fby_source_scope\"\x16source_scope_key != ''R\x0esourceScopeKey\x1a\\\n" + "\fSourcesEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x126\n" + "\x05value\x18\x02 \x01(\v2 .c1.storage.v3.GrantSourceRecordR\x05value:\x028\x01:\x19\x82\xf9+\x15\n" + @@ -1727,7 +2134,7 @@ const file_c1_storage_v3_records_proto_rawDesc = "" + "\fcontent_type\x18\x03 \x01(\tR\vcontentType\x12\x12\n" + "\x04data\x18\x04 \x01(\fR\x04data\x12?\n" + "\rdiscovered_at\x18\x05 \x01(\v2\x1a.google.protobuf.TimestampR\fdiscoveredAt:\"\x82\xf9+\x1e\n" + - "\x06assets\x12\async_id\x12\vexternal_id\"\xf1\x02\n" + + "\x06assets\x12\async_id\x12\vexternal_id\"\x8f\x03\n" + "\rSyncRunRecord\x12\x17\n" + "\async_id\x18\x01 \x01(\tR\x06syncId\x12+\n" + "\x04type\x18\x02 \x01(\x0e2\x17.c1.storage.v3.SyncTypeR\x04type\x12$\n" + @@ -1738,7 +2145,8 @@ const file_c1_storage_v3_records_proto_rawDesc = "" + "\n" + "sync_token\x18\x06 \x01(\tR\tsyncToken\x12#\n" + "\rsupports_diff\x18\a \x01(\bR\fsupportsDiff\x12$\n" + - "\x0elinked_sync_id\x18\b \x01(\tR\flinkedSyncId:\x18\x82\xf9+\x14\n" + + "\x0elinked_sync_id\x18\b \x01(\tR\flinkedSyncId\x12\x1c\n" + + "\tcompacted\x18\t \x01(\bR\tcompacted:\x18\x82\xf9+\x14\n" + "\tsync_runs\x12\async_id\"\xff\x06\n" + "\x0fSyncStatsRecord\x12\x17\n" + "\async_id\x18\x01 \x01(\tR\x06syncId\x12%\n" + @@ -1765,7 +2173,21 @@ const file_c1_storage_v3_records_proto_rawDesc = "" + "\async_id\x18\x01 \x01(\tR\x06syncId\x12\x10\n" + "\x03key\x18\x02 \x01(\tR\x03key\x12\x14\n" + "\x05value\x18\x03 \x01(\fR\x05value:\x1c\x82\xf9+\x18\n" + - "\bsessions\x12\async_id\x12\x03key*\xae\x01\n" + + "\bsessions\x12\async_id\x12\x03key\"\x8d\x02\n" + + "\x16SourceCacheEntryRecord\x12\x19\n" + + "\brow_kind\x18\x01 \x01(\tR\arowKind\x12\x1b\n" + + "\tscope_key\x18\x02 \x01(\tR\bscopeKey\x12'\n" + + "\x0fcache_validator\x18\x03 \x01(\tR\x0ecacheValidator\x12?\n" + + "\rdiscovered_at\x18\x04 \x01(\v2\x1a.google.protobuf.TimestampR\fdiscoveredAt\x12 \n" + + "\vinvalidated\x18\x05 \x01(\bR\vinvalidated:/\x82\xf9++\n" + + "\x14source_cache_entries\x12\brow_kind\x12\tscope_key\"\xcc\x02\n" + + "\x17SourceCacheCompatRecord\x12\x0e\n" + + "\x02id\x18\x01 \x01(\tR\x02id\x12<\n" + + "\x1aconnector_cache_generation\x18\x02 \x01(\tR\x18connectorCacheGeneration\x12@\n" + + "\x1cconnector_config_fingerprint\x18\x03 \x01(\tR\x1aconnectorConfigFingerprint\x12D\n" + + "\x1esdk_materialization_generation\x18\x04 \x01(\tR\x1csdkMaterializationGeneration\x12<\n" + + "\x1async_selection_fingerprint\x18\x05 \x01(\tR\x18syncSelectionFingerprint:\x1d\x82\xf9+\x19\n" + + "\x13source_cache_compat\x12\x02id*\xae\x01\n" + "\bSyncType\x12\x19\n" + "\x15SYNC_TYPE_UNSPECIFIED\x10\x00\x12\x12\n" + "\x0eSYNC_TYPE_FULL\x10\x01\x12\x15\n" + @@ -1775,58 +2197,61 @@ const file_c1_storage_v3_records_proto_rawDesc = "" + "\x1bSYNC_TYPE_PARTIAL_DELETIONS\x10\x05B4Z2github.com/conductorone/baton-sdk/pb/c1/storage/v3b\x06proto3" var file_c1_storage_v3_records_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_c1_storage_v3_records_proto_msgTypes = make([]protoimpl.MessageInfo, 14) +var file_c1_storage_v3_records_proto_msgTypes = make([]protoimpl.MessageInfo, 16) var file_c1_storage_v3_records_proto_goTypes = []any{ - (SyncType)(0), // 0: c1.storage.v3.SyncType - (*GrantExpandableRecord)(nil), // 1: c1.storage.v3.GrantExpandableRecord - (*GrantSourceRecord)(nil), // 2: c1.storage.v3.GrantSourceRecord - (*ResourceTypeRecord)(nil), // 3: c1.storage.v3.ResourceTypeRecord - (*ResourceRecord)(nil), // 4: c1.storage.v3.ResourceRecord - (*EntitlementRecord)(nil), // 5: c1.storage.v3.EntitlementRecord - (*GrantRecord)(nil), // 6: c1.storage.v3.GrantRecord - (*AssetRecord)(nil), // 7: c1.storage.v3.AssetRecord - (*SyncRunRecord)(nil), // 8: c1.storage.v3.SyncRunRecord - (*SyncStatsRecord)(nil), // 9: c1.storage.v3.SyncStatsRecord - (*SessionRecord)(nil), // 10: c1.storage.v3.SessionRecord - nil, // 11: c1.storage.v3.GrantRecord.SourcesEntry - nil, // 12: c1.storage.v3.SyncStatsRecord.ResourcesByResourceTypeEntry - nil, // 13: c1.storage.v3.SyncStatsRecord.GrantsByEntitlementResourceTypeEntry - nil, // 14: c1.storage.v3.SyncStatsRecord.EntitlementsByResourceTypeEntry - (*anypb.Any)(nil), // 15: google.protobuf.Any - (*timestamppb.Timestamp)(nil), // 16: google.protobuf.Timestamp - (*ResourceRef)(nil), // 17: c1.storage.v3.ResourceRef - (*EntitlementRef)(nil), // 18: c1.storage.v3.EntitlementRef - (*PrincipalRef)(nil), // 19: c1.storage.v3.PrincipalRef + (SyncType)(0), // 0: c1.storage.v3.SyncType + (*GrantExpandableRecord)(nil), // 1: c1.storage.v3.GrantExpandableRecord + (*GrantSourceRecord)(nil), // 2: c1.storage.v3.GrantSourceRecord + (*ResourceTypeRecord)(nil), // 3: c1.storage.v3.ResourceTypeRecord + (*ResourceRecord)(nil), // 4: c1.storage.v3.ResourceRecord + (*EntitlementRecord)(nil), // 5: c1.storage.v3.EntitlementRecord + (*GrantRecord)(nil), // 6: c1.storage.v3.GrantRecord + (*AssetRecord)(nil), // 7: c1.storage.v3.AssetRecord + (*SyncRunRecord)(nil), // 8: c1.storage.v3.SyncRunRecord + (*SyncStatsRecord)(nil), // 9: c1.storage.v3.SyncStatsRecord + (*SessionRecord)(nil), // 10: c1.storage.v3.SessionRecord + (*SourceCacheEntryRecord)(nil), // 11: c1.storage.v3.SourceCacheEntryRecord + (*SourceCacheCompatRecord)(nil), // 12: c1.storage.v3.SourceCacheCompatRecord + nil, // 13: c1.storage.v3.GrantRecord.SourcesEntry + nil, // 14: c1.storage.v3.SyncStatsRecord.ResourcesByResourceTypeEntry + nil, // 15: c1.storage.v3.SyncStatsRecord.GrantsByEntitlementResourceTypeEntry + nil, // 16: c1.storage.v3.SyncStatsRecord.EntitlementsByResourceTypeEntry + (*anypb.Any)(nil), // 17: google.protobuf.Any + (*timestamppb.Timestamp)(nil), // 18: google.protobuf.Timestamp + (*ResourceRef)(nil), // 19: c1.storage.v3.ResourceRef + (*EntitlementRef)(nil), // 20: c1.storage.v3.EntitlementRef + (*PrincipalRef)(nil), // 21: c1.storage.v3.PrincipalRef } var file_c1_storage_v3_records_proto_depIdxs = []int32{ - 15, // 0: c1.storage.v3.ResourceTypeRecord.annotations:type_name -> google.protobuf.Any - 16, // 1: c1.storage.v3.ResourceTypeRecord.discovered_at:type_name -> google.protobuf.Timestamp - 17, // 2: c1.storage.v3.ResourceRecord.parent:type_name -> c1.storage.v3.ResourceRef - 15, // 3: c1.storage.v3.ResourceRecord.annotations:type_name -> google.protobuf.Any - 16, // 4: c1.storage.v3.ResourceRecord.discovered_at:type_name -> google.protobuf.Timestamp - 17, // 5: c1.storage.v3.EntitlementRecord.resource:type_name -> c1.storage.v3.ResourceRef - 15, // 6: c1.storage.v3.EntitlementRecord.annotations:type_name -> google.protobuf.Any - 16, // 7: c1.storage.v3.EntitlementRecord.discovered_at:type_name -> google.protobuf.Timestamp - 18, // 8: c1.storage.v3.GrantRecord.entitlement:type_name -> c1.storage.v3.EntitlementRef - 19, // 9: c1.storage.v3.GrantRecord.principal:type_name -> c1.storage.v3.PrincipalRef - 16, // 10: c1.storage.v3.GrantRecord.discovered_at:type_name -> google.protobuf.Timestamp + 17, // 0: c1.storage.v3.ResourceTypeRecord.annotations:type_name -> google.protobuf.Any + 18, // 1: c1.storage.v3.ResourceTypeRecord.discovered_at:type_name -> google.protobuf.Timestamp + 19, // 2: c1.storage.v3.ResourceRecord.parent:type_name -> c1.storage.v3.ResourceRef + 17, // 3: c1.storage.v3.ResourceRecord.annotations:type_name -> google.protobuf.Any + 18, // 4: c1.storage.v3.ResourceRecord.discovered_at:type_name -> google.protobuf.Timestamp + 19, // 5: c1.storage.v3.EntitlementRecord.resource:type_name -> c1.storage.v3.ResourceRef + 17, // 6: c1.storage.v3.EntitlementRecord.annotations:type_name -> google.protobuf.Any + 18, // 7: c1.storage.v3.EntitlementRecord.discovered_at:type_name -> google.protobuf.Timestamp + 20, // 8: c1.storage.v3.GrantRecord.entitlement:type_name -> c1.storage.v3.EntitlementRef + 21, // 9: c1.storage.v3.GrantRecord.principal:type_name -> c1.storage.v3.PrincipalRef + 18, // 10: c1.storage.v3.GrantRecord.discovered_at:type_name -> google.protobuf.Timestamp 1, // 11: c1.storage.v3.GrantRecord.expansion:type_name -> c1.storage.v3.GrantExpandableRecord - 15, // 12: c1.storage.v3.GrantRecord.annotations:type_name -> google.protobuf.Any - 11, // 13: c1.storage.v3.GrantRecord.sources:type_name -> c1.storage.v3.GrantRecord.SourcesEntry - 16, // 14: c1.storage.v3.AssetRecord.discovered_at:type_name -> google.protobuf.Timestamp + 17, // 12: c1.storage.v3.GrantRecord.annotations:type_name -> google.protobuf.Any + 13, // 13: c1.storage.v3.GrantRecord.sources:type_name -> c1.storage.v3.GrantRecord.SourcesEntry + 18, // 14: c1.storage.v3.AssetRecord.discovered_at:type_name -> google.protobuf.Timestamp 0, // 15: c1.storage.v3.SyncRunRecord.type:type_name -> c1.storage.v3.SyncType - 16, // 16: c1.storage.v3.SyncRunRecord.started_at:type_name -> google.protobuf.Timestamp - 16, // 17: c1.storage.v3.SyncRunRecord.ended_at:type_name -> google.protobuf.Timestamp - 12, // 18: c1.storage.v3.SyncStatsRecord.resources_by_resource_type:type_name -> c1.storage.v3.SyncStatsRecord.ResourcesByResourceTypeEntry - 13, // 19: c1.storage.v3.SyncStatsRecord.grants_by_entitlement_resource_type:type_name -> c1.storage.v3.SyncStatsRecord.GrantsByEntitlementResourceTypeEntry - 14, // 20: c1.storage.v3.SyncStatsRecord.entitlements_by_resource_type:type_name -> c1.storage.v3.SyncStatsRecord.EntitlementsByResourceTypeEntry - 16, // 21: c1.storage.v3.SyncStatsRecord.written_at:type_name -> google.protobuf.Timestamp - 2, // 22: c1.storage.v3.GrantRecord.SourcesEntry.value:type_name -> c1.storage.v3.GrantSourceRecord - 23, // [23:23] is the sub-list for method output_type - 23, // [23:23] is the sub-list for method input_type - 23, // [23:23] is the sub-list for extension type_name - 23, // [23:23] is the sub-list for extension extendee - 0, // [0:23] is the sub-list for field type_name + 18, // 16: c1.storage.v3.SyncRunRecord.started_at:type_name -> google.protobuf.Timestamp + 18, // 17: c1.storage.v3.SyncRunRecord.ended_at:type_name -> google.protobuf.Timestamp + 14, // 18: c1.storage.v3.SyncStatsRecord.resources_by_resource_type:type_name -> c1.storage.v3.SyncStatsRecord.ResourcesByResourceTypeEntry + 15, // 19: c1.storage.v3.SyncStatsRecord.grants_by_entitlement_resource_type:type_name -> c1.storage.v3.SyncStatsRecord.GrantsByEntitlementResourceTypeEntry + 16, // 20: c1.storage.v3.SyncStatsRecord.entitlements_by_resource_type:type_name -> c1.storage.v3.SyncStatsRecord.EntitlementsByResourceTypeEntry + 18, // 21: c1.storage.v3.SyncStatsRecord.written_at:type_name -> google.protobuf.Timestamp + 18, // 22: c1.storage.v3.SourceCacheEntryRecord.discovered_at:type_name -> google.protobuf.Timestamp + 2, // 23: c1.storage.v3.GrantRecord.SourcesEntry.value:type_name -> c1.storage.v3.GrantSourceRecord + 24, // [24:24] is the sub-list for method output_type + 24, // [24:24] is the sub-list for method input_type + 24, // [24:24] is the sub-list for extension type_name + 24, // [24:24] is the sub-list for extension extendee + 0, // [0:24] is the sub-list for field type_name } func init() { file_c1_storage_v3_records_proto_init() } @@ -1842,7 +2267,7 @@ func file_c1_storage_v3_records_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_c1_storage_v3_records_proto_rawDesc), len(file_c1_storage_v3_records_proto_rawDesc)), NumEnums: 1, - NumMessages: 14, + NumMessages: 16, NumExtensions: 0, NumServices: 0, }, diff --git a/pb/c1/storage/v3/records.pb.validate.go b/pb/c1/storage/v3/records.pb.validate.go index 292877242..cd79694c1 100644 --- a/pb/c1/storage/v3/records.pb.validate.go +++ b/pb/c1/storage/v3/records.pb.validate.go @@ -544,6 +544,8 @@ func (m *ResourceRecord) validate(all bool) error { } } + // no validation rules for SourceScopeKey + if len(errors) > 0 { return ResourceRecordMultiError(errors) } @@ -746,6 +748,8 @@ func (m *EntitlementRecord) validate(all bool) error { // no validation rules for Slug + // no validation rules for SourceScopeKey + if len(errors) > 0 { return EntitlementRecordMultiError(errors) } @@ -1048,6 +1052,8 @@ func (m *GrantRecord) validate(all bool) error { } } + // no validation rules for SourceScopeKey + if len(errors) > 0 { return GrantRecordMultiError(errors) } @@ -1353,6 +1359,8 @@ func (m *SyncRunRecord) validate(all bool) error { // no validation rules for LinkedSyncId + // no validation rules for Compacted + if len(errors) > 0 { return SyncRunRecordMultiError(errors) } @@ -1683,3 +1691,254 @@ var _ interface { Cause() error ErrorName() string } = SessionRecordValidationError{} + +// Validate checks the field values on SourceCacheEntryRecord with the rules +// defined in the proto definition for this message. If any rules are +// violated, the first error encountered is returned, or nil if there are no violations. +func (m *SourceCacheEntryRecord) Validate() error { + return m.validate(false) +} + +// ValidateAll checks the field values on SourceCacheEntryRecord with the rules +// defined in the proto definition for this message. If any rules are +// violated, the result is a list of violation errors wrapped in +// SourceCacheEntryRecordMultiError, or nil if none found. +func (m *SourceCacheEntryRecord) ValidateAll() error { + return m.validate(true) +} + +func (m *SourceCacheEntryRecord) validate(all bool) error { + if m == nil { + return nil + } + + var errors []error + + // no validation rules for RowKind + + // no validation rules for ScopeKey + + // no validation rules for CacheValidator + + if all { + switch v := interface{}(m.GetDiscoveredAt()).(type) { + case interface{ ValidateAll() error }: + if err := v.ValidateAll(); err != nil { + errors = append(errors, SourceCacheEntryRecordValidationError{ + field: "DiscoveredAt", + reason: "embedded message failed validation", + cause: err, + }) + } + case interface{ Validate() error }: + if err := v.Validate(); err != nil { + errors = append(errors, SourceCacheEntryRecordValidationError{ + field: "DiscoveredAt", + reason: "embedded message failed validation", + cause: err, + }) + } + } + } else if v, ok := interface{}(m.GetDiscoveredAt()).(interface{ Validate() error }); ok { + if err := v.Validate(); err != nil { + return SourceCacheEntryRecordValidationError{ + field: "DiscoveredAt", + reason: "embedded message failed validation", + cause: err, + } + } + } + + // no validation rules for Invalidated + + if len(errors) > 0 { + return SourceCacheEntryRecordMultiError(errors) + } + + return nil +} + +// SourceCacheEntryRecordMultiError is an error wrapping multiple validation +// errors returned by SourceCacheEntryRecord.ValidateAll() if the designated +// constraints aren't met. +type SourceCacheEntryRecordMultiError []error + +// Error returns a concatenation of all the error messages it wraps. +func (m SourceCacheEntryRecordMultiError) Error() string { + msgs := make([]string, 0, len(m)) + for _, err := range m { + msgs = append(msgs, err.Error()) + } + return strings.Join(msgs, "; ") +} + +// AllErrors returns a list of validation violation errors. +func (m SourceCacheEntryRecordMultiError) AllErrors() []error { return m } + +// SourceCacheEntryRecordValidationError is the validation error returned by +// SourceCacheEntryRecord.Validate if the designated constraints aren't met. +type SourceCacheEntryRecordValidationError struct { + field string + reason string + cause error + key bool +} + +// Field function returns field value. +func (e SourceCacheEntryRecordValidationError) Field() string { return e.field } + +// Reason function returns reason value. +func (e SourceCacheEntryRecordValidationError) Reason() string { return e.reason } + +// Cause function returns cause value. +func (e SourceCacheEntryRecordValidationError) Cause() error { return e.cause } + +// Key function returns key value. +func (e SourceCacheEntryRecordValidationError) Key() bool { return e.key } + +// ErrorName returns error name. +func (e SourceCacheEntryRecordValidationError) ErrorName() string { + return "SourceCacheEntryRecordValidationError" +} + +// Error satisfies the builtin error interface +func (e SourceCacheEntryRecordValidationError) Error() string { + cause := "" + if e.cause != nil { + cause = fmt.Sprintf(" | caused by: %v", e.cause) + } + + key := "" + if e.key { + key = "key for " + } + + return fmt.Sprintf( + "invalid %sSourceCacheEntryRecord.%s: %s%s", + key, + e.field, + e.reason, + cause) +} + +var _ error = SourceCacheEntryRecordValidationError{} + +var _ interface { + Field() string + Reason() string + Key() bool + Cause() error + ErrorName() string +} = SourceCacheEntryRecordValidationError{} + +// Validate checks the field values on SourceCacheCompatRecord with the rules +// defined in the proto definition for this message. If any rules are +// violated, the first error encountered is returned, or nil if there are no violations. +func (m *SourceCacheCompatRecord) Validate() error { + return m.validate(false) +} + +// ValidateAll checks the field values on SourceCacheCompatRecord with the +// rules defined in the proto definition for this message. If any rules are +// violated, the result is a list of violation errors wrapped in +// SourceCacheCompatRecordMultiError, or nil if none found. +func (m *SourceCacheCompatRecord) ValidateAll() error { + return m.validate(true) +} + +func (m *SourceCacheCompatRecord) validate(all bool) error { + if m == nil { + return nil + } + + var errors []error + + // no validation rules for Id + + // no validation rules for ConnectorCacheGeneration + + // no validation rules for ConnectorConfigFingerprint + + // no validation rules for SdkMaterializationGeneration + + // no validation rules for SyncSelectionFingerprint + + if len(errors) > 0 { + return SourceCacheCompatRecordMultiError(errors) + } + + return nil +} + +// SourceCacheCompatRecordMultiError is an error wrapping multiple validation +// errors returned by SourceCacheCompatRecord.ValidateAll() if the designated +// constraints aren't met. +type SourceCacheCompatRecordMultiError []error + +// Error returns a concatenation of all the error messages it wraps. +func (m SourceCacheCompatRecordMultiError) Error() string { + msgs := make([]string, 0, len(m)) + for _, err := range m { + msgs = append(msgs, err.Error()) + } + return strings.Join(msgs, "; ") +} + +// AllErrors returns a list of validation violation errors. +func (m SourceCacheCompatRecordMultiError) AllErrors() []error { return m } + +// SourceCacheCompatRecordValidationError is the validation error returned by +// SourceCacheCompatRecord.Validate if the designated constraints aren't met. +type SourceCacheCompatRecordValidationError struct { + field string + reason string + cause error + key bool +} + +// Field function returns field value. +func (e SourceCacheCompatRecordValidationError) Field() string { return e.field } + +// Reason function returns reason value. +func (e SourceCacheCompatRecordValidationError) Reason() string { return e.reason } + +// Cause function returns cause value. +func (e SourceCacheCompatRecordValidationError) Cause() error { return e.cause } + +// Key function returns key value. +func (e SourceCacheCompatRecordValidationError) Key() bool { return e.key } + +// ErrorName returns error name. +func (e SourceCacheCompatRecordValidationError) ErrorName() string { + return "SourceCacheCompatRecordValidationError" +} + +// Error satisfies the builtin error interface +func (e SourceCacheCompatRecordValidationError) Error() string { + cause := "" + if e.cause != nil { + cause = fmt.Sprintf(" | caused by: %v", e.cause) + } + + key := "" + if e.key { + key = "key for " + } + + return fmt.Sprintf( + "invalid %sSourceCacheCompatRecord.%s: %s%s", + key, + e.field, + e.reason, + cause) +} + +var _ error = SourceCacheCompatRecordValidationError{} + +var _ interface { + Field() string + Reason() string + Key() bool + Cause() error + ErrorName() string +} = SourceCacheCompatRecordValidationError{} diff --git a/pb/c1/storage/v3/records_protoopaque.pb.go b/pb/c1/storage/v3/records_protoopaque.pb.go index bf2acc8c6..f4ce080cb 100644 --- a/pb/c1/storage/v3/records_protoopaque.pb.go +++ b/pb/c1/storage/v3/records_protoopaque.pb.go @@ -449,6 +449,7 @@ type ResourceRecord struct { xxx_hidden_Parent *ResourceRef `protobuf:"bytes,6,opt,name=parent,proto3"` xxx_hidden_Annotations *[]*anypb.Any `protobuf:"bytes,7,rep,name=annotations,proto3"` xxx_hidden_DiscoveredAt *timestamppb.Timestamp `protobuf:"bytes,8,opt,name=discovered_at,json=discoveredAt,proto3"` + xxx_hidden_SourceScopeKey string `protobuf:"bytes,9,opt,name=source_scope_key,json=sourceScopeKey,proto3"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -529,6 +530,13 @@ func (x *ResourceRecord) GetDiscoveredAt() *timestamppb.Timestamp { return nil } +func (x *ResourceRecord) GetSourceScopeKey() string { + if x != nil { + return x.xxx_hidden_SourceScopeKey + } + return "" +} + func (x *ResourceRecord) SetResourceTypeId(v string) { x.xxx_hidden_ResourceTypeId = v } @@ -557,6 +565,10 @@ func (x *ResourceRecord) SetDiscoveredAt(v *timestamppb.Timestamp) { x.xxx_hidden_DiscoveredAt = v } +func (x *ResourceRecord) SetSourceScopeKey(v string) { + x.xxx_hidden_SourceScopeKey = v +} + func (x *ResourceRecord) HasParent() bool { if x == nil { return false @@ -589,6 +601,9 @@ type ResourceRecord_builder struct { Parent *ResourceRef Annotations []*anypb.Any DiscoveredAt *timestamppb.Timestamp + // Source-cache scope stamp (see annotation_source_cache.proto). + // Empty for rows not produced under a source-cache scope. + SourceScopeKey string } func (b0 ResourceRecord_builder) Build() *ResourceRecord { @@ -602,6 +617,7 @@ func (b0 ResourceRecord_builder) Build() *ResourceRecord { x.xxx_hidden_Parent = b.Parent x.xxx_hidden_Annotations = &b.Annotations x.xxx_hidden_DiscoveredAt = b.DiscoveredAt + x.xxx_hidden_SourceScopeKey = b.SourceScopeKey return m0 } @@ -616,6 +632,7 @@ type EntitlementRecord struct { xxx_hidden_DiscoveredAt *timestamppb.Timestamp `protobuf:"bytes,8,opt,name=discovered_at,json=discoveredAt,proto3"` xxx_hidden_Slug string `protobuf:"bytes,9,opt,name=slug,proto3"` xxx_hidden_GrantableToResourceTypeIds []string `protobuf:"bytes,10,rep,name=grantable_to_resource_type_ids,json=grantableToResourceTypeIds,proto3"` + xxx_hidden_SourceScopeKey string `protobuf:"bytes,11,opt,name=source_scope_key,json=sourceScopeKey,proto3"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -710,6 +727,13 @@ func (x *EntitlementRecord) GetGrantableToResourceTypeIds() []string { return nil } +func (x *EntitlementRecord) GetSourceScopeKey() string { + if x != nil { + return x.xxx_hidden_SourceScopeKey + } + return "" +} + func (x *EntitlementRecord) SetExternalId(v string) { x.xxx_hidden_ExternalId = v } @@ -746,6 +770,10 @@ func (x *EntitlementRecord) SetGrantableToResourceTypeIds(v []string) { x.xxx_hidden_GrantableToResourceTypeIds = v } +func (x *EntitlementRecord) SetSourceScopeKey(v string) { + x.xxx_hidden_SourceScopeKey = v +} + func (x *EntitlementRecord) HasResource() bool { if x == nil { return false @@ -792,6 +820,9 @@ type EntitlementRecord_builder struct { // resource_types table when needed. Consumed by // pkg/sync/syncer.go's principal-type narrowing. GrantableToResourceTypeIds []string + // Source-cache scope stamp (see annotation_source_cache.proto). + // Empty for rows not produced under a source-cache scope. + SourceScopeKey string } func (b0 EntitlementRecord_builder) Build() *EntitlementRecord { @@ -807,6 +838,7 @@ func (b0 EntitlementRecord_builder) Build() *EntitlementRecord { x.xxx_hidden_DiscoveredAt = b.DiscoveredAt x.xxx_hidden_Slug = b.Slug x.xxx_hidden_GrantableToResourceTypeIds = b.GrantableToResourceTypeIds + x.xxx_hidden_SourceScopeKey = b.SourceScopeKey return m0 } @@ -820,6 +852,7 @@ type GrantRecord struct { xxx_hidden_NeedsExpansion bool `protobuf:"varint,7,opt,name=needs_expansion,json=needsExpansion,proto3"` xxx_hidden_Annotations *[]*anypb.Any `protobuf:"bytes,8,rep,name=annotations,proto3"` xxx_hidden_Sources map[string]*GrantSourceRecord `protobuf:"bytes,9,rep,name=sources,proto3" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + xxx_hidden_SourceScopeKey string `protobuf:"bytes,10,opt,name=source_scope_key,json=sourceScopeKey,proto3"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -907,6 +940,13 @@ func (x *GrantRecord) GetSources() map[string]*GrantSourceRecord { return nil } +func (x *GrantRecord) GetSourceScopeKey() string { + if x != nil { + return x.xxx_hidden_SourceScopeKey + } + return "" +} + func (x *GrantRecord) SetExternalId(v string) { x.xxx_hidden_ExternalId = v } @@ -939,6 +979,10 @@ func (x *GrantRecord) SetSources(v map[string]*GrantSourceRecord) { x.xxx_hidden_Sources = v } +func (x *GrantRecord) SetSourceScopeKey(v string) { + x.xxx_hidden_SourceScopeKey = v +} + func (x *GrantRecord) HasEntitlement() bool { if x == nil { return false @@ -1007,6 +1051,12 @@ type GrantRecord_builder struct { // map — same wire shape as // c1.connector.v2.GrantSources.sources. Sources map[string]*GrantSourceRecord + // Source-cache scope stamp (see annotation_source_cache.proto). + // Empty for rows not produced under a source-cache scope — notably + // expander-derived grants, which are recreated by expansion each sync + // and never replayed. StoreExpandedGrants preserves this field on + // rewrites of existing rows, exactly like expansion/needs_expansion. + SourceScopeKey string } func (b0 GrantRecord_builder) Build() *GrantRecord { @@ -1021,6 +1071,7 @@ func (b0 GrantRecord_builder) Build() *GrantRecord { x.xxx_hidden_NeedsExpansion = b.NeedsExpansion x.xxx_hidden_Annotations = &b.Annotations x.xxx_hidden_Sources = b.Sources + x.xxx_hidden_SourceScopeKey = b.SourceScopeKey return m0 } @@ -1161,6 +1212,7 @@ type SyncRunRecord struct { xxx_hidden_SyncToken string `protobuf:"bytes,6,opt,name=sync_token,json=syncToken,proto3"` xxx_hidden_SupportsDiff bool `protobuf:"varint,7,opt,name=supports_diff,json=supportsDiff,proto3"` xxx_hidden_LinkedSyncId string `protobuf:"bytes,8,opt,name=linked_sync_id,json=linkedSyncId,proto3"` + xxx_hidden_Compacted bool `protobuf:"varint,9,opt,name=compacted,proto3"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -1246,6 +1298,13 @@ func (x *SyncRunRecord) GetLinkedSyncId() string { return "" } +func (x *SyncRunRecord) GetCompacted() bool { + if x != nil { + return x.xxx_hidden_Compacted + } + return false +} + func (x *SyncRunRecord) SetSyncId(v string) { x.xxx_hidden_SyncId = v } @@ -1278,6 +1337,10 @@ func (x *SyncRunRecord) SetLinkedSyncId(v string) { x.xxx_hidden_LinkedSyncId = v } +func (x *SyncRunRecord) SetCompacted(v bool) { + x.xxx_hidden_Compacted = v +} + func (x *SyncRunRecord) HasStartedAt() bool { if x == nil { return false @@ -1311,6 +1374,20 @@ type SyncRunRecord_builder struct { SyncToken string SupportsDiff bool LinkedSyncId string + // compacted marks a sync produced by compaction (fold or rebuild) + // rather than by a real connector run. Compacted artifacts are + // keep-newer UPSERT merges — base rows a newer input deleted survive — + // so no input sync's upstream validators (source-cache validators) describe + // their contents. + // + // "Can this sync be used as a source-cache replay source?" is the + // predicate c1zstore.SyncRun.UsableAsReplaySource: type == FULL and + // !compacted. The syncer enforces it — feeding a compacted or + // partial/derived sync to WithPreviousSyncC1ZPath degrades to a cold + // (full-fetch) sync. Orchestrators deciding which artifact to + // materialize as the previous sync should apply the same predicate + // instead of guessing from provenance. + Compacted bool } func (b0 SyncRunRecord_builder) Build() *SyncRunRecord { @@ -1325,6 +1402,7 @@ func (b0 SyncRunRecord_builder) Build() *SyncRunRecord { x.xxx_hidden_SyncToken = b.SyncToken x.xxx_hidden_SupportsDiff = b.SupportsDiff x.xxx_hidden_LinkedSyncId = b.LinkedSyncId + x.xxx_hidden_Compacted = b.Compacted return m0 } @@ -1622,6 +1700,283 @@ func (b0 SessionRecord_builder) Build() *SessionRecord { return m0 } +// SourceCacheEntryRecord is the per-scope manifest entry for source-cache +// replay (see c1/connector/v2/annotation_source_cache.proto). One entry +// per (row_kind, scope_key), written for every freshly fetched scope — +// including zero-row responses — and rewritten on replay with the scope's +// current validator. The previous sync's entries (read from the previous +// c1z) are the lookup surface connectors revalidate against. +type SourceCacheEntryRecord struct { + state protoimpl.MessageState `protogen:"opaque.v1"` + xxx_hidden_RowKind string `protobuf:"bytes,1,opt,name=row_kind,json=rowKind,proto3"` + xxx_hidden_ScopeKey string `protobuf:"bytes,2,opt,name=scope_key,json=scopeKey,proto3"` + xxx_hidden_CacheValidator string `protobuf:"bytes,3,opt,name=cache_validator,json=cacheValidator,proto3"` + xxx_hidden_DiscoveredAt *timestamppb.Timestamp `protobuf:"bytes,4,opt,name=discovered_at,json=discoveredAt,proto3"` + xxx_hidden_Invalidated bool `protobuf:"varint,5,opt,name=invalidated,proto3"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SourceCacheEntryRecord) Reset() { + *x = SourceCacheEntryRecord{} + mi := &file_c1_storage_v3_records_proto_msgTypes[10] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SourceCacheEntryRecord) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SourceCacheEntryRecord) ProtoMessage() {} + +func (x *SourceCacheEntryRecord) ProtoReflect() protoreflect.Message { + mi := &file_c1_storage_v3_records_proto_msgTypes[10] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *SourceCacheEntryRecord) GetRowKind() string { + if x != nil { + return x.xxx_hidden_RowKind + } + return "" +} + +func (x *SourceCacheEntryRecord) GetScopeKey() string { + if x != nil { + return x.xxx_hidden_ScopeKey + } + return "" +} + +func (x *SourceCacheEntryRecord) GetCacheValidator() string { + if x != nil { + return x.xxx_hidden_CacheValidator + } + return "" +} + +func (x *SourceCacheEntryRecord) GetDiscoveredAt() *timestamppb.Timestamp { + if x != nil { + return x.xxx_hidden_DiscoveredAt + } + return nil +} + +func (x *SourceCacheEntryRecord) GetInvalidated() bool { + if x != nil { + return x.xxx_hidden_Invalidated + } + return false +} + +func (x *SourceCacheEntryRecord) SetRowKind(v string) { + x.xxx_hidden_RowKind = v +} + +func (x *SourceCacheEntryRecord) SetScopeKey(v string) { + x.xxx_hidden_ScopeKey = v +} + +func (x *SourceCacheEntryRecord) SetCacheValidator(v string) { + x.xxx_hidden_CacheValidator = v +} + +func (x *SourceCacheEntryRecord) SetDiscoveredAt(v *timestamppb.Timestamp) { + x.xxx_hidden_DiscoveredAt = v +} + +func (x *SourceCacheEntryRecord) SetInvalidated(v bool) { + x.xxx_hidden_Invalidated = v +} + +func (x *SourceCacheEntryRecord) HasDiscoveredAt() bool { + if x == nil { + return false + } + return x.xxx_hidden_DiscoveredAt != nil +} + +func (x *SourceCacheEntryRecord) ClearDiscoveredAt() { + x.xxx_hidden_DiscoveredAt = nil +} + +type SourceCacheEntryRecord_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + // Row kind partition: "resources", "entitlements", or "grants" + // (pkg/sourcecache.RowKind values). + RowKind string + // Connector-computed stable scope identifier (conventionally lowercase + // hex when hashed via sourcecache.HashScope; any byte-stable string). + ScopeKey string + // Opaque upstream validator: HTTP ETag, delta token, etc. + CacheValidator string + DiscoveredAt *timestamppb.Timestamp + // Set when the dangling-reference drops (ingestion invariants I7/I8/I9, + // pkg/sync/ingest_invariants.go) deleted rows stamped with this scope: + // the artifact no longer holds the full row set the validator vouches + // for, so a future sync must not 304-replay it. Lookups treat an + // invalidated entry as a miss (the scope re-fetches cold and converges + // with a cold sync); the entry itself is kept so the scope's surviving + // stamped rows do not read as an I6 orphan (lost manifest write). + Invalidated bool +} + +func (b0 SourceCacheEntryRecord_builder) Build() *SourceCacheEntryRecord { + m0 := &SourceCacheEntryRecord{} + b, x := &b0, m0 + _, _ = b, x + x.xxx_hidden_RowKind = b.RowKind + x.xxx_hidden_ScopeKey = b.ScopeKey + x.xxx_hidden_CacheValidator = b.CacheValidator + x.xxx_hidden_DiscoveredAt = b.DiscoveredAt + x.xxx_hidden_Invalidated = b.Invalidated + return m0 +} + +// SourceCacheCompatRecord is the sync's replay-compatibility key: the +// exact conditions under which this artifact's source-cache manifest was +// recorded. A future sync may replay from this artifact ONLY when its +// own key matches byte-exactly on every component; an absent, unreadable, +// or mismatched record degrades that sync to cold (no-op lookup). +// Compatibility is never inferred from versions — each component is an +// explicit declaration by its owner. +// +// One record per sync, written when the write side of the source cache +// enables (before any manifest entry), cleared by compaction folds with +// the manifest. +type SourceCacheCompatRecord struct { + state protoimpl.MessageState `protogen:"opaque.v1"` + xxx_hidden_Id string `protobuf:"bytes,1,opt,name=id,proto3"` + xxx_hidden_ConnectorCacheGeneration string `protobuf:"bytes,2,opt,name=connector_cache_generation,json=connectorCacheGeneration,proto3"` + xxx_hidden_ConnectorConfigFingerprint string `protobuf:"bytes,3,opt,name=connector_config_fingerprint,json=connectorConfigFingerprint,proto3"` + xxx_hidden_SdkMaterializationGeneration string `protobuf:"bytes,4,opt,name=sdk_materialization_generation,json=sdkMaterializationGeneration,proto3"` + xxx_hidden_SyncSelectionFingerprint string `protobuf:"bytes,5,opt,name=sync_selection_fingerprint,json=syncSelectionFingerprint,proto3"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SourceCacheCompatRecord) Reset() { + *x = SourceCacheCompatRecord{} + mi := &file_c1_storage_v3_records_proto_msgTypes[11] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SourceCacheCompatRecord) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SourceCacheCompatRecord) ProtoMessage() {} + +func (x *SourceCacheCompatRecord) ProtoReflect() protoreflect.Message { + mi := &file_c1_storage_v3_records_proto_msgTypes[11] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *SourceCacheCompatRecord) GetId() string { + if x != nil { + return x.xxx_hidden_Id + } + return "" +} + +func (x *SourceCacheCompatRecord) GetConnectorCacheGeneration() string { + if x != nil { + return x.xxx_hidden_ConnectorCacheGeneration + } + return "" +} + +func (x *SourceCacheCompatRecord) GetConnectorConfigFingerprint() string { + if x != nil { + return x.xxx_hidden_ConnectorConfigFingerprint + } + return "" +} + +func (x *SourceCacheCompatRecord) GetSdkMaterializationGeneration() string { + if x != nil { + return x.xxx_hidden_SdkMaterializationGeneration + } + return "" +} + +func (x *SourceCacheCompatRecord) GetSyncSelectionFingerprint() string { + if x != nil { + return x.xxx_hidden_SyncSelectionFingerprint + } + return "" +} + +func (x *SourceCacheCompatRecord) SetId(v string) { + x.xxx_hidden_Id = v +} + +func (x *SourceCacheCompatRecord) SetConnectorCacheGeneration(v string) { + x.xxx_hidden_ConnectorCacheGeneration = v +} + +func (x *SourceCacheCompatRecord) SetConnectorConfigFingerprint(v string) { + x.xxx_hidden_ConnectorConfigFingerprint = v +} + +func (x *SourceCacheCompatRecord) SetSdkMaterializationGeneration(v string) { + x.xxx_hidden_SdkMaterializationGeneration = v +} + +func (x *SourceCacheCompatRecord) SetSyncSelectionFingerprint(v string) { + x.xxx_hidden_SyncSelectionFingerprint = v +} + +type SourceCacheCompatRecord_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + // Fixed singleton id ("compat"); present so the record satisfies the + // table shape. + Id string + // Connector-declared components (SourceCacheCapability annotations): + // the connector's cache generation and configuration/permission + // fingerprint. + ConnectorCacheGeneration string + ConnectorConfigFingerprint string + // SDK materialization-policy generation: bumped when the SDK changes + // how response rows or their side effects materialize into the store + // in a replay-visible way (sourcecache.MaterializationPolicyGeneration). + SdkMaterializationGeneration string + // SDK sync-selection fingerprint: digest of the sync-shaping inputs + // (enabled resource types, skip flags) — a selection change means the + // prior artifact's scopes cover a different row universe. + SyncSelectionFingerprint string +} + +func (b0 SourceCacheCompatRecord_builder) Build() *SourceCacheCompatRecord { + m0 := &SourceCacheCompatRecord{} + b, x := &b0, m0 + _, _ = b, x + x.xxx_hidden_Id = b.Id + x.xxx_hidden_ConnectorCacheGeneration = b.ConnectorCacheGeneration + x.xxx_hidden_ConnectorConfigFingerprint = b.ConnectorConfigFingerprint + x.xxx_hidden_SdkMaterializationGeneration = b.SdkMaterializationGeneration + x.xxx_hidden_SyncSelectionFingerprint = b.SyncSelectionFingerprint + return m0 +} + var File_c1_storage_v3_records_proto protoreflect.FileDescriptor const file_c1_storage_v3_records_proto_rawDesc = "" + @@ -1646,7 +2001,7 @@ const file_c1_storage_v3_records_proto_rawDesc = "" + "\rdiscovered_at\x18\x06 \x01(\v2\x1a.google.protobuf.TimestampR\fdiscoveredAt\x12 \n" + "\vdescription\x18\a \x01(\tR\vdescription\x12-\n" + "\x12sourced_externally\x18\b \x01(\bR\x11sourcedExternally:!\x82\xf9+\x1d\n" + - "\x0eresource_types\x12\vexternal_idJ\x04\b\x01\x10\x02R\async_id\"\xbc\x03\n" + + "\x0eresource_types\x12\vexternal_idJ\x04\b\x01\x10\x02R\async_id\"\x95\x04\n" + "\x0eResourceRecord\x12(\n" + "\x10resource_type_id\x18\x02 \x01(\tR\x0eresourceTypeId\x12\x1f\n" + "\vresource_id\x18\x03 \x01(\tR\n" + @@ -1656,8 +2011,10 @@ const file_c1_storage_v3_records_proto_rawDesc = "" + "\x06parent\x18\x06 \x01(\v2\x1a.c1.storage.v3.ResourceRefB.\x8a\xf9+*\n" + "\tby_parent\x1a\x10resource_type_id\x1a\vresource_idR\x06parent\x126\n" + "\vannotations\x18\a \x03(\v2\x14.google.protobuf.AnyR\vannotations\x12?\n" + - "\rdiscovered_at\x18\b \x01(\v2\x1a.google.protobuf.TimestampR\fdiscoveredAt:.\x82\xf9+*\n" + - "\tresources\x12\x10resource_type_id\x12\vresource_idJ\x04\b\x01\x10\x02R\async_id\"\xfe\x03\n" + + "\rdiscovered_at\x18\b \x01(\v2\x1a.google.protobuf.TimestampR\fdiscoveredAt\x12W\n" + + "\x10source_scope_key\x18\t \x01(\tB-\x8a\xf9+)\n" + + "\x0fby_source_scope\"\x16source_scope_key != ''R\x0esourceScopeKey:.\x82\xf9+*\n" + + "\tresources\x12\x10resource_type_id\x12\vresource_idJ\x04\b\x01\x10\x02R\async_id\"\xd7\x04\n" + "\x11EntitlementRecord\x12\x1f\n" + "\vexternal_id\x18\x02 \x01(\tR\n" + "externalId\x12h\n" + @@ -1670,8 +2027,10 @@ const file_c1_storage_v3_records_proto_rawDesc = "" + "\rdiscovered_at\x18\b \x01(\v2\x1a.google.protobuf.TimestampR\fdiscoveredAt\x12\x12\n" + "\x04slug\x18\t \x01(\tR\x04slug\x12B\n" + "\x1egrantable_to_resource_type_ids\x18\n" + - " \x03(\tR\x1agrantableToResourceTypeIds:\x1f\x82\xf9+\x1b\n" + - "\fentitlements\x12\vexternal_idJ\x04\b\x01\x10\x02R\async_id\"\x9a\x06\n" + + " \x03(\tR\x1agrantableToResourceTypeIds\x12W\n" + + "\x10source_scope_key\x18\v \x01(\tB-\x8a\xf9+)\n" + + "\x0fby_source_scope\"\x16source_scope_key != ''R\x0esourceScopeKey:\x1f\x82\xf9+\x1b\n" + + "\fentitlements\x12\vexternal_idJ\x04\b\x01\x10\x02R\async_id\"\xf3\x06\n" + "\vGrantRecord\x12\x1f\n" + "\vexternal_id\x18\x02 \x01(\tR\n" + "externalId\x12\x98\x01\n" + @@ -1684,7 +2043,10 @@ const file_c1_storage_v3_records_proto_rawDesc = "" + "\x0fneeds_expansion\x18\a \x01(\bB0\x8a\xf9+,\n" + "\x12by_needs_expansion\"\x16needs_expansion = trueR\x0eneedsExpansion\x126\n" + "\vannotations\x18\b \x03(\v2\x14.google.protobuf.AnyR\vannotations\x12A\n" + - "\asources\x18\t \x03(\v2'.c1.storage.v3.GrantRecord.SourcesEntryR\asources\x1a\\\n" + + "\asources\x18\t \x03(\v2'.c1.storage.v3.GrantRecord.SourcesEntryR\asources\x12W\n" + + "\x10source_scope_key\x18\n" + + " \x01(\tB-\x8a\xf9+)\n" + + "\x0fby_source_scope\"\x16source_scope_key != ''R\x0esourceScopeKey\x1a\\\n" + "\fSourcesEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x126\n" + "\x05value\x18\x02 \x01(\v2 .c1.storage.v3.GrantSourceRecordR\x05value:\x028\x01:\x19\x82\xf9+\x15\n" + @@ -1696,7 +2058,7 @@ const file_c1_storage_v3_records_proto_rawDesc = "" + "\fcontent_type\x18\x03 \x01(\tR\vcontentType\x12\x12\n" + "\x04data\x18\x04 \x01(\fR\x04data\x12?\n" + "\rdiscovered_at\x18\x05 \x01(\v2\x1a.google.protobuf.TimestampR\fdiscoveredAt:\"\x82\xf9+\x1e\n" + - "\x06assets\x12\async_id\x12\vexternal_id\"\xf1\x02\n" + + "\x06assets\x12\async_id\x12\vexternal_id\"\x8f\x03\n" + "\rSyncRunRecord\x12\x17\n" + "\async_id\x18\x01 \x01(\tR\x06syncId\x12+\n" + "\x04type\x18\x02 \x01(\x0e2\x17.c1.storage.v3.SyncTypeR\x04type\x12$\n" + @@ -1707,7 +2069,8 @@ const file_c1_storage_v3_records_proto_rawDesc = "" + "\n" + "sync_token\x18\x06 \x01(\tR\tsyncToken\x12#\n" + "\rsupports_diff\x18\a \x01(\bR\fsupportsDiff\x12$\n" + - "\x0elinked_sync_id\x18\b \x01(\tR\flinkedSyncId:\x18\x82\xf9+\x14\n" + + "\x0elinked_sync_id\x18\b \x01(\tR\flinkedSyncId\x12\x1c\n" + + "\tcompacted\x18\t \x01(\bR\tcompacted:\x18\x82\xf9+\x14\n" + "\tsync_runs\x12\async_id\"\xff\x06\n" + "\x0fSyncStatsRecord\x12\x17\n" + "\async_id\x18\x01 \x01(\tR\x06syncId\x12%\n" + @@ -1734,7 +2097,21 @@ const file_c1_storage_v3_records_proto_rawDesc = "" + "\async_id\x18\x01 \x01(\tR\x06syncId\x12\x10\n" + "\x03key\x18\x02 \x01(\tR\x03key\x12\x14\n" + "\x05value\x18\x03 \x01(\fR\x05value:\x1c\x82\xf9+\x18\n" + - "\bsessions\x12\async_id\x12\x03key*\xae\x01\n" + + "\bsessions\x12\async_id\x12\x03key\"\x8d\x02\n" + + "\x16SourceCacheEntryRecord\x12\x19\n" + + "\brow_kind\x18\x01 \x01(\tR\arowKind\x12\x1b\n" + + "\tscope_key\x18\x02 \x01(\tR\bscopeKey\x12'\n" + + "\x0fcache_validator\x18\x03 \x01(\tR\x0ecacheValidator\x12?\n" + + "\rdiscovered_at\x18\x04 \x01(\v2\x1a.google.protobuf.TimestampR\fdiscoveredAt\x12 \n" + + "\vinvalidated\x18\x05 \x01(\bR\vinvalidated:/\x82\xf9++\n" + + "\x14source_cache_entries\x12\brow_kind\x12\tscope_key\"\xcc\x02\n" + + "\x17SourceCacheCompatRecord\x12\x0e\n" + + "\x02id\x18\x01 \x01(\tR\x02id\x12<\n" + + "\x1aconnector_cache_generation\x18\x02 \x01(\tR\x18connectorCacheGeneration\x12@\n" + + "\x1cconnector_config_fingerprint\x18\x03 \x01(\tR\x1aconnectorConfigFingerprint\x12D\n" + + "\x1esdk_materialization_generation\x18\x04 \x01(\tR\x1csdkMaterializationGeneration\x12<\n" + + "\x1async_selection_fingerprint\x18\x05 \x01(\tR\x18syncSelectionFingerprint:\x1d\x82\xf9+\x19\n" + + "\x13source_cache_compat\x12\x02id*\xae\x01\n" + "\bSyncType\x12\x19\n" + "\x15SYNC_TYPE_UNSPECIFIED\x10\x00\x12\x12\n" + "\x0eSYNC_TYPE_FULL\x10\x01\x12\x15\n" + @@ -1744,58 +2121,61 @@ const file_c1_storage_v3_records_proto_rawDesc = "" + "\x1bSYNC_TYPE_PARTIAL_DELETIONS\x10\x05B4Z2github.com/conductorone/baton-sdk/pb/c1/storage/v3b\x06proto3" var file_c1_storage_v3_records_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_c1_storage_v3_records_proto_msgTypes = make([]protoimpl.MessageInfo, 14) +var file_c1_storage_v3_records_proto_msgTypes = make([]protoimpl.MessageInfo, 16) var file_c1_storage_v3_records_proto_goTypes = []any{ - (SyncType)(0), // 0: c1.storage.v3.SyncType - (*GrantExpandableRecord)(nil), // 1: c1.storage.v3.GrantExpandableRecord - (*GrantSourceRecord)(nil), // 2: c1.storage.v3.GrantSourceRecord - (*ResourceTypeRecord)(nil), // 3: c1.storage.v3.ResourceTypeRecord - (*ResourceRecord)(nil), // 4: c1.storage.v3.ResourceRecord - (*EntitlementRecord)(nil), // 5: c1.storage.v3.EntitlementRecord - (*GrantRecord)(nil), // 6: c1.storage.v3.GrantRecord - (*AssetRecord)(nil), // 7: c1.storage.v3.AssetRecord - (*SyncRunRecord)(nil), // 8: c1.storage.v3.SyncRunRecord - (*SyncStatsRecord)(nil), // 9: c1.storage.v3.SyncStatsRecord - (*SessionRecord)(nil), // 10: c1.storage.v3.SessionRecord - nil, // 11: c1.storage.v3.GrantRecord.SourcesEntry - nil, // 12: c1.storage.v3.SyncStatsRecord.ResourcesByResourceTypeEntry - nil, // 13: c1.storage.v3.SyncStatsRecord.GrantsByEntitlementResourceTypeEntry - nil, // 14: c1.storage.v3.SyncStatsRecord.EntitlementsByResourceTypeEntry - (*anypb.Any)(nil), // 15: google.protobuf.Any - (*timestamppb.Timestamp)(nil), // 16: google.protobuf.Timestamp - (*ResourceRef)(nil), // 17: c1.storage.v3.ResourceRef - (*EntitlementRef)(nil), // 18: c1.storage.v3.EntitlementRef - (*PrincipalRef)(nil), // 19: c1.storage.v3.PrincipalRef + (SyncType)(0), // 0: c1.storage.v3.SyncType + (*GrantExpandableRecord)(nil), // 1: c1.storage.v3.GrantExpandableRecord + (*GrantSourceRecord)(nil), // 2: c1.storage.v3.GrantSourceRecord + (*ResourceTypeRecord)(nil), // 3: c1.storage.v3.ResourceTypeRecord + (*ResourceRecord)(nil), // 4: c1.storage.v3.ResourceRecord + (*EntitlementRecord)(nil), // 5: c1.storage.v3.EntitlementRecord + (*GrantRecord)(nil), // 6: c1.storage.v3.GrantRecord + (*AssetRecord)(nil), // 7: c1.storage.v3.AssetRecord + (*SyncRunRecord)(nil), // 8: c1.storage.v3.SyncRunRecord + (*SyncStatsRecord)(nil), // 9: c1.storage.v3.SyncStatsRecord + (*SessionRecord)(nil), // 10: c1.storage.v3.SessionRecord + (*SourceCacheEntryRecord)(nil), // 11: c1.storage.v3.SourceCacheEntryRecord + (*SourceCacheCompatRecord)(nil), // 12: c1.storage.v3.SourceCacheCompatRecord + nil, // 13: c1.storage.v3.GrantRecord.SourcesEntry + nil, // 14: c1.storage.v3.SyncStatsRecord.ResourcesByResourceTypeEntry + nil, // 15: c1.storage.v3.SyncStatsRecord.GrantsByEntitlementResourceTypeEntry + nil, // 16: c1.storage.v3.SyncStatsRecord.EntitlementsByResourceTypeEntry + (*anypb.Any)(nil), // 17: google.protobuf.Any + (*timestamppb.Timestamp)(nil), // 18: google.protobuf.Timestamp + (*ResourceRef)(nil), // 19: c1.storage.v3.ResourceRef + (*EntitlementRef)(nil), // 20: c1.storage.v3.EntitlementRef + (*PrincipalRef)(nil), // 21: c1.storage.v3.PrincipalRef } var file_c1_storage_v3_records_proto_depIdxs = []int32{ - 15, // 0: c1.storage.v3.ResourceTypeRecord.annotations:type_name -> google.protobuf.Any - 16, // 1: c1.storage.v3.ResourceTypeRecord.discovered_at:type_name -> google.protobuf.Timestamp - 17, // 2: c1.storage.v3.ResourceRecord.parent:type_name -> c1.storage.v3.ResourceRef - 15, // 3: c1.storage.v3.ResourceRecord.annotations:type_name -> google.protobuf.Any - 16, // 4: c1.storage.v3.ResourceRecord.discovered_at:type_name -> google.protobuf.Timestamp - 17, // 5: c1.storage.v3.EntitlementRecord.resource:type_name -> c1.storage.v3.ResourceRef - 15, // 6: c1.storage.v3.EntitlementRecord.annotations:type_name -> google.protobuf.Any - 16, // 7: c1.storage.v3.EntitlementRecord.discovered_at:type_name -> google.protobuf.Timestamp - 18, // 8: c1.storage.v3.GrantRecord.entitlement:type_name -> c1.storage.v3.EntitlementRef - 19, // 9: c1.storage.v3.GrantRecord.principal:type_name -> c1.storage.v3.PrincipalRef - 16, // 10: c1.storage.v3.GrantRecord.discovered_at:type_name -> google.protobuf.Timestamp + 17, // 0: c1.storage.v3.ResourceTypeRecord.annotations:type_name -> google.protobuf.Any + 18, // 1: c1.storage.v3.ResourceTypeRecord.discovered_at:type_name -> google.protobuf.Timestamp + 19, // 2: c1.storage.v3.ResourceRecord.parent:type_name -> c1.storage.v3.ResourceRef + 17, // 3: c1.storage.v3.ResourceRecord.annotations:type_name -> google.protobuf.Any + 18, // 4: c1.storage.v3.ResourceRecord.discovered_at:type_name -> google.protobuf.Timestamp + 19, // 5: c1.storage.v3.EntitlementRecord.resource:type_name -> c1.storage.v3.ResourceRef + 17, // 6: c1.storage.v3.EntitlementRecord.annotations:type_name -> google.protobuf.Any + 18, // 7: c1.storage.v3.EntitlementRecord.discovered_at:type_name -> google.protobuf.Timestamp + 20, // 8: c1.storage.v3.GrantRecord.entitlement:type_name -> c1.storage.v3.EntitlementRef + 21, // 9: c1.storage.v3.GrantRecord.principal:type_name -> c1.storage.v3.PrincipalRef + 18, // 10: c1.storage.v3.GrantRecord.discovered_at:type_name -> google.protobuf.Timestamp 1, // 11: c1.storage.v3.GrantRecord.expansion:type_name -> c1.storage.v3.GrantExpandableRecord - 15, // 12: c1.storage.v3.GrantRecord.annotations:type_name -> google.protobuf.Any - 11, // 13: c1.storage.v3.GrantRecord.sources:type_name -> c1.storage.v3.GrantRecord.SourcesEntry - 16, // 14: c1.storage.v3.AssetRecord.discovered_at:type_name -> google.protobuf.Timestamp + 17, // 12: c1.storage.v3.GrantRecord.annotations:type_name -> google.protobuf.Any + 13, // 13: c1.storage.v3.GrantRecord.sources:type_name -> c1.storage.v3.GrantRecord.SourcesEntry + 18, // 14: c1.storage.v3.AssetRecord.discovered_at:type_name -> google.protobuf.Timestamp 0, // 15: c1.storage.v3.SyncRunRecord.type:type_name -> c1.storage.v3.SyncType - 16, // 16: c1.storage.v3.SyncRunRecord.started_at:type_name -> google.protobuf.Timestamp - 16, // 17: c1.storage.v3.SyncRunRecord.ended_at:type_name -> google.protobuf.Timestamp - 12, // 18: c1.storage.v3.SyncStatsRecord.resources_by_resource_type:type_name -> c1.storage.v3.SyncStatsRecord.ResourcesByResourceTypeEntry - 13, // 19: c1.storage.v3.SyncStatsRecord.grants_by_entitlement_resource_type:type_name -> c1.storage.v3.SyncStatsRecord.GrantsByEntitlementResourceTypeEntry - 14, // 20: c1.storage.v3.SyncStatsRecord.entitlements_by_resource_type:type_name -> c1.storage.v3.SyncStatsRecord.EntitlementsByResourceTypeEntry - 16, // 21: c1.storage.v3.SyncStatsRecord.written_at:type_name -> google.protobuf.Timestamp - 2, // 22: c1.storage.v3.GrantRecord.SourcesEntry.value:type_name -> c1.storage.v3.GrantSourceRecord - 23, // [23:23] is the sub-list for method output_type - 23, // [23:23] is the sub-list for method input_type - 23, // [23:23] is the sub-list for extension type_name - 23, // [23:23] is the sub-list for extension extendee - 0, // [0:23] is the sub-list for field type_name + 18, // 16: c1.storage.v3.SyncRunRecord.started_at:type_name -> google.protobuf.Timestamp + 18, // 17: c1.storage.v3.SyncRunRecord.ended_at:type_name -> google.protobuf.Timestamp + 14, // 18: c1.storage.v3.SyncStatsRecord.resources_by_resource_type:type_name -> c1.storage.v3.SyncStatsRecord.ResourcesByResourceTypeEntry + 15, // 19: c1.storage.v3.SyncStatsRecord.grants_by_entitlement_resource_type:type_name -> c1.storage.v3.SyncStatsRecord.GrantsByEntitlementResourceTypeEntry + 16, // 20: c1.storage.v3.SyncStatsRecord.entitlements_by_resource_type:type_name -> c1.storage.v3.SyncStatsRecord.EntitlementsByResourceTypeEntry + 18, // 21: c1.storage.v3.SyncStatsRecord.written_at:type_name -> google.protobuf.Timestamp + 18, // 22: c1.storage.v3.SourceCacheEntryRecord.discovered_at:type_name -> google.protobuf.Timestamp + 2, // 23: c1.storage.v3.GrantRecord.SourcesEntry.value:type_name -> c1.storage.v3.GrantSourceRecord + 24, // [24:24] is the sub-list for method output_type + 24, // [24:24] is the sub-list for method input_type + 24, // [24:24] is the sub-list for extension type_name + 24, // [24:24] is the sub-list for extension extendee + 0, // [0:24] is the sub-list for field type_name } func init() { file_c1_storage_v3_records_proto_init() } @@ -1811,7 +2191,7 @@ func file_c1_storage_v3_records_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_c1_storage_v3_records_proto_rawDesc), len(file_c1_storage_v3_records_proto_rawDesc)), NumEnums: 1, - NumMessages: 14, + NumMessages: 16, NumExtensions: 0, NumServices: 0, }, diff --git a/pkg/cli/cli.go b/pkg/cli/cli.go index 036e9ee9a..59d872484 100644 --- a/pkg/cli/cli.go +++ b/pkg/cli/cli.go @@ -11,6 +11,7 @@ import ( "github.com/conductorone/baton-sdk/pkg/connectorbuilder" "github.com/conductorone/baton-sdk/pkg/field" + "github.com/conductorone/baton-sdk/pkg/sourcecache" "github.com/conductorone/baton-sdk/pkg/types" "github.com/conductorone/baton-sdk/pkg/types/sessions" "github.com/spf13/cobra" @@ -20,7 +21,12 @@ import ( ) type RunTimeOpts struct { - SessionStore sessions.SessionStore + SessionStore sessions.SessionStore + // SourceCacheLookup resolves a scope's previous-sync validator for + // source-cache replay (see pkg/sourcecache). In subprocess mode this + // is a gRPC client to the parent SDK's BatonSourceCacheService; when + // unset the framework falls back to NoopLookup. + SourceCacheLookup sourcecache.Lookup TokenSource oauth2.TokenSource SelectedAuthMethod string SyncResourceTypeIDs []string diff --git a/pkg/cli/commands.go b/pkg/cli/commands.go index 19dd598f8..11cf9648e 100644 --- a/pkg/cli/commands.go +++ b/pkg/cli/commands.go @@ -9,6 +9,7 @@ import ( "fmt" "io" "os" + "sync" "time" "github.com/conductorone/baton-sdk/pkg/connectorbuilder" @@ -35,6 +36,7 @@ import ( "github.com/conductorone/baton-sdk/pkg/field" "github.com/conductorone/baton-sdk/pkg/logging" "github.com/conductorone/baton-sdk/pkg/session" + "github.com/conductorone/baton-sdk/pkg/sourcecache" "github.com/conductorone/baton-sdk/pkg/tempdir" "github.com/conductorone/baton-sdk/pkg/types/sessions" "github.com/conductorone/baton-sdk/pkg/uhttp" @@ -53,29 +55,55 @@ type eventLogEnabledKey struct{} type ContrainstSetter func(*cobra.Command, field.Configuration) error -// In one shot & service mode, the child process uses this client to connect to the session store server... +// parentControlPlaneDialer opens (lazily, at most once) a single gRPC +// connection from the connector subprocess back to the parent SDK's +// control-plane listener. The same connection multiplexes +// BatonSessionService (connector session data) and BatonSourceCacheService +// (source-cache scope lookups) — the two services share a listener on the +// parent (internal/connector.runServer), so they share a client conn here. // -// which uses the C1Z for storage. Unfortunately the C1Z is instantiated well after we fork the child process, -// so there is quite a bit of pass through. -func getGRPCSessionStoreClient(ctx context.Context, serverCfg *v1.ServerConfig) func(ctx context.Context, opt ...sessions.SessionStoreConstructorOption) (sessions.SessionStore, error) { - return func(_ context.Context, opt ...sessions.SessionStoreConstructorOption) (sessions.SessionStore, error) { - l := ctxzap.Extract(ctx) - clientTLSConfig, err := utls2.ClientConfig(ctx, serverCfg.GetCredential()) - if err != nil { - return nil, err +// dial returns (nil, nil) when the parent did not start a control-plane +// listener (session store disabled); callers substitute no-op +// implementations in that case. +type parentControlPlaneDialer struct { + once sync.Once + conn *grpc.ClientConn + dialErr error + ctx context.Context + cfg *v1.ServerConfig +} + +func newParentControlPlaneDialer(ctx context.Context, serverCfg *v1.ServerConfig) *parentControlPlaneDialer { + return &parentControlPlaneDialer{ctx: ctx, cfg: serverCfg} +} + +// Close releases the underlying gRPC connection if dial ever succeeded. +// Safe when dial never ran or failed, and safe to call multiple times. +func (d *parentControlPlaneDialer) Close() { + if d.conn == nil { + return + } + _ = d.conn.Close() +} + +func (d *parentControlPlaneDialer) dial() (*grpc.ClientConn, error) { + d.once.Do(func() { + if d.cfg.GetSessionStoreListenPort() == 0 { + return } - if serverCfg.GetSessionStoreListenPort() == 0 { - return &session.NoOpSessionStore{}, nil + clientTLSConfig, err := utls2.ClientConfig(d.ctx, d.cfg.GetCredential()) + if err != nil { + d.dialErr = err + return } - // connected, grpc will handle retries for us. - dialCtx, canc := context.WithTimeout(ctx, 5*time.Second) + dialCtx, canc := context.WithTimeout(d.ctx, 5*time.Second) defer canc() var dialErr error var conn *grpc.ClientConn for { conn, err = grpc.DialContext( //nolint:staticcheck // grpc.DialContext is deprecated but we are using it still. - ctx, - fmt.Sprintf("127.0.0.1:%d", serverCfg.GetSessionStoreListenPort()), + d.ctx, + fmt.Sprintf("127.0.0.1:%d", d.cfg.GetSessionStoreListenPort()), grpc.WithTransportCredentials(credentials.NewTLS(clientTLSConfig)), grpc.WithBlock(), //nolint:staticcheck // grpc.WithBlock is deprecated but we are using it still. ) @@ -84,26 +112,59 @@ func getGRPCSessionStoreClient(ctx context.Context, serverCfg *v1.ServerConfig) select { case <-time.After(time.Millisecond * 500): case <-dialCtx.Done(): - return nil, dialErr + d.dialErr = dialErr + return } continue } break } + d.conn = conn + }) + return d.conn, d.dialErr +} +// In one shot & service mode, the child process uses this client to connect to the session store server... +// +// which uses the C1Z for storage. Unfortunately the C1Z is instantiated well after we fork the child process, +// so there is quite a bit of pass through. +func getGRPCSessionStoreClient(ctx context.Context, dialer *parentControlPlaneDialer) func(ctx context.Context, opt ...sessions.SessionStoreConstructorOption) (sessions.SessionStore, error) { + return func(_ context.Context, _ ...sessions.SessionStoreConstructorOption) (sessions.SessionStore, error) { + l := ctxzap.Extract(ctx) + conn, err := dialer.dial() + if err != nil { + return nil, err + } + if conn == nil { + return &session.NoOpSessionStore{}, nil + } client := baton_v1.NewBatonSessionServiceClient(conn) - ss, err := session.NewGRPCSessionStore(ctx, client, opt...) + ss, err := session.NewGRPCSessionStore(ctx, client) if err != nil { - err2 := conn.Close() - if err2 != nil { - l.Error("error closing connection", zap.Error(err2)) - } + l.Error("error creating session store client", zap.Error(err)) return nil, err } return ss, nil } } +// buildGRPCSourceCacheLookup returns the connector-side source-cache Lookup +// that talks to the parent's BatonSourceCacheService over the same +// loopback-TLS connection the session client uses. Returns NoopLookup when +// no parent control-plane listener is configured (matches the "no previous +// sync" behavior). +func buildGRPCSourceCacheLookup(dialer *parentControlPlaneDialer) (sourcecache.Lookup, error) { + conn, err := dialer.dial() + if err != nil { + return nil, err + } + if conn == nil { + return sourcecache.NoopLookup{}, nil + } + client := baton_v1.NewBatonSourceCacheServiceClient(conn) + return sourcecache.NewGRPCLookup(client), nil +} + func MakeMainCommand[T field.Configurable]( ctx context.Context, name string, @@ -377,13 +438,20 @@ func MakeMainCommand[T field.Configurable]( } } - if v.GetBool(field.ParallelSyncField.GetName()) { - opts = append(opts, connectorrunner.WithWorkerCount(-1)) - } - + // Worker count resolves through viper with no zero sentinel in the + // default chain: explicit flag > env (BATON_WORKERS) > config file > + // connector default (field.WithConnectorDefault on WorkerCountField) + // > shared default (0). A resolved 0 — explicit or defaulted — means + // sequential, which is the runner's zero-value behavior (normalized + // to one worker downstream), so no option is appended; -1 means + // auto-detect. The deprecated --parallel-sync only applies when + // workers resolved to 0. workers := v.GetInt(field.WorkerCountField.GetName()) - if workers != 0 { + switch { + case workers != 0: opts = append(opts, connectorrunner.WithWorkerCount(workers)) + case v.GetBool(field.ParallelSyncField.GetName()): + opts = append(opts, connectorrunner.WithWorkerCount(-1)) } c1zTmpDir := tempdir.Resolve(v.GetString("c1z-temp-dir")) @@ -396,13 +464,20 @@ func MakeMainCommand[T field.Configurable]( if v.GetString("external-resource-c1z") != "" { externalResourceC1ZPath := v.GetString("external-resource-c1z") - _, err := os.Open(externalResourceC1ZPath) - if err != nil { + if _, err := os.Stat(externalResourceC1ZPath); err != nil { return fmt.Errorf("the specified external resource c1z file does not exist: %s", externalResourceC1ZPath) } opts = append(opts, connectorrunner.WithExternalResourceC1Z(externalResourceC1ZPath)) } + if v.GetString(field.PreviousSyncC1ZField.GetName()) != "" { + previousSyncC1ZPath := v.GetString(field.PreviousSyncC1ZField.GetName()) + if _, err := os.Stat(previousSyncC1ZPath); err != nil { + return fmt.Errorf("the specified previous sync c1z file does not exist: %s", previousSyncC1ZPath) + } + opts = append(opts, connectorrunner.WithPreviousSyncC1Z(previousSyncC1ZPath)) + } + if v.GetString("external-resource-entitlement-id-filter") != "" { externalResourceEntitlementIdFilter := v.GetString("external-resource-entitlement-id-filter") opts = append(opts, connectorrunner.WithExternalResourceEntitlementFilter(externalResourceEntitlementIdFilter)) @@ -626,7 +701,17 @@ func MakeGRPCServerCommand[T field.Configurable]( runCtx = context.WithValue(runCtx, uhttp.ContextHTTPTimeoutKey, time.Duration(httpTimeout)*time.Second) sessionStoreMaximumSize := v.GetInt(field.ServerSessionStoreMaximumSizeField.GetName()) - sessionConstructor := getGRPCSessionStoreClient(runCtx, serverCfg) + // One dialer per subprocess; the session store client and the + // source-cache lookup client share the underlying gRPC connection + // (both services are registered on the same parent listener in + // internal/connector.runServer). + controlPlaneDialer := newParentControlPlaneDialer(runCtx, serverCfg) + defer controlPlaneDialer.Close() + sessionConstructor := getGRPCSessionStoreClient(runCtx, controlPlaneDialer) + sourceCacheLookup, err := buildGRPCSourceCacheLookup(controlPlaneDialer) + if err != nil { + return fmt.Errorf("failed to build source cache lookup: %w", err) + } c, err := getconnector(runCtx, t, RunTimeOpts{ SessionStore: NewLazyCachingSessionStore(sessionConstructor, func(otterOptions *otter.Options[string, []byte]) { if sessionStoreMaximumSize <= 0 { @@ -635,6 +720,7 @@ func MakeGRPCServerCommand[T field.Configurable]( otterOptions.MaximumWeight = uint64(sessionStoreMaximumSize) } }), + SourceCacheLookup: sourceCacheLookup, SelectedAuthMethod: v.GetString("auth-method"), SyncResourceTypeIDs: v.GetStringSlice("sync-resource-types"), }) diff --git a/pkg/cli/connector_default_test.go b/pkg/cli/connector_default_test.go new file mode 100644 index 000000000..930fda637 --- /dev/null +++ b/pkg/cli/connector_default_test.go @@ -0,0 +1,77 @@ +package cli + +import ( + "testing" + + "github.com/spf13/viper" + "github.com/stretchr/testify/require" + + "github.com/conductorone/baton-sdk/pkg/field" +) + +// TestWithConnectorDefault_WorkerCount pins the per-connector default +// mechanism end to end at the flag layer: the connector's default value +// lands on the cobra flag (so --help shows it and viper resolves it when +// nothing else is set), while explicit values — including the documented +// zero sentinel for sequential sync — still win. +func TestWithConnectorDefault_WorkerCount(t *testing.T) { + newCmdAndViper := func(t *testing.T) (*viper.Viper, *field.SchemaField) { + t.Helper() + f := field.WithConnectorDefault(field.WorkerCountField, 16) + return viper.New(), &f + } + + t.Run("default flows to flag, help, and viper", func(t *testing.T) { + v, f := newCmdAndViper(t) + cmd := setupCommand(t, field.Configuration{Fields: []field.SchemaField{*f}}, v) + + flag := cmd.PersistentFlags().Lookup("workers") + require.NotNil(t, flag) + require.Equal(t, "16", flag.DefValue, "connector default must be the flag default (what --help renders)") + require.False(t, flag.Changed) + + require.NoError(t, v.BindPFlags(cmd.PersistentFlags())) + require.Equal(t, 16, v.GetInt("workers"), "unset resolves to the connector default") + }) + + t.Run("explicit zero beats the connector default (sequential stays reachable)", func(t *testing.T) { + v, f := newCmdAndViper(t) + cmd := setupCommand(t, field.Configuration{Fields: []field.SchemaField{*f}}, v) + + require.NoError(t, cmd.PersistentFlags().Set("workers", "0")) + require.NoError(t, v.BindPFlags(cmd.PersistentFlags())) + require.Equal(t, 0, v.GetInt("workers")) + }) + + t.Run("explicit value beats the connector default", func(t *testing.T) { + v, f := newCmdAndViper(t) + cmd := setupCommand(t, field.Configuration{Fields: []field.SchemaField{*f}}, v) + + require.NoError(t, cmd.PersistentFlags().Set("workers", "4")) + require.NoError(t, v.BindPFlags(cmd.PersistentFlags())) + require.Equal(t, 4, v.GetInt("workers")) + }) + + t.Run("config file beats the connector default", func(t *testing.T) { + dir := t.TempDir() + writeYAML(t, dir, "workers: 8\n") + v := viper.New() + v.SetConfigName("config") + v.SetConfigType("yaml") + v.AddConfigPath(dir) + require.NoError(t, v.ReadInConfig()) + + f := field.WithConnectorDefault(field.WorkerCountField, 16) + cmd := setupCommand(t, field.Configuration{Fields: []field.SchemaField{f}}, v) + require.NoError(t, v.BindPFlags(cmd.PersistentFlags())) + require.Equal(t, 8, v.GetInt("workers")) + }) + + t.Run("original shared field is untouched", func(t *testing.T) { + require.Equal(t, 0, field.WorkerCountField.DefaultValue) + require.False(t, field.WorkerCountField.WasReExported) + f := field.WithConnectorDefault(field.WorkerCountField, 16) + require.Equal(t, 16, f.DefaultValue) + require.True(t, f.WasReExported, "the copy must be re-exported so DefineConfiguration replaces the SDK copy instead of erroring on the duplicate") + }) +} diff --git a/pkg/config/config.go b/pkg/config/config.go index 3586605fa..f250a20ae 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -40,6 +40,7 @@ func RunConnector[T field.Configurable]( } builderOpts = append(builderOpts, connectorbuilder.WithSessionStore(runTimeOpts.SessionStore)) + builderOpts = append(builderOpts, connectorbuilder.WithSourceCache(runTimeOpts.SourceCacheLookup)) c, err := connectorbuilder.NewConnector(ctx, connector, builderOpts...) if err != nil { @@ -160,6 +161,20 @@ func DefineConfigurationV2[T field.Configurable]( } confschema.Fields = fields + // The replay flags are hidden by default (source-cache replay is + // author-opt-in functionality); surface them in help only for + // connectors whose author baked the capability into their runner + // options. Hidden flags still parse, so this is a visibility decision, + // not a behavioral one. + if connectorrunner.DeclaresPreviousSyncCapability(ctx, options...) { + for i, f := range confschema.Fields { + switch f.FieldName { + case field.PreviousSyncC1ZField.FieldName, field.KeepPreviousSyncC1ZField.FieldName: + confschema.Fields[i].SyncerConfig.Hidden = false + } + } + } + // setup CLI with cobra mainCMD := &cobra.Command{ Use: connectorName, @@ -292,6 +307,12 @@ func verifyStructFields[T field.Configurable](schema field.Configuration) error return fmt.Errorf("T must be a struct type, got %v", configType.Kind()) //nolint:staticcheck // we want to capital letter here } for _, field := range schema.Fields { + if field.WasReExported { + // Re-exported shared SDK fields (field.WithConnectorDefault) + // are parsed by the SDK's own flag handling, not the + // connector's configuration struct. + continue + } fieldFound := false for i := 0; i < configType.NumField(); i++ { structField := configType.Field(i) diff --git a/pkg/connectorbuilder/connectorbuilder.go b/pkg/connectorbuilder/connectorbuilder.go index 223beb018..782a4ce34 100644 --- a/pkg/connectorbuilder/connectorbuilder.go +++ b/pkg/connectorbuilder/connectorbuilder.go @@ -6,6 +6,7 @@ import ( "fmt" "slices" "sort" + stdsync "sync" "time" "github.com/go-jose/go-jose/v4" @@ -23,6 +24,7 @@ import ( "github.com/conductorone/baton-sdk/pkg/retry" "github.com/conductorone/baton-sdk/pkg/sdk" "github.com/conductorone/baton-sdk/pkg/session" + "github.com/conductorone/baton-sdk/pkg/sourcecache" "github.com/conductorone/baton-sdk/pkg/types" "github.com/conductorone/baton-sdk/pkg/types/sessions" "github.com/conductorone/baton-sdk/pkg/types/tasks" @@ -72,11 +74,26 @@ type closeWithoutContext interface { } type builder struct { - ticketingEnabled bool - m *metrics.M - nowFunc func() time.Time - clientSecret *jose.JSONWebKey - sessionStore sessions.SessionStore + ticketingEnabled bool + m *metrics.M + nowFunc func() time.Time + clientSecret *jose.JSONWebKey + sessionStore sessions.SessionStore + // sourceCache is the single connector-facing lookup slot, guarded by + // sourceCacheMu: SetSourceCache (per-sync install/clear from the + // runner) can race list RPCs reading it via syncOpAttrs. Access + // through sourceCacheLookup / SetSourceCache only. + sourceCacheMu stdsync.RWMutex + sourceCache sourcecache.Lookup + // sourceCacheInstalls / sourceCacheContended implement the same + // contention blinding as sourcecache.GRPCServer: the slot carries no + // sync-id routing, so with two live installs a lookup cannot be + // attributed to either sync and serving EITHER cross-wires them (one + // sync validates against the other's previous artifact and replays + // from its own — stale rows sealed as current). A miss is always + // safe, so on overlap every lookup misses until the slot drains. + sourceCacheInstalls int + sourceCacheContended bool metadataProvider MetadataProvider validateProvider ValidateProvider ticketManager TicketManagerLimited @@ -196,6 +213,9 @@ func NewConnector(ctx context.Context, in interface{}, opts ...Opt) (types.Conne if cb, ok := in.(ConnectorBuilder); ok { for _, rb := range cb.ResourceSyncers(ctx) { rType := rb.ResourceType(ctx) + if err := validateTypeScopedRegistration(rType, rb); err != nil { + return nil, err + } if err := addResourceType(ctx, rType.GetId(), rb); err != nil { return nil, err } @@ -206,6 +226,9 @@ func NewConnector(ctx context.Context, in interface{}, opts ...Opt) (types.Conne if cb2, ok := in.(ConnectorBuilderV2); ok { for _, rb := range cb2.ResourceSyncers(ctx) { rType := rb.ResourceType(ctx) + if err := validateTypeScopedRegistration(rType, rb); err != nil { + return nil, err + } if err := addResourceType(ctx, rType.GetId(), rb); err != nil { return nil, err } @@ -242,6 +265,79 @@ func WithSessionStore(ss sessions.SessionStore) Opt { } } +// WithSourceCache supplies a direct connector-facing source-cache Lookup +// exposed on SyncOpAttrs.SourceCache (see pkg/sourcecache). +// +// A nil lookup deliberately leaves the builder unset: SyncOpAttrs still +// supplies NoopLookup by default, but an unset builder is how the +// single-shot ask/answer continuation recognizes that it may install its +// per-request ContinuationLookup. Lambda construction passes nil here. +func WithSourceCache(lookup sourcecache.Lookup) Opt { + return func(b *builder) error { + if lookup == nil { + return nil + } + b.sourceCacheMu.Lock() + b.sourceCache = lookup + b.sourceCacheMu.Unlock() + return nil + } +} + +// SetSourceCache implements sourcecache.SourceCacheSetter so an in-process runner +// (the syncer, via its connector client) can install/clear the active +// lookup per sync. Subprocess mode instead routes lookups over +// BatonSourceCacheService and never calls this. +// +// The slot is guarded: the runner supports concurrent tasks, so a set or +// clear can race in-flight list RPCs reading the slot via syncOpAttrs. +// The slot is still SINGLE — there is no sync-id routing — so two live +// syncs installing lookups would cross-wire each other (one sync's +// connector would resolve validators against the other's previous store, +// then replay rows from its OWN store under a validator the other +// artifact vouched for — stale rows sealed as current). A miss is always +// safe (the connector fetches cold), so overlapping installs BLIND the +// slot — every lookup misses until all concurrent owners have cleared — +// mirroring sourcecache.GRPCServer's contention defense, and warn loudly +// so the degraded overlap is visible. +func (b *builder) SetSourceCache(ctx context.Context, lookup sourcecache.Lookup) { + b.sourceCacheMu.Lock() + if lookup == nil { + // Clear: the finished sync's lookup must not serve late RPCs. The + // slot stays occupied (NoopLookup, never back to nil) so the + // continuation's "never set" signal remains accurate. + if b.sourceCacheInstalls > 0 { + b.sourceCacheInstalls-- + } + if b.sourceCacheInstalls == 0 { + // Slot fully drained: the next sync starts clean. + b.sourceCacheContended = false + } + b.sourceCache = sourcecache.NoopLookup{} + b.sourceCacheMu.Unlock() + return + } + b.sourceCacheInstalls++ + if b.sourceCacheInstalls > 1 || b.sourceCacheContended { + b.sourceCacheContended = true + b.sourceCache = sourcecache.NoopLookup{} + b.sourceCacheMu.Unlock() + ctxzap.Extract(ctx).Warn("source cache: concurrent syncs share this connector's single lookup slot; " + + "serving misses to every sync (cold fetches) until the overlap drains, to avoid cross-wiring validators between previous-sync artifacts") + return + } + b.sourceCache = lookup + b.sourceCacheMu.Unlock() +} + +// sourceCacheLookup returns the active slot value (nil when never set — +// the continuation's signal that it may install a per-request lookup). +func (b *builder) sourceCacheLookup() sourcecache.Lookup { + b.sourceCacheMu.RLock() + defer b.sourceCacheMu.RUnlock() + return b.sourceCache +} + func (b *builder) options(opts ...Opt) error { for _, opt := range opts { if err := opt(b); err != nil { diff --git a/pkg/connectorbuilder/continuation_test.go b/pkg/connectorbuilder/continuation_test.go new file mode 100644 index 000000000..b22db4100 --- /dev/null +++ b/pkg/connectorbuilder/continuation_test.go @@ -0,0 +1,256 @@ +package connectorbuilder + +// Lookup continuation, builder side: a connector without a runner-supplied +// lookup gets a per-request ContinuationLookup when the request carries +// SourceCacheLookupOffer/Answers. Phase 1 (deferred lookups) must become an +// ask response with no failure recorded; phase 2 must serve answers to the +// unchanged connector code; a swallowed ErrLookupDeferred must fail loudly. + +import ( + "context" + "fmt" + "testing" + + "github.com/stretchr/testify/require" + + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + "github.com/conductorone/baton-sdk/pkg/annotations" + "github.com/conductorone/baton-sdk/pkg/sourcecache" + "github.com/conductorone/baton-sdk/pkg/types/resource" +) + +// continuationTestSyncer's Grants consults the source-cache lookup for one +// scope per page and serves either a replay annotation (hit) or a cold row +// + scope annotation (miss). Mode switches test the misuse paths. +type continuationTestSyncer struct { + testResourceSyncerV2Simple + + scope string + swallow bool // ignore ErrLookupDeferred and return rows anyway + batch bool // resolve via LookupMany over three scopes + + lookupCalls int +} + +func (s *continuationTestSyncer) Grants(ctx context.Context, r *v2.Resource, opts resource.SyncOpAttrs) ([]*v2.Grant, *resource.SyncOpResults, error) { + s.lookupCalls++ + if s.batch { + queries := []sourcecache.Query{ + {RowKind: sourcecache.RowKindGrants, ScopeKey: s.scope + "/1"}, + {RowKind: sourcecache.RowKindGrants, ScopeKey: s.scope + "/2"}, + {RowKind: sourcecache.RowKindGrants, ScopeKey: s.scope + "/3"}, + } + answers, err := sourcecache.LookupMany(ctx, opts.SourceCache, queries) + if err != nil { + return nil, nil, fmt.Errorf("batch revalidation: %w", err) + } + ret := &resource.SyncOpResults{Annotations: annotations.Annotations{}} + for _, a := range answers { + if !a.Found || a.CacheValidator != "current" { + return nil, nil, fmt.Errorf("test wants all hits, got %+v", a) + } + } + ret.Annotations.Update(v2.SourceCacheReplay_builder{ScopeKey: s.scope + "/1", CacheValidator: "current"}.Build()) + return nil, ret, nil + } + + entry, found, err := opts.SourceCache.Lookup(ctx, sourcecache.RowKindGrants, s.scope) + if err != nil && !s.swallow { + // Idiomatic wrapping — the builder must match via errors.Is. + return nil, nil, fmt.Errorf("listing grants for %s: %w", r.GetId().GetResource(), err) + } + ret := &resource.SyncOpResults{Annotations: annotations.Annotations{}} + if err == nil && found && entry.CacheValidator == "current" { + ret.Annotations.Update(v2.SourceCacheReplay_builder{ScopeKey: s.scope, CacheValidator: entry.CacheValidator}.Build()) + return nil, ret, nil + } + // Miss (or swallowed deferral): cold row. + ret.Annotations.Update(v2.SourceCacheRecord_builder{ScopeKey: s.scope, CacheValidator: "current"}.Build()) + return []*v2.Grant{mkTestGrant(r)}, ret, nil +} + +func mkTestGrant(r *v2.Resource) *v2.Grant { + return v2.Grant_builder{ + Id: "grant-1", + Entitlement: v2.Entitlement_builder{ + Id: fmt.Sprintf("%s:%s:member", r.GetId().GetResourceType(), r.GetId().GetResource()), + Resource: r, + }.Build(), + Principal: r, + }.Build() +} + +func newContinuationTestBuilder(t *testing.T, ts *continuationTestSyncer) *builder { + t.Helper() + cb := &testConnectorBuilderV2Full{resourceSyncers: []ResourceSyncerV2{ts}, hasActionManager: true} + c, err := NewConnector(context.Background(), cb) + require.NoError(t, err) + b, ok := c.(*builder) + require.True(t, ok) + return b +} + +func continuationGrantsRequest(annos annotations.Annotations) *v2.GrantsServiceListGrantsRequest { + return v2.GrantsServiceListGrantsRequest_builder{ + Resource: v2.Resource_builder{ + Id: v2.ResourceId_builder{ResourceType: "group", Resource: "g1"}.Build(), + }.Build(), + Annotations: annos, + }.Build() +} + +// clearedSlotTestLookup is a trivially non-noop direct lookup for the +// cleared-slot test below. +type clearedSlotTestLookup struct{} + +func (clearedSlotTestLookup) Lookup(context.Context, sourcecache.RowKind, string) (sourcecache.Entry, bool, error) { + return sourcecache.Entry{CacheValidator: "current"}, true, nil +} + +// TestContinuation_SurvivesClearedLookupSlot pins the slot-semantics +// contract between SetSourceCache and continuationOpAttrs: clearing the +// slot (SetSourceCache(nil) stores NoopLookup so late RPCs read a +// deterministic miss) must NOT count as "a direct lookup is installed" — +// that would silently disable the ask/answer continuation for every +// later request, degrading to permanent cold syncs with no error. +func TestContinuation_SurvivesClearedLookupSlot(t *testing.T) { + ctx := context.Background() + ts := &continuationTestSyncer{testResourceSyncerV2Simple: testResourceSyncerV2Simple{resourceType: "group"}, scope: "groups/g1/members"} + b := newContinuationTestBuilder(t, ts) + + // Install then clear, as a completed direct-topology sync would. + b.SetSourceCache(ctx, clearedSlotTestLookup{}) + b.SetSourceCache(ctx, nil) + + // A continuation request must still defer into an ask. + resp, err := b.ListGrants(ctx, continuationGrantsRequest(annotations.New(&v2.SourceCacheLookupOffer{}))) + require.NoError(t, err) + require.Empty(t, resp.GetList()) + ask := &v2.SourceCacheLookupAsk{} + respAnnos := annotations.Annotations(resp.GetAnnotations()) + hasAsk, err := respAnnos.Pick(ask) + require.NoError(t, err) + require.True(t, hasAsk, "a cleared lookup slot must not disable the continuation") +} + +func TestContinuation_DeferThenAnswer(t *testing.T) { + ctx := context.Background() + ts := &continuationTestSyncer{testResourceSyncerV2Simple: testResourceSyncerV2Simple{resourceType: "group"}, scope: "groups/g1/members"} + b := newContinuationTestBuilder(t, ts) + + // Phase 1: offer only → the handler's deferred lookup becomes an ask. + offer := annotations.New(&v2.SourceCacheLookupOffer{}) + resp, err := b.ListGrants(ctx, continuationGrantsRequest(offer)) + require.NoError(t, err, "a deferral is a protocol turn, not a failure") + require.Empty(t, resp.GetList(), "ask response must carry no rows") + require.Empty(t, resp.GetNextPageToken()) + + ask := &v2.SourceCacheLookupAsk{} + respAnnos := annotations.Annotations(resp.GetAnnotations()) + hasAsk, err := respAnnos.Pick(ask) + require.NoError(t, err) + require.True(t, hasAsk) + require.Len(t, ask.GetQueries(), 1) + require.Equal(t, "groups/g1/members", ask.GetQueries()[0].GetScopeKey()) + require.Equal(t, string(sourcecache.RowKindGrants), ask.GetQueries()[0].GetRowKind()) + + // Phase 2: same request + answers → the SAME connector code replays. + answers := annotations.New(&v2.SourceCacheLookupOffer{}) + answers.Update(sourcecache.AnswersProto([]sourcecache.Answer{ + {Query: sourcecache.Query{RowKind: sourcecache.RowKindGrants, ScopeKey: "groups/g1/members"}, Found: true, CacheValidator: "current"}, + })) + resp2, err := b.ListGrants(ctx, continuationGrantsRequest(answers)) + require.NoError(t, err) + replay := &v2.SourceCacheReplay{} + resp2Annos := annotations.Annotations(resp2.GetAnnotations()) + hasReplay, err := resp2Annos.Pick(replay) + require.NoError(t, err) + require.True(t, hasReplay, "phase 2 must reach the connector's replay branch") + require.Empty(t, resp2.GetList()) + + // A not-found answer is a MISS, not a deferral: cold fetch. + miss := annotations.New(&v2.SourceCacheLookupOffer{}) + miss.Update(sourcecache.AnswersProto([]sourcecache.Answer{ + {Query: sourcecache.Query{RowKind: sourcecache.RowKindGrants, ScopeKey: "groups/g1/members"}, Found: false}, + })) + resp3, err := b.ListGrants(ctx, continuationGrantsRequest(miss)) + require.NoError(t, err) + require.Len(t, resp3.GetList(), 1, "not-found answer must fall through to cold fetch") +} + +func TestContinuation_LookupManyBatchesOneAsk(t *testing.T) { + ctx := context.Background() + ts := &continuationTestSyncer{testResourceSyncerV2Simple: testResourceSyncerV2Simple{resourceType: "group"}, scope: "chunk", batch: true} + b := newContinuationTestBuilder(t, ts) + + resp, err := b.ListGrants(ctx, continuationGrantsRequest(annotations.New(&v2.SourceCacheLookupOffer{}))) + require.NoError(t, err) + ask := &v2.SourceCacheLookupAsk{} + respAnnos := annotations.Annotations(resp.GetAnnotations()) + hasAsk, err := respAnnos.Pick(ask) + require.NoError(t, err) + require.True(t, hasAsk) + require.Len(t, ask.GetQueries(), 3, "LookupMany must collect the whole batch into ONE ask") + + var answers []sourcecache.Answer + for _, q := range ask.GetQueries() { + answers = append(answers, sourcecache.Answer{ + Query: sourcecache.Query{RowKind: sourcecache.RowKind(q.GetRowKind()), ScopeKey: q.GetScopeKey()}, + Found: true, CacheValidator: "current", + }) + } + req := annotations.New(&v2.SourceCacheLookupOffer{}) + req.Update(sourcecache.AnswersProto(answers)) + resp2, err := b.ListGrants(ctx, continuationGrantsRequest(req)) + require.NoError(t, err) + resp2Annos := annotations.Annotations(resp2.GetAnnotations()) + require.True(t, resp2Annos.Contains(&v2.SourceCacheReplay{})) +} + +func TestContinuation_SwallowedDeferralFailsLoudly(t *testing.T) { + ctx := context.Background() + ts := &continuationTestSyncer{testResourceSyncerV2Simple: testResourceSyncerV2Simple{resourceType: "group"}, scope: "s", swallow: true} + b := newContinuationTestBuilder(t, ts) + + _, err := b.ListGrants(ctx, continuationGrantsRequest(annotations.New(&v2.SourceCacheLookupOffer{}))) + require.Error(t, err) + require.Contains(t, err.Error(), "swallowed", "a swallowed ErrLookupDeferred must be caught in-process, not at the production bounce cap") +} + +func TestContinuation_NoOfferMeansNoopLookup(t *testing.T) { + ctx := context.Background() + ts := &continuationTestSyncer{testResourceSyncerV2Simple: testResourceSyncerV2Simple{resourceType: "group"}, scope: "s"} + b := newContinuationTestBuilder(t, ts) + + // No offer, no answers: lookup misses (NoopLookup), connector fetches + // cold, no ask, no error — exactly today's behavior. + resp, err := b.ListGrants(ctx, continuationGrantsRequest(nil)) + require.NoError(t, err) + require.Len(t, resp.GetList(), 1) + respAnnos := annotations.Annotations(resp.GetAnnotations()) + require.False(t, respAnnos.Contains(&v2.SourceCacheLookupAsk{})) +} + +func TestContinuation_DirectLookupWins(t *testing.T) { + ctx := context.Background() + ts := &continuationTestSyncer{testResourceSyncerV2Simple: testResourceSyncerV2Simple{resourceType: "group"}, scope: "s"} + cb := &testConnectorBuilderV2Full{resourceSyncers: []ResourceSyncerV2{ts}, hasActionManager: true} + c, err := NewConnector(context.Background(), cb, WithSourceCache(staticTestLookup{etag: "current"})) + require.NoError(t, err) + b, ok := c.(*builder) + require.True(t, ok) + + // Offer present AND a direct lookup installed: the direct lookup wins, + // the handler resolves in one phase, and no ask is emitted. + resp, err := b.ListGrants(ctx, continuationGrantsRequest(annotations.New(&v2.SourceCacheLookupOffer{}))) + require.NoError(t, err) + respAnnos := annotations.Annotations(resp.GetAnnotations()) + require.False(t, respAnnos.Contains(&v2.SourceCacheLookupAsk{})) + require.True(t, respAnnos.Contains(&v2.SourceCacheReplay{})) +} + +type staticTestLookup struct{ etag string } + +func (s staticTestLookup) Lookup(context.Context, sourcecache.RowKind, string) (sourcecache.Entry, bool, error) { + return sourcecache.Entry{CacheValidator: s.etag}, true, nil +} diff --git a/pkg/connectorbuilder/resource_syncer.go b/pkg/connectorbuilder/resource_syncer.go index 0360c3059..5c6adf654 100644 --- a/pkg/connectorbuilder/resource_syncer.go +++ b/pkg/connectorbuilder/resource_syncer.go @@ -2,12 +2,14 @@ package connectorbuilder import ( "context" + "errors" "fmt" v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" "github.com/conductorone/baton-sdk/pkg/annotations" "github.com/conductorone/baton-sdk/pkg/pagination" "github.com/conductorone/baton-sdk/pkg/session" + "github.com/conductorone/baton-sdk/pkg/sourcecache" "github.com/conductorone/baton-sdk/pkg/types/resource" "github.com/conductorone/baton-sdk/pkg/types/tasks" "github.com/conductorone/baton-sdk/pkg/uotel" @@ -65,6 +67,158 @@ type StaticEntitlementSyncerV2 interface { StaticEntitlements(ctx context.Context, opts resource.SyncOpAttrs) ([]*v2.Entitlement, *resource.SyncOpResults, error) } +// TypeScopedGrantsSyncer lets a connector enumerate grants for a WHOLE +// resource type instead of being called once per resource. Implement it on +// a resource syncer whose ResourceType carries the v2.TypeScopedGrants +// annotation; the syncer then issues ListGrants requests carrying the +// v2.TypeScopedGrants REQUEST annotation as the routing marker (the +// request's resource is a self-referential {type, type} stub — wire +// validation requires a non-empty resource id), which the builder routes +// here. +// +// The first call of a sync arrives with an empty page token (the planning +// call); the connector may answer with rows directly and/or spawn +// additional independent cursors by attaching a v2.EnqueuePageTokens annotation +// whose page tokens are delivered back through opts.PageToken, one action +// each. Source-cache scope/replay/tombstone annotations work exactly as on +// per-resource grants pages. +// +// The per-resource Grants method is still required by ResourceSyncerV2 +// but is NEVER called for an annotated type during a full sync; the +// convention is to return empty results (registration fails loudly if +// the annotation is present without this interface — see +// validateTypeScopedRegistration). +type TypeScopedGrantsSyncer interface { + GrantsForResourceType(ctx context.Context, resourceTypeID string, opts resource.SyncOpAttrs) ([]*v2.Grant, *resource.SyncOpResults, error) +} + +// TypeScopedEntitlementsSyncer is the entitlements-phase analogue of +// TypeScopedGrantsSyncer: the connector enumerates entitlements for a WHOLE +// resource type instead of being called once per resource. Implement it on +// a resource syncer whose ResourceType carries the v2.TypeScopedEntitlements +// annotation; the syncer then issues ListEntitlements requests carrying +// that annotation as the routing marker (same {type, type} stub resource +// as the grants variant), which the builder routes here. +// +// Planning / EnqueuePageTokens / source-cache behavior matches TypeScopedGrantsSyncer. +type TypeScopedEntitlementsSyncer interface { + EntitlementsForResourceType(ctx context.Context, resourceTypeID string, opts resource.SyncOpAttrs) ([]*v2.Entitlement, *resource.SyncOpResults, error) +} + +// syncOpAttrs assembles the per-call SyncOpAttrs handed to V2 resource +// syncers. SourceCache is never nil: when the runner supplied no lookup +// (source cache disabled/degraded) connectors see NoopLookup and every +// lookup misses. Session is likewise never a nil-backed wrapper: without a +// configured store, connectors see the erroring no-op store instead of a +// panic on first use. +func (b *builder) syncOpAttrs(activeSyncID string, token pagination.Token) resource.SyncOpAttrs { + return b.syncOpAttrsWithLookup(activeSyncID, token, b.sourceCacheLookup()) +} + +// syncOpAttrsWithLookup is syncOpAttrs over a caller-held slot snapshot: +// SetSourceCache can install/clear the slot concurrently with an +// in-flight RPC, so a caller that also BRANCHES on the slot value +// (continuationOpAttrs) must read it exactly once and thread the same +// snapshot into both decisions — two reads could hand a connector +// NoopLookup while treating the topology as direct. +func (b *builder) syncOpAttrsWithLookup(activeSyncID string, token pagination.Token, sc sourcecache.Lookup) resource.SyncOpAttrs { + if sc == nil { + sc = sourcecache.NoopLookup{} + } + ss := b.sessionStore + if ss == nil { + ss = &session.NoOpSessionStore{} + } + return resource.SyncOpAttrs{ + SyncID: activeSyncID, + PageToken: token, + Session: WithSyncId(ss, activeSyncID), + SourceCache: sc, + } +} + +// continuationOpAttrs builds SyncOpAttrs for a list RPC, selecting the +// source-cache lookup by topology: +// +// - a runner-supplied lookup (in-process interface, subprocess loopback +// gRPC) always wins: those topologies answer lookups directly and +// never bounce; +// - otherwise, when the request carries SourceCacheLookupOffer (and/or +// SourceCacheLookupAnswers from a previous bounce), a per-request +// ContinuationLookup is installed — it serves answered scopes and +// defers the rest via ErrLookupDeferred; +// - otherwise NoopLookup (today's behavior). +// +// The returned ContinuationLookup is nil when the continuation is not in +// play for this request. +func (b *builder) continuationOpAttrs(activeSyncID string, token pagination.Token, reqAnnos annotations.Annotations) (resource.SyncOpAttrs, *sourcecache.ContinuationLookup, error) { + // One slot read for both the attrs and the topology branch below — + // see syncOpAttrsWithLookup for why two reads would be a race. + slot := b.sourceCacheLookup() + attrs := b.syncOpAttrsWithLookup(activeSyncID, token, slot) + if l := slot; l != nil { + // A NoopLookup slot is a CLEARED slot, not a direct lookup: + // SetSourceCache(nil) stores NoopLookup so late in-flight RPCs + // read a deterministic miss (see SetSourceCache), and treating + // it as "direct lookup wins" here would silently disable the + // ask/answer continuation for every later request — permanent + // cold syncs with no error. A degraded sync can't sneak into + // continuation through this branch either: without a warm + // lookup the syncer never attaches SourceCacheLookupOffer, so + // the offer check below already returns the noop path. + if _, isNoop := l.(sourcecache.NoopLookup); !isNoop { + return attrs, nil, nil + } + } + answersMsg := &v2.SourceCacheLookupAnswers{} + hasAnswers, err := reqAnnos.Pick(answersMsg) + if err != nil { + return attrs, nil, fmt.Errorf("error parsing source-cache lookup answers annotation: %w", err) + } + if !hasAnswers && !reqAnnos.Contains(&v2.SourceCacheLookupOffer{}) { + return attrs, nil, nil + } + var answers []sourcecache.Answer + if hasAnswers { + answers, err = sourcecache.AnswersFromProto(answersMsg) + if err != nil { + return attrs, nil, err + } + } + cl := sourcecache.NewContinuationLookup(answers) + attrs.SourceCache = cl + return attrs, cl, nil +} + +// continuationOutcome inspects a list handler's result when a +// ContinuationLookup was installed. +// +// Returns (askAnnos, deferred, misuseErr): +// - deferred=true: the handler deferred on recorded scopes. Respond with +// askAnnos (the SourceCacheLookupAsk), NO rows, NO next page token, +// and no failure metrics — this is a protocol turn, not a failure. +// - misuseErr != nil: the handler swallowed a deferred lookup (scopes +// were recorded but no ErrLookupDeferred propagated). Loud failure: +// silently continuing past a deferral re-asks forever and dies at the +// bounce cap in production, so it must fail here, visibly. +// - all-zero: the handler resolved normally; use its result as-is. +func continuationOutcome(cl *sourcecache.ContinuationLookup, handlerErr error) (annotations.Annotations, bool, error) { + if cl == nil { + return nil, false, nil + } + asked := cl.Asked() + if errors.Is(handlerErr, sourcecache.ErrLookupDeferred) && len(asked) > 0 { + annos := annotations.Annotations{} + annos.Update(sourcecache.AskProto(asked)) + return annos, true, nil + } + if handlerErr == nil && len(asked) > 0 { + return nil, false, status.Errorf(codes.Internal, + "connector swallowed a deferred source-cache lookup (%d scope(s) asked, no error propagated): ErrLookupDeferred must be returned — wrap with %%w, never swallow", len(asked)) + } + return nil, false, nil +} + // ResourceTargetedSyncer extends ResourceSyncer to add capabilities for directly syncing an individual resource // // Implementing this interface indicates the connector supports calling "get" on a resource @@ -131,12 +285,33 @@ func (b *builder) ListResources(ctx context.Context, request *v2.ResourcesServic Size: int(request.GetPageSize()), Token: request.GetPageToken(), } - opts := resource.SyncOpAttrs{SyncID: request.GetActiveSyncId(), PageToken: token, Session: WithSyncId(b.sessionStore, request.GetActiveSyncId())} + opts, contLookup, err := b.continuationOpAttrs(request.GetActiveSyncId(), token, annotations.Annotations(request.GetAnnotations())) + if err != nil { + b.m.RecordTaskFailure(ctx, tt, b.nowFunc().Sub(start), err) + return nil, err + } out, retOptions, err := rb.List(ctx, request.GetParentResourceId(), opts) if retOptions == nil { retOptions = &resource.SyncOpResults{} } + askAnnos, deferred, misuseErr := continuationOutcome(contLookup, err) + if misuseErr != nil { + b.m.RecordTaskFailure(ctx, tt, b.nowFunc().Sub(start), misuseErr) + return nil, misuseErr + } + if deferred { + b.m.RecordTaskSuccess(ctx, tt, b.nowFunc().Sub(start)) + // A deferral is a protocol turn, not a failure: clear the + // handler's ErrLookupDeferred so the span defer above doesn't + // record every normal ask/answer bounce as a trace error. + err = nil + // Session usage rides the ask too: the phase-1 handler may have + // touched the session before deferring, and the protocol turn is + // the only response that work will ever produce. + return v2.ResourcesServiceListResourcesResponse_builder{Annotations: appendSessionUsage(askAnnos, sessionUsage)}.Build(), nil + } + resp := v2.ResourcesServiceListResourcesResponse_builder{ List: out, NextPageToken: retOptions.NextPageToken, @@ -220,7 +395,7 @@ func (b *builder) ListStaticEntitlements(ctx context.Context, request *v2.Entitl Size: int(request.GetPageSize()), Token: request.GetPageToken(), } - opts := resource.SyncOpAttrs{SyncID: request.GetActiveSyncId(), PageToken: token, Session: WithSyncId(b.sessionStore, request.GetActiveSyncId())} + opts := b.syncOpAttrs(request.GetActiveSyncId(), token) out, retOptions, err := rbse.StaticEntitlements(ctx, opts) if retOptions == nil { retOptions = &resource.SyncOpResults{} @@ -258,9 +433,17 @@ func (b *builder) ListEntitlements(ctx context.Context, request *v2.Entitlements start := b.nowFunc() tt := tasks.ListEntitlementsType - rb, ok := b.resourceSyncers[request.GetResource().GetId().GetResourceType()] + + if request.GetResource() == nil { + err = status.Error(codes.InvalidArgument, "error: list entitlements requires a resource") + b.m.RecordTaskFailure(ctx, tt, b.nowFunc().Sub(start), err) + return nil, err + } + + rid := request.GetResource().GetId() + rb, ok := b.resourceSyncers[rid.GetResourceType()] if !ok { - err = status.Errorf(codes.NotFound, "error: list entitlements with unknown resource type %s", request.GetResource().GetId().GetResourceType()) + err = status.Errorf(codes.NotFound, "error: list entitlements with unknown resource type %s", rid.GetResourceType()) b.m.RecordTaskFailure(ctx, tt, b.nowFunc().Sub(start), err) return nil, err } @@ -269,12 +452,51 @@ func (b *builder) ListEntitlements(ctx context.Context, request *v2.Entitlements Size: int(request.GetPageSize()), Token: request.GetPageToken(), } - opts := resource.SyncOpAttrs{SyncID: request.GetActiveSyncId(), PageToken: token, Session: WithSyncId(b.sessionStore, request.GetActiveSyncId())} - out, retOptions, err := rb.Entitlements(ctx, request.GetResource(), opts) + reqAnnos := annotations.Annotations(request.GetAnnotations()) + opts, contLookup, err := b.continuationOpAttrs(request.GetActiveSyncId(), token, reqAnnos) + if err != nil { + b.m.RecordTaskFailure(ctx, tt, b.nowFunc().Sub(start), err) + return nil, err + } + + typeScoped := reqAnnos.Contains(&v2.TypeScopedEntitlements{}) + + var out []*v2.Entitlement + var retOptions *resource.SyncOpResults + if typeScoped { + // Type-scoped entitlements call (see v2.TypeScopedEntitlements): the + // request annotation is the routing marker (the resource is a + // self-referential {type, type} stub to satisfy wire validation). + // Only legal against a syncer that opted in. + tses, tsOk := rb.(TypeScopedEntitlementsSyncer) + if !tsOk { + err = status.Errorf(codes.InvalidArgument, + "error: type-scoped list entitlements for resource type %s, but its syncer does not implement TypeScopedEntitlementsSyncer", rid.GetResourceType()) + b.m.RecordTaskFailure(ctx, tt, b.nowFunc().Sub(start), err) + return nil, err + } + out, retOptions, err = tses.EntitlementsForResourceType(ctx, rid.GetResourceType(), opts) + } else { + out, retOptions, err = rb.Entitlements(ctx, request.GetResource(), opts) + } if retOptions == nil { retOptions = &resource.SyncOpResults{} } + askAnnos, deferred, misuseErr := continuationOutcome(contLookup, err) + if misuseErr != nil { + b.m.RecordTaskFailure(ctx, tt, b.nowFunc().Sub(start), misuseErr) + return nil, misuseErr + } + if deferred { + b.m.RecordTaskSuccess(ctx, tt, b.nowFunc().Sub(start)) + // Protocol turn, not a failure — clear ErrLookupDeferred for the + // span defer; see the ListResources deferred path. + err = nil + // Session usage rides the ask; see the ListResources deferred path. + return v2.EntitlementsServiceListEntitlementsResponse_builder{Annotations: appendSessionUsage(askAnnos, sessionUsage)}.Build(), nil + } + resp := v2.EntitlementsServiceListEntitlementsResponse_builder{ List: out, NextPageToken: retOptions.NextPageToken, @@ -327,12 +549,51 @@ func (b *builder) ListGrants(ctx context.Context, request *v2.GrantsServiceListG Size: int(request.GetPageSize()), Token: request.GetPageToken(), } - opts := resource.SyncOpAttrs{SyncID: request.GetActiveSyncId(), PageToken: token, Session: WithSyncId(b.sessionStore, request.GetActiveSyncId())} - out, retOptions, err := rb.Grants(ctx, request.GetResource(), opts) + reqAnnos := annotations.Annotations(request.GetAnnotations()) + opts, contLookup, err := b.continuationOpAttrs(request.GetActiveSyncId(), token, reqAnnos) + if err != nil { + b.m.RecordTaskFailure(ctx, tt, b.nowFunc().Sub(start), err) + return nil, err + } + + typeScoped := reqAnnos.Contains(&v2.TypeScopedGrants{}) + + var out []*v2.Grant + var retOptions *resource.SyncOpResults + if typeScoped { + // Type-scoped grants call (see v2.TypeScopedGrants): the request + // annotation is the routing marker (the resource is a + // self-referential {type, type} stub to satisfy wire validation). + // Only legal against a syncer that opted in. + tsgs, tsOk := rb.(TypeScopedGrantsSyncer) + if !tsOk { + err = status.Errorf(codes.InvalidArgument, + "error: type-scoped list grants for resource type %s, but its syncer does not implement TypeScopedGrantsSyncer", rid.GetResourceType()) + b.m.RecordTaskFailure(ctx, tt, b.nowFunc().Sub(start), err) + return nil, err + } + out, retOptions, err = tsgs.GrantsForResourceType(ctx, rid.GetResourceType(), opts) + } else { + out, retOptions, err = rb.Grants(ctx, request.GetResource(), opts) + } if retOptions == nil { retOptions = &resource.SyncOpResults{} } + askAnnos, deferred, misuseErr := continuationOutcome(contLookup, err) + if misuseErr != nil { + b.m.RecordTaskFailure(ctx, tt, b.nowFunc().Sub(start), misuseErr) + return nil, misuseErr + } + if deferred { + b.m.RecordTaskSuccess(ctx, tt, b.nowFunc().Sub(start)) + // Protocol turn, not a failure — clear ErrLookupDeferred for the + // span defer; see the ListResources deferred path. + err = nil + // Session usage rides the ask; see the ListResources deferred path. + return v2.GrantsServiceListGrantsResponse_builder{Annotations: appendSessionUsage(askAnnos, sessionUsage)}.Build(), nil + } + resp := v2.GrantsServiceListGrantsResponse_builder{ List: out, Annotations: appendSessionUsage(retOptions.Annotations, sessionUsage), @@ -355,8 +616,39 @@ func (b *builder) ListGrants(ctx context.Context, request *v2.GrantsServiceListG return resp, nil } +// newResourceSyncerV1toV2 adapts a V1 ResourceSyncer to the V2 surface. +// The adapter must FORWARD the optional type-scoped interfaces: routing +// and validation type-assert TypeScopedGrantsSyncer / +// TypeScopedEntitlementsSyncer, validation against the ORIGINAL syncer +// but routing against the STORED (wrapped) one — a plain wrapper would +// pass construction and then fail every type-scoped call at runtime +// with InvalidArgument. The composite shapes below satisfy exactly the +// interfaces the wrapped syncer does, so the tsOk routing checks stay +// meaningful for syncers that never opted in. func newResourceSyncerV1toV2(rb ResourceSyncer) ResourceSyncerV2 { - return &resourceSyncerV1toV2{rb: rb} + base := &resourceSyncerV1toV2{rb: rb} + tsg, hasTSG := rb.(TypeScopedGrantsSyncer) + tse, hasTSE := rb.(TypeScopedEntitlementsSyncer) + switch { + case hasTSG && hasTSE: + return &struct { + *resourceSyncerV1toV2 + TypeScopedGrantsSyncer + TypeScopedEntitlementsSyncer + }{base, tsg, tse} + case hasTSG: + return &struct { + *resourceSyncerV1toV2 + TypeScopedGrantsSyncer + }{base, tsg} + case hasTSE: + return &struct { + *resourceSyncerV1toV2 + TypeScopedEntitlementsSyncer + }{base, tse} + default: + return base + } } type resourceSyncerV1toV2 struct { @@ -399,6 +691,36 @@ func (rw *resourceSyncerV1toV2) Grants(ctx context.Context, r *v2.Resource, opts return grants, ret, err } +// validateTypeScopedRegistration fails connector construction when a +// resource type's annotations promise a type-scoped phase the syncer +// cannot serve: a TypeScopedGrants / TypeScopedEntitlements annotation +// without the matching TypeScoped*Syncer interface would otherwise die +// mid-sync with InvalidArgument on the first type-scoped call — a +// runtime failure for what is a wiring bug, caught here at build. +// +// The reverse (interface implemented, annotation absent) is deliberately +// allowed: shared embedded bases may implement the interface across many +// syncers while only some types opt in via the annotation, and the +// unreferenced method is harmless. +func validateTypeScopedRegistration(rType *v2.ResourceType, in any) error { + rtAnnos := annotations.Annotations(rType.GetAnnotations()) + if rtAnnos.Contains(&v2.TypeScopedGrants{}) { + if _, ok := in.(TypeScopedGrantsSyncer); !ok { + return fmt.Errorf( + "resource type %s carries the TypeScopedGrants annotation but its syncer does not implement TypeScopedGrantsSyncer", + rType.GetId()) + } + } + if rtAnnos.Contains(&v2.TypeScopedEntitlements{}) { + if _, ok := in.(TypeScopedEntitlementsSyncer); !ok { + return fmt.Errorf( + "resource type %s carries the TypeScopedEntitlements annotation but its syncer does not implement TypeScopedEntitlementsSyncer", + rType.GetId()) + } + } + return nil +} + func (b *builder) addTargetedSyncer(_ context.Context, typeId string, in any) error { if targetedSyncer, ok := in.(ResourceTargetedSyncerLimited); ok { if _, ok := b.resourceTargetedSyncers[typeId]; ok { diff --git a/pkg/connectorbuilder/source_cache_slot_test.go b/pkg/connectorbuilder/source_cache_slot_test.go new file mode 100644 index 000000000..bede40a41 --- /dev/null +++ b/pkg/connectorbuilder/source_cache_slot_test.go @@ -0,0 +1,106 @@ +package connectorbuilder + +// The builder holds ONE mutable source-cache lookup slot shared by every +// sync against it (see SetSourceCache). The runner supports concurrent +// tasks, so installs/clears can race list RPCs reading the slot: the slot +// accesses must be synchronized, and an install over an existing lookup +// (two live syncs cross-wiring each other) must be loudly visible. + +import ( + "context" + "sync" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/conductorone/baton-sdk/pkg/sourcecache" +) + +// TestSetSourceCacheConcurrentWithListCalls hammers SetSourceCache +// (install/clear cycles, as concurrent syncs would) while list RPCs read +// the slot. Run under -race: unsynchronized slot access is a data race +// even when every interleaving happens to behave. +func TestSetSourceCacheConcurrentWithListCalls(t *testing.T) { + ctx := context.Background() + ts := &continuationTestSyncer{ + testResourceSyncerV2Simple: testResourceSyncerV2Simple{resourceType: "group"}, + scope: "groups/g1/members", + } + b := newContinuationTestBuilder(t, ts) + + stop := make(chan struct{}) + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + for i := 0; ; i++ { + select { + case <-stop: + return + default: + } + if i%2 == 0 { + b.SetSourceCache(ctx, sourcecache.NoopLookup{}) + } else { + b.SetSourceCache(ctx, nil) + } + } + }() + + for i := 0; i < 200; i++ { + _, err := b.ListGrants(ctx, continuationGrantsRequest(nil)) + require.NoError(t, err) + } + close(stop) + wg.Wait() +} + +// slotTestLookup is a non-noop lookup whose hits identify their owner. +type slotTestLookup struct{ validator string } + +func (l slotTestLookup) Lookup(context.Context, sourcecache.RowKind, string) (sourcecache.Entry, bool, error) { + return sourcecache.Entry{CacheValidator: l.validator}, true, nil +} + +// TestSetSourceCacheContentionBlindsSlot pins the in-process slot's +// contention defense (mirroring sourcecache.GRPCServer): two live +// installs cannot be attributed by lookups (no sync-id routing), so the +// slot must serve misses to EVERYONE until every owner clears — serving +// either lookup would cross-wire validators between previous-sync +// artifacts. After the overlap drains, a fresh install serves normally. +func TestSetSourceCacheContentionBlindsSlot(t *testing.T) { + ctx := context.Background() + b := &builder{} + + lookupHits := func() (string, bool) { + lk := b.sourceCacheLookup() + require.NotNil(t, lk) + entry, found, err := lk.Lookup(ctx, sourcecache.RowKindGrants, "scope") + require.NoError(t, err) + return entry.CacheValidator, found + } + + // Single owner: served. + b.SetSourceCache(ctx, slotTestLookup{validator: "sync-a"}) + v, found := lookupHits() + require.True(t, found) + require.Equal(t, "sync-a", v) + + // Second concurrent owner: blinded for everyone. + b.SetSourceCache(ctx, slotTestLookup{validator: "sync-b"}) + _, found = lookupHits() + require.False(t, found, "contended slot must serve misses, not either sync's lookup") + + // One owner clears; the survivor must STAY blinded (the slot cannot + // know which lookup the survivor owns). + b.SetSourceCache(ctx, nil) + _, found = lookupHits() + require.False(t, found, "slot must stay blinded until the overlap fully drains") + + // Fully drained: a fresh install serves again. + b.SetSourceCache(ctx, nil) + b.SetSourceCache(ctx, slotTestLookup{validator: "sync-c"}) + v, found = lookupHits() + require.True(t, found, "a drained slot must serve a fresh single owner") + require.Equal(t, "sync-c", v) +} diff --git a/pkg/connectorbuilder/span_error_test.go b/pkg/connectorbuilder/span_error_test.go index 4a01e1e38..6a55ebe9b 100644 --- a/pkg/connectorbuilder/span_error_test.go +++ b/pkg/connectorbuilder/span_error_test.go @@ -6,6 +6,7 @@ import ( "testing" v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + "github.com/conductorone/baton-sdk/pkg/annotations" "github.com/stretchr/testify/require" "go.opentelemetry.io/otel" otelcodes "go.opentelemetry.io/otel/codes" @@ -40,26 +41,30 @@ func (r *recordingExporter) spansByName(name string) []sdktrace.ReadOnlySpan { return out } -// installSpanRecorder swaps the global otel tracer provider for an in-memory -// recorder for the duration of the test. The package-level tracer var in -// connectorbuilder.go is captured at init via otel.Tracer(name), which -// returns a delegating wrapper that lazily routes through the current global -// provider — so this swap reaches the existing tracer references. +// installSpanRecorder installs an in-memory span recorder as the global +// otel tracer provider, ONCE per test process, and returns it. A singleton +// because otel's global delegation is one-shot: the package-level tracer in +// connectorbuilder.go (captured at init via otel.Tracer) wires itself to +// the FIRST SetTracerProvider and never re-delegates — a second install +// would silently record nothing, and restoring the previous provider on +// cleanup would shut the only wired exporter down for every later test. // -// Mutates global state; callers must not call t.Parallel(). +// The recorder accumulates spans across tests; assert on the LAST span of +// a name. Mutates global state; callers must not call t.Parallel(). func installSpanRecorder(t *testing.T) *recordingExporter { t.Helper() - exp := &recordingExporter{} - tp := sdktrace.NewTracerProvider(sdktrace.WithSyncer(exp)) - prev := otel.GetTracerProvider() - otel.SetTracerProvider(tp) - t.Cleanup(func() { - otel.SetTracerProvider(prev) - _ = tp.Shutdown(context.Background()) + spanRecorderOnce.Do(func() { + spanRecorderExp = &recordingExporter{} + otel.SetTracerProvider(sdktrace.NewTracerProvider(sdktrace.WithSyncer(spanRecorderExp))) }) - return exp + return spanRecorderExp } +var ( + spanRecorderOnce sync.Once + spanRecorderExp *recordingExporter +) + // TestSpanErrorRecorded_EarlyReturns is the regression test for OPS-1543. // Pre-fix, ~30 early-return error paths in pkg/connectorbuilder shadowed // the function-scope err that the deferred uotel.EndSpanWithError reads, @@ -69,6 +74,31 @@ func installSpanRecorder(t *testing.T) *recordingExporter { // one representative path per bug-pattern class: err := inside if-block // with InvalidArgument, err := inside if-block with NotFound, and direct // return (no outer assignment) with NotFound. +// TestSpanOKOnDeferredProtocolTurn is the inverse regression: a deferred +// source-cache lookup (ask/answer bounce) is a PROTOCOL TURN — the RPC +// succeeds — but the handler's ErrLookupDeferred used to linger in the +// function-scoped err that the deferred EndSpanWithError reads, marking +// every normal bounce as a trace error. The span must end status:ok. +func TestSpanOKOnDeferredProtocolTurn(t *testing.T) { + ctx := context.Background() + exp := installSpanRecorder(t) + + ts := &continuationTestSyncer{testResourceSyncerV2Simple: testResourceSyncerV2Simple{resourceType: "group"}, scope: "groups/g1/members"} + b := newContinuationTestBuilder(t, ts) + + offer := annotations.New(&v2.SourceCacheLookupOffer{}) + resp, err := b.ListGrants(ctx, continuationGrantsRequest(offer)) + require.NoError(t, err, "a deferral is a protocol turn, not a failure") + respAnnos := annotations.Annotations(resp.GetAnnotations()) + require.True(t, respAnnos.Contains(&v2.SourceCacheLookupAsk{}), "sanity: this must be the ask turn") + + spans := exp.spansByName("builder.ListGrants") + require.NotEmpty(t, spans) + s := spans[len(spans)-1] + require.NotEqual(t, otelcodes.Error, s.Status().Code, + "a deferred ask/answer turn must not record a span error; ErrLookupDeferred must be cleared before the deferred EndSpanWithError reads err") +} + func TestSpanErrorRecorded_EarlyReturns(t *testing.T) { ctx := context.Background() exp := installSpanRecorder(t) diff --git a/pkg/connectorbuilder/type_scoped_entitlements_test.go b/pkg/connectorbuilder/type_scoped_entitlements_test.go new file mode 100644 index 000000000..2bec00f4a --- /dev/null +++ b/pkg/connectorbuilder/type_scoped_entitlements_test.go @@ -0,0 +1,146 @@ +package connectorbuilder + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + "github.com/conductorone/baton-sdk/pkg/annotations" + "github.com/conductorone/baton-sdk/pkg/sourcecache" + "github.com/conductorone/baton-sdk/pkg/types/resource" +) + +type typeScopedEntitlementsSyncer struct { + testResourceSyncerV2Simple + calls int + perResource int + deferredLookup bool +} + +func (s *typeScopedEntitlementsSyncer) Entitlements(ctx context.Context, r *v2.Resource, opts resource.SyncOpAttrs) ([]*v2.Entitlement, *resource.SyncOpResults, error) { + s.perResource++ + return s.testResourceSyncerV2Simple.Entitlements(ctx, r, opts) +} + +func (s *typeScopedEntitlementsSyncer) EntitlementsForResourceType(ctx context.Context, resourceTypeID string, opts resource.SyncOpAttrs) ([]*v2.Entitlement, *resource.SyncOpResults, error) { + s.calls++ + if s.deferredLookup { + _, _, err := opts.SourceCache.Lookup(ctx, sourcecache.RowKindEntitlements, "ents/type") + if err != nil { + return nil, nil, err + } + } + r := v2.Resource_builder{ + Id: v2.ResourceId_builder{ResourceType: resourceTypeID, Resource: "g1"}.Build(), + }.Build() + return []*v2.Entitlement{ + v2.Entitlement_builder{ + Id: "ent-type-1", + Resource: r, + }.Build(), + }, &resource.SyncOpResults{}, nil +} + +func typeScopedEntitlementsRequest(annos annotations.Annotations) *v2.EntitlementsServiceListEntitlementsRequest { + return v2.EntitlementsServiceListEntitlementsRequest_builder{ + Resource: v2.Resource_builder{ + Id: v2.ResourceId_builder{ResourceType: "group", Resource: "group"}.Build(), + }.Build(), + Annotations: annos, + }.Build() +} + +// annotatedNoInterfaceSyncer carries the TypeScopedEntitlements annotation +// on its resource type but does NOT implement TypeScopedEntitlementsSyncer. +type annotatedNoInterfaceSyncer struct { + testResourceSyncerV2Simple +} + +func (s *annotatedNoInterfaceSyncer) ResourceType(ctx context.Context) *v2.ResourceType { + rt := s.testResourceSyncerV2Simple.ResourceType(ctx) + rt.SetAnnotations(annotations.New(&v2.TypeScopedEntitlements{})) + return rt +} + +// TestNewConnector_TypeScopedAnnotationRequiresInterface pins the +// registration contract: an annotated type whose syncer lacks the +// matching TypeScoped*Syncer interface fails at NewConnector instead of +// dying mid-sync with InvalidArgument on the first type-scoped call. +func TestNewConnector_TypeScopedAnnotationRequiresInterface(t *testing.T) { + ctx := context.Background() + bad := &annotatedNoInterfaceSyncer{testResourceSyncerV2Simple{resourceType: "group"}} + _, err := NewConnector(ctx, &testConnectorV2{resourceSyncers: []ResourceSyncerV2{bad}}) + require.Error(t, err) + require.Contains(t, err.Error(), "TypeScopedEntitlements annotation") +} + +func TestListEntitlements_TypeScopedRoutes(t *testing.T) { + ctx := context.Background() + ts := &typeScopedEntitlementsSyncer{testResourceSyncerV2Simple: testResourceSyncerV2Simple{resourceType: "group"}} + conn, err := NewConnector(ctx, &testConnectorV2{resourceSyncers: []ResourceSyncerV2{ts}}) + require.NoError(t, err) + + resp, err := conn.ListEntitlements(ctx, typeScopedEntitlementsRequest(annotations.New(&v2.TypeScopedEntitlements{}))) + require.NoError(t, err) + require.Len(t, resp.GetList(), 1) + require.Equal(t, "ent-type-1", resp.GetList()[0].GetId()) + require.Equal(t, 1, ts.calls) + require.Zero(t, ts.perResource) +} + +func TestListEntitlements_TypeScopedMissingInterface(t *testing.T) { + ctx := context.Background() + ts := &testResourceSyncerV2Simple{resourceType: "group"} + conn, err := NewConnector(ctx, &testConnectorV2{resourceSyncers: []ResourceSyncerV2{ts}}) + require.NoError(t, err) + + _, err = conn.ListEntitlements(ctx, typeScopedEntitlementsRequest(annotations.New(&v2.TypeScopedEntitlements{}))) + require.Error(t, err) + require.Equal(t, codes.InvalidArgument, status.Code(err)) +} + +func TestListEntitlements_UnmarkedStillPerResource(t *testing.T) { + ctx := context.Background() + ts := &typeScopedEntitlementsSyncer{testResourceSyncerV2Simple: testResourceSyncerV2Simple{resourceType: "group"}} + conn, err := NewConnector(ctx, &testConnectorV2{resourceSyncers: []ResourceSyncerV2{ts}}) + require.NoError(t, err) + + resp, err := conn.ListEntitlements(ctx, typeScopedEntitlementsRequest(nil)) + require.NoError(t, err) + require.Len(t, resp.GetList(), 1) + require.Equal(t, "entitlement-1", resp.GetList()[0].GetId()) + require.Zero(t, ts.calls) + require.Equal(t, 1, ts.perResource) +} + +func TestListEntitlements_TypeScopedContinuationPreservesMarker(t *testing.T) { + ctx := context.Background() + ts := &typeScopedEntitlementsSyncer{ + testResourceSyncerV2Simple: testResourceSyncerV2Simple{resourceType: "group"}, + deferredLookup: true, + } + conn, err := NewConnector(ctx, &testConnectorV2{resourceSyncers: []ResourceSyncerV2{ts}}) + require.NoError(t, err) + + offer := annotations.New(&v2.SourceCacheLookupOffer{}, &v2.TypeScopedEntitlements{}) + resp, err := conn.ListEntitlements(ctx, typeScopedEntitlementsRequest(offer)) + require.NoError(t, err) + require.Empty(t, resp.GetList()) + respAnnos := annotations.Annotations(resp.GetAnnotations()) + require.True(t, respAnnos.Contains(&v2.SourceCacheLookupAsk{})) + require.Equal(t, 1, ts.calls) + + answers := annotations.New(&v2.SourceCacheLookupOffer{}, &v2.TypeScopedEntitlements{}) + answers.Update(sourcecache.AnswersProto([]sourcecache.Answer{ + {Query: sourcecache.Query{RowKind: sourcecache.RowKindEntitlements, ScopeKey: "ents/type"}, Found: false}, + })) + resp2, err := conn.ListEntitlements(ctx, typeScopedEntitlementsRequest(answers)) + require.NoError(t, err) + require.Len(t, resp2.GetList(), 1) + require.Equal(t, 2, ts.calls) + require.Zero(t, ts.perResource) +} diff --git a/pkg/connectorbuilder/type_scoped_v1_adapter_test.go b/pkg/connectorbuilder/type_scoped_v1_adapter_test.go new file mode 100644 index 000000000..9e84deee9 --- /dev/null +++ b/pkg/connectorbuilder/type_scoped_v1_adapter_test.go @@ -0,0 +1,125 @@ +package connectorbuilder + +// The V1→V2 resource-syncer adapter must FORWARD the optional +// type-scoped interfaces. Registration validates the ORIGINAL syncer +// (validateTypeScopedRegistration) but routing type-asserts the STORED +// value — before the composite adapter shapes in newResourceSyncerV1toV2, +// a V1 syncer implementing TypeScopedGrantsSyncer / +// TypeScopedEntitlementsSyncer passed construction and then failed every +// type-scoped call at runtime with InvalidArgument. + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + "github.com/conductorone/baton-sdk/pkg/annotations" + "github.com/conductorone/baton-sdk/pkg/pagination" + "github.com/conductorone/baton-sdk/pkg/types/resource" +) + +// typeScopedV1Syncer is a V1 ResourceSyncer (pagination.Token +// signatures) that also implements both type-scoped interfaces and +// carries both annotations on its resource type. +type typeScopedV1Syncer struct { + grantCalls int + entCalls int +} + +func (s *typeScopedV1Syncer) ResourceType(context.Context) *v2.ResourceType { + return v2.ResourceType_builder{ + Id: "group", + DisplayName: "Group", + Annotations: annotations.New(&v2.TypeScopedGrants{}, &v2.TypeScopedEntitlements{}), + }.Build() +} + +func (s *typeScopedV1Syncer) List(context.Context, *v2.ResourceId, *pagination.Token) ([]*v2.Resource, string, annotations.Annotations, error) { + return nil, "", nil, nil +} + +func (s *typeScopedV1Syncer) Entitlements(context.Context, *v2.Resource, *pagination.Token) ([]*v2.Entitlement, string, annotations.Annotations, error) { + return nil, "", nil, nil +} + +func (s *typeScopedV1Syncer) Grants(context.Context, *v2.Resource, *pagination.Token) ([]*v2.Grant, string, annotations.Annotations, error) { + return nil, "", nil, nil +} + +func (s *typeScopedV1Syncer) GrantsForResourceType(_ context.Context, resourceTypeID string, _ resource.SyncOpAttrs) ([]*v2.Grant, *resource.SyncOpResults, error) { + s.grantCalls++ + r := v2.Resource_builder{ + Id: v2.ResourceId_builder{ResourceType: resourceTypeID, Resource: "g1"}.Build(), + }.Build() + return []*v2.Grant{ + v2.Grant_builder{ + Id: "grant-type-1", + Entitlement: v2.Entitlement_builder{Id: "ent-1", Resource: r}.Build(), + Principal: v2.Resource_builder{Id: v2.ResourceId_builder{ResourceType: "user", Resource: "u1"}.Build()}.Build(), + }.Build(), + }, &resource.SyncOpResults{}, nil +} + +func (s *typeScopedV1Syncer) EntitlementsForResourceType(_ context.Context, resourceTypeID string, _ resource.SyncOpAttrs) ([]*v2.Entitlement, *resource.SyncOpResults, error) { + s.entCalls++ + r := v2.Resource_builder{ + Id: v2.ResourceId_builder{ResourceType: resourceTypeID, Resource: "g1"}.Build(), + }.Build() + return []*v2.Entitlement{ + v2.Entitlement_builder{Id: "ent-type-1", Resource: r}.Build(), + }, &resource.SyncOpResults{}, nil +} + +func typeScopedV1GrantsRequest(annos annotations.Annotations) *v2.GrantsServiceListGrantsRequest { + return v2.GrantsServiceListGrantsRequest_builder{ + Resource: v2.Resource_builder{ + Id: v2.ResourceId_builder{ResourceType: "group", Resource: "group"}.Build(), + }.Build(), + Annotations: annos, + }.Build() +} + +// TestTypeScopedV1SyncerRoutesThroughAdapter pins the adapter +// forwarding: a V1 syncer's type-scoped methods must be reachable +// through the stored (wrapped) syncer for both phases. +func TestTypeScopedV1SyncerRoutesThroughAdapter(t *testing.T) { + ctx := context.Background() + ts := &typeScopedV1Syncer{} + conn, err := NewConnector(ctx, newTestConnector([]ResourceSyncer{ts})) + require.NoError(t, err, "a V1 type-scoped syncer must pass registration") + + gResp, err := conn.ListGrants(ctx, typeScopedV1GrantsRequest(annotations.New(&v2.TypeScopedGrants{}))) + require.NoError(t, err, "type-scoped grants routing must reach the V1 syncer through the adapter") + require.Len(t, gResp.GetList(), 1) + require.Equal(t, "grant-type-1", gResp.GetList()[0].GetId()) + require.Equal(t, 1, ts.grantCalls) + + eResp, err := conn.ListEntitlements(ctx, typeScopedEntitlementsRequest(annotations.New(&v2.TypeScopedEntitlements{}))) + require.NoError(t, err, "type-scoped entitlements routing must reach the V1 syncer through the adapter") + require.Len(t, eResp.GetList(), 1) + require.Equal(t, "ent-type-1", eResp.GetList()[0].GetId()) + require.Equal(t, 1, ts.entCalls) +} + +// TestTypeScopedV1AdapterDoesNotOverclaim: a plain V1 syncer (no +// type-scoped interfaces) wrapped by the adapter must still FAIL the +// type-scoped routing assertions — the composite shapes must not make +// every adapted syncer claim the interfaces. +func TestTypeScopedV1AdapterDoesNotOverclaim(t *testing.T) { + adapted := newResourceSyncerV1toV2(newTestResourceSyncer("group")) + _, isTSG := adapted.(TypeScopedGrantsSyncer) + require.False(t, isTSG) + _, isTSE := adapted.(TypeScopedEntitlementsSyncer) + require.False(t, isTSE) + + ctx := context.Background() + conn, err := NewConnector(ctx, newTestConnector([]ResourceSyncer{newTestResourceSyncer("group")})) + require.NoError(t, err) + _, err = conn.ListGrants(ctx, typeScopedV1GrantsRequest(annotations.New(&v2.TypeScopedGrants{}))) + require.Error(t, err) + require.Equal(t, codes.InvalidArgument, status.Code(err)) +} diff --git a/pkg/connectorrunner/runner.go b/pkg/connectorrunner/runner.go index c404401b0..b4295036b 100644 --- a/pkg/connectorrunner/runner.go +++ b/pkg/connectorrunner/runner.go @@ -426,6 +426,7 @@ type runnerConfig struct { workerCount int targetedSyncResourceIDs []string externalResourceC1Z string + previousSyncC1Z string externalResourceEntitlementIdFilter string keepPreviousSyncC1ZCapable bool keepPreviousSyncC1ZEnabled bool @@ -760,6 +761,13 @@ func WithExternalResourceC1Z(externalResourceC1Z string) Option { } } +func WithPreviousSyncC1Z(previousSyncC1Z string) Option { + return func(ctx context.Context, cfg *runnerConfig) error { + cfg.previousSyncC1Z = previousSyncC1Z + return nil + } +} + func WithExternalResourceEntitlementFilter(entitlementId string) Option { return func(ctx context.Context, cfg *runnerConfig) error { cfg.externalResourceEntitlementIdFilter = entitlementId @@ -802,6 +810,25 @@ func WithKeepPreviousSyncC1ZRuntimeOptIn() Option { } } +// DeclaresPreviousSyncCapability reports whether opts include the connector +// author's replay declaration (WithKeepPreviousSyncC1Z). Used at +// command-definition time to decide whether the replay CLI flags +// (--previous-sync-c1z / --keep-previous-sync-c1z) appear in help: they are +// hidden for the overwhelming majority of connectors that don't support +// replay, and surfaced only for the ones whose author baked the capability +// into their RunConnector options. Options are applied to a scratch config; +// they are plain setters, so this is side-effect free. +func DeclaresPreviousSyncCapability(ctx context.Context, opts ...Option) bool { + cfg := &runnerConfig{} + for _, o := range opts { + if o == nil { + continue + } + _ = o(ctx, cfg) + } + return cfg.keepPreviousSyncC1ZCapable +} + func WithDiffSyncs(c1zPath string, baseSyncID string, newSyncID string) Option { return func(ctx context.Context, cfg *runnerConfig) error { cfg.onDemand = true @@ -973,7 +1000,15 @@ func NewConnectorRunner(ctx context.Context, c types.ConnectorServer, opts ...Op wrapperOpts = append(wrapperOpts, connector.WithTargetedSyncResources(cfg.targetedSyncResourceIDs)) } - if cfg.sessionStoreEnabled { + // The parent control-plane listener serves BOTH BatonSessionService and + // BatonSourceCacheService (internal/connector.runServer). Source-cache + // replay therefore needs the listener even when the connector never uses + // sessions: without it the subprocess connector's lookup degrades to + // NoopLookup, every scope misses, and every sync runs a full cold + // enumeration despite a valid --previous-sync-c1z. Start it whenever + // replay could be in play: the author declared the capability + // (WithKeepPreviousSyncC1Z) or a previous-sync c1z was configured. + if cfg.sessionStoreEnabled || cfg.keepPreviousSyncC1ZCapable || cfg.previousSyncC1Z != "" { wrapperOpts = append(wrapperOpts, connector.WithSessionStoreEnabled()) } @@ -1081,6 +1116,7 @@ func NewConnectorRunner(ctx context.Context, c types.ConnectorServer, opts ...Op tm, err = local.NewSyncer(ctx, cfg.c1zPath, local.WithTmpDir(cfg.tempDir), local.WithExternalResourceC1Z(cfg.externalResourceC1Z), + local.WithPreviousSyncC1Z(cfg.previousSyncC1Z), local.WithExternalResourceEntitlementIdFilter(cfg.externalResourceEntitlementIdFilter), local.WithTargetedSyncResources(resources), local.WithSkipEntitlementsAndGrants(cfg.skipEntitlementsAndGrants), @@ -1130,6 +1166,7 @@ func NewConnectorRunner(ctx context.Context, c types.ConnectorServer, opts ...Op cfg.storageEngine, runner.taskConcurrency, keepPreviousSyncC1Z, + cfg.previousSyncC1Z, ) if err != nil { return nil, err diff --git a/pkg/dotc1z/assets.go b/pkg/dotc1z/assets.go index 618f36318..863127cb4 100644 --- a/pkg/dotc1z/assets.go +++ b/pkg/dotc1z/assets.go @@ -104,7 +104,7 @@ func (c *C1File) PutAsset(ctx context.Context, assetRef *v2.AssetRef, contentTyp return err } - c.dbUpdated = true + c.dbUpdated.Store(true) return nil } diff --git a/pkg/dotc1z/c1file.go b/pkg/dotc1z/c1file.go index a9feeec68..564a44e35 100644 --- a/pkg/dotc1z/c1file.go +++ b/pkg/dotc1z/c1file.go @@ -10,6 +10,7 @@ import ( "path/filepath" "strings" "sync" + "sync/atomic" "time" "github.com/doug-martin/goqu/v9" @@ -43,13 +44,16 @@ type pragma struct { } type C1File struct { - rawDb *sql.DB - db *goqu.Database - currentSyncID string - viewSyncID string - outputFilePath string - dbFilePath string - dbUpdated bool + rawDb *sql.DB + db *goqu.Database + currentSyncID string + viewSyncID string + outputFilePath string + dbFilePath string + // dbUpdated records that any mutation ran; parallel sync workers set + // it concurrently, so it is atomic (it was a plain bool data race + // under -race). Only ever set true during a session; read at Close. + dbUpdated atomic.Bool tempDir string pragmas []pragma readOnly bool @@ -57,6 +61,15 @@ type C1File struct { closed bool closedMu sync.Mutex + // dbClosed is set by closeRawDB just before the pool closes. It is + // the race-safe "no more operations" signal: c.db itself is never + // nil'ed on close (an operation concurrent with Close may already + // hold the pointer, and the unsynchronized nil-write is itself a + // data race against validateDb's read), so validateDb checks this + // flag and anything already past validateDb fails on the closed + // pool's own "sql: database is closed" error instead. + dbClosed atomic.Bool + // bulkLoad defers secondary-index creation on a freshly-created // destination. When set, the per-table non-unique secondary indexes // are dropped right after table creation (instant on the empty table) @@ -624,7 +637,7 @@ func (c *C1File) Close(ctx context.Context) (retErr error) { span.SetAttributes( attribute.Bool("read_only", c.readOnly), - attribute.Bool("db_updated", c.dbUpdated), + attribute.Bool("db_updated", c.dbUpdated.Load()), attribute.String("db_path", c.dbFilePath), ) @@ -635,13 +648,13 @@ func (c *C1File) Close(ctx context.Context) (retErr error) { // open) before returning so a misuse like opening read-only and // then dirtying via an attached-db mutation still releases the // SQLite handle and any FDs/goroutines it owns. - if !c.dbUpdated || c.readOnly { + if !c.dbUpdated.Load() || c.readOnly { if c.rawDb != nil { if err := c.closeRawDB(ctx); err != nil { return cleanupDbDir(c.dbFilePath, err) } } - if c.dbUpdated && c.readOnly { + if c.dbUpdated.Load() && c.readOnly { c.closed = true return cleanupDbDir(c.dbFilePath, ErrReadOnly) } @@ -811,8 +824,16 @@ func (c *C1File) closeRawDB(ctx context.Context) error { // Copy the rawDb to a local variable to avoid race conditions. rawDb := c.rawDb c.rawDb = nil + // Flag first, then close: operations that haven't passed validateDb + // yet are rejected with ErrDbNotOpen; anything already past it fails + // on the closed pool's own "sql: database is closed". c.db is + // deliberately NOT nil'ed — a concurrent operation may already have + // read the pointer and goqu dereferences it (nil here means a + // SIGSEGV in BeginTx, and the unsynchronized write itself is a data + // race against validateDb's read); the closed pool underneath makes + // the handle inert. + c.dbClosed.Store(true) err = rawDb.Close() - c.db = nil return err } @@ -1331,7 +1352,7 @@ func (c *C1File) stats(ctx context.Context, syncType connectorstore.SyncType, sy if err != nil { return nil, nil, fmt.Errorf("c1file-stats: error saving stats: %w", err) } - c.dbUpdated = true + c.dbUpdated.Store(true) } return &reader_v2.SyncRun{ @@ -1414,9 +1435,9 @@ func (c *C1File) countBySyncAndResourceType( return out, nil } -// validateDb ensures that the database has been opened. +// validateDb ensures that the database has been opened and not closed. func (c *C1File) validateDb(ctx context.Context) error { - if c.db == nil { + if c.db == nil || c.dbClosed.Load() { return ErrDbNotOpen } diff --git a/pkg/dotc1z/c1file_attached.go b/pkg/dotc1z/c1file_attached.go index fa6a4c96b..6ae5a76f4 100644 --- a/pkg/dotc1z/c1file_attached.go +++ b/pkg/dotc1z/c1file_attached.go @@ -346,7 +346,7 @@ func (c *C1FileAttached) GenerateSyncDiffFromFile(ctx context.Context, oldSyncID return "", "", fmt.Errorf("failed to commit transaction: %w", err) } committed = true - c.file.dbUpdated = true + c.file.dbUpdated.Store(true) return upsertsSyncID, deletionsSyncID, nil } diff --git a/pkg/dotc1z/c1file_close_cancel_test.go b/pkg/dotc1z/c1file_close_cancel_test.go index 58eab5d5e..dcd2126d2 100644 --- a/pkg/dotc1z/c1file_close_cancel_test.go +++ b/pkg/dotc1z/c1file_close_cancel_test.go @@ -138,7 +138,7 @@ func TestC1FileCloseReadOnlyButDirtyClosesRawDb(t *testing.T) { require.NoError(t, err) require.NotNil(t, f2.rawDb) // Force the readonly+dbUpdated cheap-path branch. - f2.dbUpdated = true + f2.dbUpdated.Store(true) err = f2.Close(openCtx) require.ErrorIs(t, err, ErrReadOnly) diff --git a/pkg/dotc1z/c1file_concurrent_test.go b/pkg/dotc1z/c1file_concurrent_test.go index 32fb8fa90..84c761d1a 100644 --- a/pkg/dotc1z/c1file_concurrent_test.go +++ b/pkg/dotc1z/c1file_concurrent_test.go @@ -1,9 +1,11 @@ package dotc1z import ( + "errors" "fmt" "os" "path/filepath" + "strings" "sync" "testing" "time" @@ -115,7 +117,11 @@ func TestC1ZConcurrentClose(t *testing.T) { var err error // Close will finish at some point, causing DB operations to fail. defer wg.Done() - // Put grants in a loop until we get a DbNotOpen error. + // Put grants in a loop until Close shuts the database down under + // us. Two benign shapes exist: ErrDbNotOpen (validateDb saw the + // closed flag) or database/sql's "database is closed" (the + // operation passed validateDb just before Close flagged the + // handle, then hit the closed pool). i := 0 for { err = f.PutGrants(ctx, v2.Grant_builder{ @@ -137,7 +143,9 @@ func TestC1ZConcurrentClose(t *testing.T) { }.Build(), }.Build()) if err != nil { - require.ErrorIs(t, err, ErrDbNotOpen) + require.Truef(t, + errors.Is(err, ErrDbNotOpen) || strings.Contains(err.Error(), "database is closed"), + "PutGrants during close error = %v, want ErrDbNotOpen or closed pool", err) break } i++ diff --git a/pkg/dotc1z/c1zstore/grant_store.go b/pkg/dotc1z/c1zstore/grant_store.go index 55f972387..c0caccc91 100644 --- a/pkg/dotc1z/c1zstore/grant_store.go +++ b/pkg/dotc1z/c1zstore/grant_store.go @@ -145,4 +145,13 @@ type GrantAnnotation struct { PrincipalResourceTypeID string PrincipalResourceID string NeedsExpansion bool + + // SourceScopeKey is the grant row's source-cache scope stamp ("" for + // unstamped rows, and always "" on engines without source-cache + // support). The external-principal post-processing step writes its + // transformed grants under the SOURCE grant's scope so a future + // sync's replay of that scope carries them forward — the source + // grant itself is deleted by that step, and a scope left holding + // neither source nor transformed rows would silently replay empty. + SourceScopeKey string } diff --git a/pkg/dotc1z/c1zstore/sync_meta.go b/pkg/dotc1z/c1zstore/sync_meta.go index 90af84b4b..174fd168a 100644 --- a/pkg/dotc1z/c1zstore/sync_meta.go +++ b/pkg/dotc1z/c1zstore/sync_meta.go @@ -57,5 +57,35 @@ type SyncRun struct { ParentSyncID string LinkedSyncID string SupportsDiff bool - Stats *reader_v2.SyncStats + // Compacted marks a sync produced by compaction rather than a real + // connector run. Compacted artifacts are keep-newer upsert merges, so + // no input's source-cache validators (etags) describe their contents: + // the syncer refuses to use a compacted sync as a replay source + // (degrades to a cold sync), and orchestrators choosing which + // artifact to materialize as the previous sync should read + // UsableAsReplaySource instead of guessing from provenance. + // Always false on the SQLite engine (v1 files cannot be replay + // sources regardless). + Compacted bool + Stats *reader_v2.SyncStats +} + +// UsableAsReplaySource answers "can this sync serve source-cache replay?" +// — the question an orchestrator must ask before materializing an +// artifact as a syncer's previous sync. The predicate is deliberately +// narrow: only a FULL, non-compacted sync is a faithful snapshot that its +// own recorded validators describe. +// +// - Partial/targeted, resources-only, and diff-typed syncs are subsets +// or derivations — their rows do not cover the scopes any validator +// would vouch for (they also never write manifest entries, but the +// type gate makes the refusal explicit rather than incidental). +// - Compacted syncs are keep-newer merges of multiple runs; no single +// input's validators describe the merged row set. +// +// The syncer enforces this same predicate when a previous-sync c1z is +// supplied (see pkg/sync's configureSourceCache): a non-qualifying file +// degrades to a cold sync rather than replaying. +func (s *SyncRun) UsableAsReplaySource() bool { + return s != nil && s.Type == connectorstore.SyncTypeFull && !s.Compacted } diff --git a/pkg/dotc1z/c1zstore/sync_meta_test.go b/pkg/dotc1z/c1zstore/sync_meta_test.go new file mode 100644 index 000000000..e734ba4bb --- /dev/null +++ b/pkg/dotc1z/c1zstore/sync_meta_test.go @@ -0,0 +1,35 @@ +package c1zstore + +// UsableAsReplaySource is the metadata gate two product non-goals hang +// off of: SQLite artifacts and compacted artifacts must never serve +// source-cache replay. The predicate itself was previously only pinned +// indirectly (through syncer degrade tests); this is its direct truth +// table. + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/conductorone/baton-sdk/pkg/connectorstore" +) + +func TestUsableAsReplaySource(t *testing.T) { + cases := []struct { + name string + run *SyncRun + want bool + }{ + {"nil run", nil, false}, + {"full sync", &SyncRun{Type: connectorstore.SyncTypeFull}, true}, + {"full but compacted", &SyncRun{Type: connectorstore.SyncTypeFull, Compacted: true}, false}, + {"partial sync", &SyncRun{Type: connectorstore.SyncTypePartial}, false}, + {"resources-only sync", &SyncRun{Type: connectorstore.SyncTypeResourcesOnly}, false}, + {"empty type", &SyncRun{}, false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + require.Equal(t, tc.want, tc.run.UsableAsReplaySource()) + }) + } +} diff --git a/pkg/dotc1z/clone_sync.go b/pkg/dotc1z/clone_sync.go index 9c34ed97f..e56c7250a 100644 --- a/pkg/dotc1z/clone_sync.go +++ b/pkg/dotc1z/clone_sync.go @@ -357,7 +357,7 @@ func (c *C1File) cloneCopy(ctx context.Context, outPath string, syncID string, s if err != nil { return err } - outFile.dbUpdated = true + outFile.dbUpdated.Store(true) outFile.outputFilePath = outPath err = outFile.Close(ctx) if err != nil { diff --git a/pkg/dotc1z/copy_isolate_sync.go b/pkg/dotc1z/copy_isolate_sync.go index 0872b5396..28eef769f 100644 --- a/pkg/dotc1z/copy_isolate_sync.go +++ b/pkg/dotc1z/copy_isolate_sync.go @@ -233,7 +233,7 @@ func (c *C1File) CopyIsolateSync(ctx context.Context, outPath string, syncID str // Step 4 — recompress the isolated copy to outPath. Mirrors cloneCopy's // finalize: dbUpdated + outputFilePath + Close() runs the // WAL-checkpoint -> close-raw-db -> saveC1z sequence. - copyFile.dbUpdated = true + copyFile.dbUpdated.Store(true) copyFile.outputFilePath = outPath finalized = true if err = copyFile.Close(ctx); err != nil { diff --git a/pkg/dotc1z/diff.go b/pkg/dotc1z/diff.go index 162001e40..c991b585a 100644 --- a/pkg/dotc1z/diff.go +++ b/pkg/dotc1z/diff.go @@ -55,7 +55,7 @@ func (c *C1File) GenerateSyncDiff(ctx context.Context, baseSyncID string, applie if err != nil { return "", err } - c.dbUpdated = true + c.dbUpdated.Store(true) } if err := c.endSyncRun(ctx, diffSyncID); err != nil { diff --git a/pkg/dotc1z/engine/pebble/adapter.go b/pkg/dotc1z/engine/pebble/adapter.go index 353eebe38..a42581cf5 100644 --- a/pkg/dotc1z/engine/pebble/adapter.go +++ b/pkg/dotc1z/engine/pebble/adapter.go @@ -21,6 +21,7 @@ import ( v3 "github.com/conductorone/baton-sdk/pb/c1/storage/v3" "github.com/conductorone/baton-sdk/pkg/connectorstore" "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" + "github.com/conductorone/baton-sdk/pkg/sourcecache" ) // Adapter wraps an *Engine and implements connectorstore.Writer @@ -276,15 +277,12 @@ func (a *Adapter) CheckpointSync(ctx context.Context, syncToken string) error { if err != nil { return err } - updated := v3.SyncRunRecord_builder{ - SyncId: existing.GetSyncId(), - Type: existing.GetType(), - ParentSyncId: existing.GetParentSyncId(), - StartedAt: existing.GetStartedAt(), - EndedAt: existing.GetEndedAt(), - SyncToken: syncToken, - }.Build() - return a.engine.PutSyncRunRecord(ctx, updated) + // Mutate the loaded record rather than rebuilding it field-by-field: + // a rebuild silently drops every field it doesn't name (this used to + // erase compaction provenance when the compactor's grant-expansion + // pass checkpointed over a compacted-marked sync). + existing.SetSyncToken(syncToken) + return a.engine.PutSyncRunRecord(ctx, existing) } // EndSync stamps the open sync_run's ended_at and detaches it. After @@ -329,7 +327,9 @@ func (a *Adapter) EndSync(ctx context.Context) error { } // endSyncFinalize runs the sealed tail of EndSync: the deferred index -// build, the ended_at stamp, the stats sidecar, and the durability flush. +// build, the stats sidecar, the durability flush, and — strictly last — +// the ended_at stamp (the stamp is the finished verdict; it must never +// precede the flush that makes the sync's data durable). // Runs with the engine SEALED (see EndSync) — every write below goes // through an AllowSealed path. Split out so EndSync can unseal on failure. func (e *Engine) endSyncFinalize(ctx context.Context, existing *v3.SyncRunRecord) error { @@ -367,25 +367,13 @@ func (e *Engine) endSyncFinalize(ctx context.Context, existing *v3.SyncRunRecord return fmt.Errorf("EndSync: repair grant digests: %w", err) } } - updated := v3.SyncRunRecord_builder{ - SyncId: existing.GetSyncId(), - Type: existing.GetType(), - ParentSyncId: existing.GetParentSyncId(), - StartedAt: existing.GetStartedAt(), - EndedAt: timestamppb.Now(), - SyncToken: existing.GetSyncToken(), - }.Build() - if err := e.PutSyncRunRecord(ctx, updated); err != nil { - return err - } // Populate the stats sidecar BEFORE the durability flush. Stats // is engine-meta keyspace; the EndFreshSync flush below covers - // the WAL fsync for both the sync_run record and the stats key. - // Failures here are non-fatal — Stats() falls back to legacy - // iteration on a missing sidecar, and the on-Open migration - // framework will backfill next time the file opens. We log a - // warning so the failure is visible in production telemetry but - // don't fail the sync end on stats-sidecar trouble. + // the WAL fsync for the stats key. Failures here are non-fatal — + // Stats() falls back to legacy iteration on a missing sidecar, and + // the on-Open migration framework will backfill next time the file + // opens. We log a warning so the failure is visible in production + // telemetry but don't fail the sync end on stats-sidecar trouble. if err := e.PersistSyncStats(ctx, existing.GetSyncId()); err != nil { ctxzap.Extract(ctx).Warn("pebble: persist sync stats sidecar failed; Stats() will fall back to O(N) iteration until the next Open backfills it", zap.String("sync_id", existing.GetSyncId()), @@ -394,8 +382,47 @@ func (e *Engine) endSyncFinalize(ctx context.Context, existing *v3.SyncRunRecord } // Single flush + WAL fsync at sync end. This is the durability // boundary — counterpart to MarkFreshSync at StartNewSync. After - // this returns, all writes from the sync are on disk. - return e.EndFreshSync(ctx) + // this returns, all writes from the sync are on disk. The sync stays + // BOUND across the flush (FlushForEndSync, not EndFreshSync) so a + // failed ended_at stamp below leaves the engine retryable in-process. + if e.testEndSyncFlushHook != nil { + if err := e.testEndSyncFlushHook(); err != nil { + return err + } + } + if err := e.FlushForEndSync(ctx); err != nil { + return err + } + // Stamp ended_at strictly AFTER the durability flush: the stamp is + // what makes the sync FINISHED (LatestUnfinishedSyncRecord skips + // stamped records), so it must never precede the flush that makes + // the sync's NoSync writes durable. A stamp committed ahead of a + // failed flush would orphan the unfinished work — resume starts a + // NEW sync instead of resuming — and, across a crash, could surface + // a partially-durable artifact as a FINISHED sync to replay/uplift + // consumers. The stamp's own commit is fsynced (PutSyncRunRecord + // writes with the engine's configured durability), so a crash after + // this line keeps the finished verdict and a crash before it keeps + // resumability. Mutate the loaded record rather than rebuilding it + // field-by-field — a rebuild drops every field it doesn't name + // (supports_diff, linked_sync_id, compacted), which erased compaction + // provenance and diff markers when a sync re-ended after expansion + // or resume. + existing.SetEndedAt(timestamppb.Now()) + if e.testEndSyncStampHook != nil { + if err := e.testEndSyncStampHook(); err != nil { + return err + } + } + if err := e.PutSyncRunRecord(ctx, existing); err != nil { + // The in-memory stamp above needs no rollback: `existing` is a + // local copy unmarshalled by this EndSync call, unreachable once + // this error returns; a retried EndSync reloads the (unstamped) + // stored record. Pinned by the endsync-stamp-commit harness row. + return err + } + e.clearCurrentSync() + return nil } // === writes === @@ -422,12 +449,27 @@ func (a *Adapter) PutGrants(ctx context.Context, grants ...*v2.Grant) error { return ErrNoCurrentSync } records := translateGrants(syncID, grants) + stampSourceScope(ctx, records, func(r *v3.GrantRecord, s string) { r.SetSourceScopeKey(s) }) if err := a.engine.PutGrantRecords(ctx, records...); err != nil { return fmt.Errorf("PutGrants: %w", err) } return nil } +// stampSourceScope stamps the source-cache scope key carried by ctx +// (sourcecache.WithScope) onto freshly translated records. The syncer +// sets the scope around a page's store writes when the page carried a +// SourceCacheRecord annotation; everything else writes unstamped rows. +func stampSourceScope[T any](ctx context.Context, records []T, set func(T, string)) { + scope := sourcecache.ScopeFromContext(ctx) + if scope == "" { + return + } + for _, r := range records { + set(r, scope) + } +} + // UnsafePutUniqueGrants writes grants on the trusted-import path: records // are encoded in parallel and written unconditionally, with no read-before-write // and no dedup pass. Do not use it for live connector output. The destination @@ -585,6 +627,7 @@ func (a *Adapter) PutResources(ctx context.Context, resources ...*v2.Resource) e } records = append(records, rec) } + stampSourceScope(ctx, records, func(r *v3.ResourceRecord, s string) { r.SetSourceScopeKey(s) }) if err := a.engine.PutResourceRecords(ctx, records...); err != nil { return fmt.Errorf("PutResources: %w", err) } @@ -612,6 +655,7 @@ func (a *Adapter) PutEntitlements(ctx context.Context, entitlements ...*v2.Entit } records = append(records, rec) } + stampSourceScope(ctx, records, func(r *v3.EntitlementRecord, s string) { r.SetSourceScopeKey(s) }) if err := a.engine.PutEntitlementRecords(ctx, records...); err != nil { return fmt.Errorf("PutEntitlements: %w", err) } diff --git a/pkg/dotc1z/engine/pebble/adapter_grants_store.go b/pkg/dotc1z/engine/pebble/adapter_grants_store.go index 30736c59e..f88453a58 100644 --- a/pkg/dotc1z/engine/pebble/adapter_grants_store.go +++ b/pkg/dotc1z/engine/pebble/adapter_grants_store.go @@ -220,6 +220,10 @@ func (g pebbleGrantStore) translateExpanded(syncID string, grants []*v2.Grant) [ // because the caller left a residual GrantExpandable annotation. newRec.SetExpansion(nil) newRec.SetNeedsExpansion(false) + // Same shape for the source-cache scope stamp: existing records get + // their prior stamp restored in PutExpandedGrantRecords; brand-new + // expander-derived rows are never part of a source scope. + newRec.SetSourceScopeKey("") merged = append(merged, newRec) } return merged @@ -327,6 +331,7 @@ func (g pebbleGrantStore) ListWithAnnotationsPage(ctx context.Context, pageToken PrincipalResourceTypeID: princ.GetResourceTypeId(), PrincipalResourceID: princ.GetResourceId(), NeedsExpansion: rec.GetNeedsExpansion(), + SourceScopeKey: rec.GetSourceScopeKey(), }) } return rows, next, nil @@ -372,6 +377,7 @@ func (g pebbleGrantStore) ListWithAnnotationsForResourcePage( PrincipalResourceTypeID: rec.GetPrincipal().GetResourceTypeId(), PrincipalResourceID: rec.GetPrincipal().GetResourceId(), NeedsExpansion: rec.GetNeedsExpansion(), + SourceScopeKey: rec.GetSourceScopeKey(), }) } return rows, next, nil diff --git a/pkg/dotc1z/engine/pebble/adapter_sync_meta.go b/pkg/dotc1z/engine/pebble/adapter_sync_meta.go index 5dc736644..ddaec85a4 100644 --- a/pkg/dotc1z/engine/pebble/adapter_sync_meta.go +++ b/pkg/dotc1z/engine/pebble/adapter_sync_meta.go @@ -115,6 +115,7 @@ func syncRunRecordToExported(r *v3.SyncRunRecord) *c1zstore.SyncRun { ParentSyncID: r.GetParentSyncId(), LinkedSyncID: r.GetLinkedSyncId(), SupportsDiff: r.GetSupportsDiff(), + Compacted: r.GetCompacted(), } if t := r.GetStartedAt(); t != nil { tt := t.AsTime() @@ -149,6 +150,7 @@ func (a *Adapter) sortedSyncRuns(ctx context.Context) ([]c1zstore.SyncRun, error ParentSyncID: r.GetParentSyncId(), SupportsDiff: r.GetSupportsDiff(), LinkedSyncID: r.GetLinkedSyncId(), + Compacted: r.GetCompacted(), } if t := r.GetStartedAt(); t != nil { tt := t.AsTime() diff --git a/pkg/dotc1z/engine/pebble/bulk_import.go b/pkg/dotc1z/engine/pebble/bulk_import.go index ce280cebb..570135f07 100644 --- a/pkg/dotc1z/engine/pebble/bulk_import.go +++ b/pkg/dotc1z/engine/pebble/bulk_import.go @@ -57,6 +57,7 @@ const bulkSpillBufferSize = 1 << 20 var grantIndexFamilies = []byte{ idxGrantByPrincipal, idxGrantByNeedsExpansion, + idxGrantBySourceScope, } // bulkSSTWriter builds one SST file for a single disjoint key bucket. @@ -695,6 +696,7 @@ func (b *BulkSyncImport) Finish(ctx context.Context) error { // paths so overwrites of imported identities clean up index entries. _ = b.e.takeFreshGrantsEmpty() _ = b.e.takeFreshResourcesEmpty() + _ = b.e.takeFreshEntitlementsEmpty() return nil }) if err != nil { diff --git a/pkg/dotc1z/engine/pebble/cleanup.go b/pkg/dotc1z/engine/pebble/cleanup.go index 66f30a06a..13b374699 100644 --- a/pkg/dotc1z/engine/pebble/cleanup.go +++ b/pkg/dotc1z/engine/pebble/cleanup.go @@ -37,6 +37,10 @@ func scopedRanges() [][2][]byte { {GrantByNeedsExpansionLowerBound(), GrantByNeedsExpansionUpperBound()}, {GrantByEntPrincHashLowerBound(), GrantByEntPrincHashUpperBound()}, {DigestLowerBound(), DigestUpperBound()}, + {GrantBySourceScopeLowerBound(), GrantBySourceScopeUpperBound()}, + {EntitlementBySourceScopeLowerBound(), EntitlementBySourceScopeUpperBound()}, + {ResourceBySourceScopeLowerBound(), ResourceBySourceScopeUpperBound()}, + {SourceCacheEntryLowerBound(), SourceCacheEntryUpperBound()}, {encodeAssetPrefix(), upperBoundOf(encodeAssetPrefix())}, // Stats sidecar — single key; the half-open range shape // contains exactly that one key. @@ -91,6 +95,19 @@ func (e *Engine) ResetForNewSync(ctx context.Context) error { // sync; the wipe is the first step of leaving the sealed state. The // engine stays sealed until MarkFreshSync unseals it right after. return e.withWriteAllowSealed(func() error { + // Flush first so the excises apply DIRECTLY to the manifest. + // Pebble downgrades an excise whose span overlaps unflushed + // memtable data to a flushable ingest — queued behind the + // memtable, taking effect only at the next flush — which defeats + // this method's immediate-reclaim contract (a short replacement + // sync could hard-link the prior sync's dead SSTs into the saved + // envelope). The sync-run record ({v3, typeSyncRun}) lives INSIDE + // the excised record span and EndSync stamps it AFTER its + // durability flush, so a just-ended sync always has exactly that + // key unflushed here. + if err := e.db.Flush(); err != nil { + return fmt.Errorf("ResetForNewSync: pre-excise flush: %w", err) + } for _, span := range spans { if err := e.db.Excise(ctx, span); err != nil { return fmt.Errorf("ResetForNewSync: excise [%x, %x): %w", span.Start, span.End, err) @@ -109,6 +126,11 @@ func (e *Engine) ResetForNewSync(ctx context.Context) error { return err } } + // Ingestion-fact markers live in engine-meta, outside the excised + // record span; the new sync must not inherit the old sync's facts. + if err := e.clearIngestFactMarkers(); err != nil { + return fmt.Errorf("ResetForNewSync: clear ingest fact markers: %w", err) + } e.noteEntitlementKeyspaceWrite() return nil }) diff --git a/pkg/dotc1z/engine/pebble/end_sync_flush_failure_test.go b/pkg/dotc1z/engine/pebble/end_sync_flush_failure_test.go new file mode 100644 index 000000000..1eadae948 --- /dev/null +++ b/pkg/dotc1z/engine/pebble/end_sync_flush_failure_test.go @@ -0,0 +1,101 @@ +package pebble + +// Pins the seal-failure contract: EndSync's finalize stamps ended_at on +// the sync-run record, and the durability flush (EndFreshSync) turns the +// sync's NoSync writes into on-disk state. If the flush FAILS, the sync +// must remain visibly UNFINISHED — LatestUnfinishedSyncRecord skips any +// record with ended_at, so a premature stamp costs resumability (the +// next StartOrResumeSync starts a new sync instead of resuming) and, +// after a crash, can surface a partially-durable artifact as a FINISHED +// sync to replay/uplift consumers. + +import ( + "context" + "errors" + "testing" + + "github.com/stretchr/testify/require" + + v3 "github.com/conductorone/baton-sdk/pb/c1/storage/v3" + "github.com/conductorone/baton-sdk/pkg/connectorstore" +) + +func TestEndSyncFlushFailureKeepsSyncResumable(t *testing.T) { + ctx := context.Background() + a := newAdapter(t) + syncID, err := a.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + e := a.PebbleEngine() + require.NoError(t, a.PutGrants(ctx, mkV2Grant("g1", "group:g1:member", "user", "alice"))) + + injected := errors.New("injected flush failure at the durability boundary") + e.testEndSyncFlushHook = func() error { return injected } + err = a.EndSync(ctx) + require.ErrorIs(t, err, injected, "the failed flush must fail EndSync") + e.testEndSyncFlushHook = nil + + // THE CONTRACT: a sync whose durability flush failed is NOT finished. + // It must still be discoverable as unfinished (resumable), and must + // not carry ended_at (a stamped record with maybe-undurable data is + // a finished-looking lie). + unfinished, err := e.LatestUnfinishedSyncRecord(ctx, nil) + require.NoError(t, err) + require.NotNil(t, unfinished, + "a failed seal must leave the sync resumable; a premature ended_at stamp orphans the work") + require.Equal(t, syncID, unfinished.GetSyncId()) + + rec, err := e.GetSyncRunRecord(ctx, syncID) + require.NoError(t, err) + require.Nil(t, rec.GetEndedAt(), + "ended_at must only be stamped after the durability flush succeeds") + + // Retry converges: the second EndSync finishes the sync for real. + require.NoError(t, a.EndSync(ctx)) + rec, err = e.GetSyncRunRecord(ctx, syncID) + require.NoError(t, err) + require.NotNil(t, rec.GetEndedAt(), "the retried seal must stamp the sync finished") + unfinished, err = e.LatestUnfinishedSyncRecord(ctx, func(v3.SyncType) bool { return true }) + require.NoError(t, err) + require.Nil(t, unfinished, "no unfinished run may remain after a successful seal") +} + +// The stamp-commit analog of the flush test above (H1 residual): the +// durability flush SUCCEEDS, then the ended_at stamp's commit fails. +// endSyncFinalize has already mutated the in-memory record +// (existing.SetEndedAt) by then — that stamp must leak nowhere, and a +// retried EndSync must converge. State assertions on the engine live in +// the obligations harness row (endsync-stamp-commit-failure); this test +// owns the adapter-level retry contract. +func TestEndSyncStampFailureKeepsSyncResumable(t *testing.T) { + ctx := context.Background() + a := newAdapter(t) + syncID, err := a.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + e := a.PebbleEngine() + require.NoError(t, a.PutGrants(ctx, mkV2Grant("g1", "group:g1:member", "user", "alice"))) + + injected := errors.New("injected stamp-commit failure after the durability flush") + e.testEndSyncStampHook = func() error { return injected } + err = a.EndSync(ctx) + require.ErrorIs(t, err, injected, "the failed stamp commit must fail EndSync") + e.testEndSyncStampHook = nil + + rec, err := e.GetSyncRunRecord(ctx, syncID) + require.NoError(t, err) + require.Nil(t, rec.GetEndedAt(), + "the in-memory stamp must not reach the store when its commit fails") + unfinished, err := e.LatestUnfinishedSyncRecord(ctx, nil) + require.NoError(t, err) + require.NotNil(t, unfinished, "a failed stamp commit must leave the sync resumable") + require.Equal(t, syncID, unfinished.GetSyncId()) + + // Retry converges: the second EndSync reloads the (unstamped) stored + // record and finishes the sync for real. + require.NoError(t, a.EndSync(ctx)) + rec, err = e.GetSyncRunRecord(ctx, syncID) + require.NoError(t, err) + require.NotNil(t, rec.GetEndedAt(), "the retried seal must stamp the sync finished") + unfinished, err = e.LatestUnfinishedSyncRecord(ctx, func(v3.SyncType) bool { return true }) + require.NoError(t, err) + require.Nil(t, unfinished, "no unfinished run may remain after a successful seal") +} diff --git a/pkg/dotc1z/engine/pebble/engine.go b/pkg/dotc1z/engine/pebble/engine.go index 166104e0e..a0997a25d 100644 --- a/pkg/dotc1z/engine/pebble/engine.go +++ b/pkg/dotc1z/engine/pebble/engine.go @@ -46,7 +46,7 @@ type Engine struct { // can skip the read-before-write index-cleanup path because this // sync_id is guaranteed to be empty. freshSync bool - // freshGrantsEmpty / freshResourcesEmpty + // freshGrantsEmpty / freshResourcesEmpty / freshEntitlementsEmpty // are one-shot bits guarded by currentSyncMu. MarkFreshSync sets // each to true; the first PutXxxRecords call of the fresh sync // reads the value via takeFreshXxxEmpty() which returns it and @@ -54,8 +54,9 @@ type Engine struct { // call only" — subsequent calls in the same fresh sync must // still read-before-write to clean up cross-call duplicate index // entries. - freshGrantsEmpty bool - freshResourcesEmpty bool + freshGrantsEmpty bool + freshResourcesEmpty bool + freshEntitlementsEmpty bool // writeWG tracks in-flight writes. Incremented at the start of // every Writer method, decremented in defer. @@ -64,6 +65,16 @@ type Engine struct { closing atomic.Bool // strict write-barrier flag, read on every Writer call closeMu sync.Mutex + // mutated latches when ANY write closure runs (see + // withWriteAllowSealed) — the engine-derived backstop for the + // store-level dirty bit. The pebbleStore wrappers mark dirty + // per-method by convention, and every missed wrapper is a lost + // envelope save (the replay pre-clear was one). Latching here is + // conservative in the safe direction: a spurious save costs seconds, + // a skipped one loses the sync. Set even when the closure errors — + // chunked paths may have committed batches before failing. + mutated atomic.Bool + // computedStats holds caller-computed stats records stashed via // StashComputedSyncStats, keyed by sync_id. PersistSyncStats pops // and persists the stashed record instead of re-scanning the @@ -85,6 +96,33 @@ type Engine struct { // as one sorted SST at EndSync — see BuildDeferredGrantIndexes). deferredIdxPending atomic.Bool + // testEndSyncFlushHook, when non-nil, runs in endSyncFinalize + // immediately before the EndFreshSync durability flush — the + // in-process analog of the flush failing. Tests use it to pin the + // seal-failure contract: a sync whose finalize failed must remain + // UNFINISHED (no ended_at), or a resume would start a new sync and a + // crash could surface a partially-durable artifact as finished. + testEndSyncFlushHook func() error + + // testEndSyncStampHook, when non-nil, runs in endSyncFinalize after + // the in-memory ended_at stamp but before PutSyncRunRecord commits + // it — the in-process analog of the stamp's commit failing (H1 + // residual). Tests use it to pin that a failed stamp commit leaks + // nothing: the stored record stays unstamped, the sync stays + // discoverable as unfinished, and a retried EndSync converges. The + // stamped in-memory record is a local, freshly-unmarshalled copy + // owned by the EndSync call (GetSyncRunRecord never returns a shared + // or cached record), so the mutation is unreachable once the error + // returns; the hook exists to keep that true against future caching. + testEndSyncStampHook func() error + + // testDropChunkHook, when non-nil, runs after every chunked + // dangling-reference drop batch commit (ingest_repair.go). Test-only: + // deterministic crash injection at the exact seam between a committed + // chunk and the drop's completion, pinning the atomicity contract of + // stageSourceCacheScopeInvalidationLocked. + testDropChunkHook func() error + // grantDigestsPresent reports whether the digest keyspace holds any // nodes — i.e. whether a grant mutation must invalidate the touched // entitlement's digest + hash-index ranges @@ -114,6 +152,12 @@ type Engine struct { testDigestBuildHook func(stage string) error testDigestNodeFlushBytes int + // externalMatchFact is the monotone existence bit for "this sync + // holds at least one ExternalResourceMatch*-annotated grant" — a + // dense ingestion fact tracked without an index (see + // ingest_facts.go). Durable twin: encodeExternalMatchFactKey. + externalMatchFact atomic.Bool + // synthLayer is the open wave-scoped layer session, if any (see // BeginSynthesizedGrantLayer). Single producer: the expansion driver // opens/adds/finishes sessions strictly sequentially. synthLayerMu @@ -271,6 +315,12 @@ func Open(ctx context.Context, dir string, opts ...Option) (*Engine, error) { _ = e.Close() return nil, err } + // Restore durable ingestion-fact markers (see ingest_facts.go) — + // same rationale as the deferred-index marker above. + if err := e.restoreIngestFactMarkers(); err != nil { + _ = e.Close() + return nil, err + } // Run secondary-index migrations before returning. Migrations // are skipped for read-only opens (the on-disk file is // immutable, so we'd error out trying to backfill). @@ -342,6 +392,7 @@ func (e *Engine) SetCurrentSync(syncID string) error { e.freshSync = false e.freshGrantsEmpty = false e.freshResourcesEmpty = false + e.freshEntitlementsEmpty = false e.currentSyncMu.Unlock() // Binding a sync means more writes are coming; leave the sealed state // and resume compactions so L0 keeps draining (see seal). @@ -420,6 +471,7 @@ func (e *Engine) MarkFreshSync(syncID string) error { e.freshSync = true e.freshGrantsEmpty = true e.freshResourcesEmpty = true + e.freshEntitlementsEmpty = true e.currentSyncMu.Unlock() // A fresh sync writes heavily; leave the sealed state and resume // compactions so L0 keeps draining (see seal). @@ -437,6 +489,7 @@ func (e *Engine) clearCurrentSync() { e.freshSync = false e.freshGrantsEmpty = false e.freshResourcesEmpty = false + e.freshEntitlementsEmpty = false e.currentSyncMu.Unlock() } @@ -481,6 +534,16 @@ func (e *Engine) takeFreshResourcesEmpty() bool { return true } +func (e *Engine) takeFreshEntitlementsEmpty() bool { + e.currentSyncMu.Lock() + defer e.currentSyncMu.Unlock() + if !e.freshEntitlementsEmpty { + return false + } + e.freshEntitlementsEmpty = false + return true +} + // EndFreshSync clears the fresh-sync flag and flushes the memtable // + fsyncs the WAL so the data written during the sync is on disk // before the caller returns. Called by Adapter.EndSync. @@ -489,6 +552,19 @@ func (e *Engine) takeFreshResourcesEmpty() bool { // closing check and writeWG: Close tears e.db down after writeWG.Wait, // and a bare-mutex EndFreshSync racing Close would flush a nil db. func (e *Engine) EndFreshSync(ctx context.Context) error { + if err := e.FlushForEndSync(ctx); err != nil { + return err + } + e.clearCurrentSync() + return nil +} + +// FlushForEndSync is EndFreshSync's durability flush WITHOUT the +// current-sync unbind: endSyncFinalize flushes first, stamps ended_at +// second, and unbinds last — so a failed stamp leaves the sync bound +// (and fresh) for an in-process retry, and a failed flush never follows +// a committed stamp (see the ordering contract in endSyncFinalize). +func (e *Engine) FlushForEndSync(ctx context.Context) error { // AllowSealed: this is the last step of EndSync's sealed finalize // window (see Adapter.EndSync). return e.withWriteAllowSealed(func() error { @@ -496,7 +572,6 @@ func (e *Engine) EndFreshSync(ctx context.Context) error { wasFresh := e.freshSync e.currentSyncMu.RUnlock() if !wasFresh { - e.clearCurrentSync() return nil } // Flush the memtable (turns NoSync-buffered writes into on-disk @@ -507,7 +582,6 @@ func (e *Engine) EndFreshSync(ctx context.Context) error { if err := e.db.LogData(nil, pebble.Sync); err != nil { return fmt.Errorf("EndFreshSync: fsync WAL: %w", err) } - e.clearCurrentSync() return nil }) } @@ -607,9 +681,22 @@ func (e *Engine) withWriteAllowSealed(fn func() error) error { } e.writeMu.Lock() defer e.writeMu.Unlock() + // Latch BEFORE the closure runs, not on success: a failing closure + // may have committed chunks (see the streaming drops), and a lost + // envelope save is the unrecoverable direction. + e.mutated.Store(true) return fn() } +// Mutated reports whether any write closure has run since Open — the +// engine-derived dirty signal. Store-level Close saves the envelope when +// EITHER its own per-method dirty bit or this latch is set, so an engine +// mutation path missing a store wrapper can no longer silently skip the +// save. +func (e *Engine) Mutated() bool { + return e.mutated.Load() +} + func (e *Engine) Save(ctx context.Context, dest string) error { return errors.New("pebble engine: Save requires the dotc1z.Save shim (envelope write); use CheckpointTo for direct directory access") } diff --git a/pkg/dotc1z/engine/pebble/entitlements.go b/pkg/dotc1z/engine/pebble/entitlements.go index 2182c3a2d..6ac0e1cd8 100644 --- a/pkg/dotc1z/engine/pebble/entitlements.go +++ b/pkg/dotc1z/engine/pebble/entitlements.go @@ -20,9 +20,16 @@ func (e *Engine) PutEntitlementRecord(ctx context.Context, r *v3.EntitlementReco // PutEntitlementRecords writes N entitlements by structured primary key. // The identity key is a pure function of the record (it contains the raw -// external id), so overwrites are idempotent and no read-before-write or -// index cleanup is needed; a within-call dedup pre-pass keeps last-wins -// semantics for same-identity duplicates in one batch. +// external id), so overwrites of the PRIMARY row are idempotent; a +// within-call dedup pre-pass keeps last-wins semantics for same-identity +// duplicates in one batch. The by_source_scope index (the only +// entitlement secondary index) is NOT idempotent across scope changes, so +// a read-before-write Get cleans a differing old stamp — mirroring +// PutGrantRecords/PutResourceRecords — gated off only on the provably +// empty first call of a fresh sync. Without this, a same-identity rewrite +// under a new scope leaves a stale index entry AND the old scope's +// manifest entry survives, so the NEXT sync's replay of the old scope +// copies zero rows where the manifest promised content. func (e *Engine) PutEntitlementRecords(ctx context.Context, records ...*v3.EntitlementRecord) error { if len(records) == 0 { return nil @@ -35,6 +42,7 @@ func (e *Engine) PutEntitlementRecords(ctx context.Context, records ...*v3.Entit defer priBatch.Close() fresh := e.IsFreshSync() + skipGet := e.takeFreshEntitlementsEmpty() type dedupKey struct { id entitlementIdentity @@ -72,9 +80,35 @@ func (e *Engine) PutEntitlementRecords(ctx context.Context, records ...*v3.Entit if err != nil { return err } + newScope := r.GetSourceScopeKey() + if !skipGet { + oldVal, closer, getErr := e.db.Get(key) + switch { + case getErr == nil: + oldScope, scanErr := scanEntitlementSourceScopeRaw(oldVal) + closer.Close() + if scanErr != nil { + return scanErr + } + if oldScope != "" && oldScope != newScope { + if err := priBatch.Delete(encodeEntitlementBySourceScopeIndexKey(oldScope, id), nil); err != nil { + return err + } + } + case errors.Is(getErr, pebble.ErrNotFound): + // no prior record — write unconditionally + default: + return fmt.Errorf("PutEntitlementRecords: get old: %w", getErr) + } + } if err := priBatch.Set(key, val, nil); err != nil { return err } + if newScope != "" { + if err := priBatch.Set(encodeEntitlementBySourceScopeIndexKey(newScope, id), nil, nil); err != nil { + return err + } + } } opts := writeOpts(e.opts.durability) if fresh { @@ -108,10 +142,11 @@ func (e *Engine) GetEntitlementRecord(ctx context.Context, externalID string) (* } // DeleteEntitlementRecord deletes by raw public id. A missing id is a -// no-op; an ambiguous id is an error (a lossy string must never guess a -// delete). -func (e *Engine) DeleteEntitlementRecord(ctx context.Context, externalID string) error { - return e.withWrite(func() error { +// no-op (returns false); an ambiguous id is an error (a lossy string +// must never guess a delete). Returns whether a row was deleted. +func (e *Engine) DeleteEntitlementRecord(ctx context.Context, externalID string) (bool, error) { + deleted := false + err := e.withWrite(func() error { id, err := e.resolveEntitlementIdentityByExternalID(ctx, externalID) if err != nil { if errors.Is(err, pebble.ErrNotFound) { @@ -122,6 +157,22 @@ func (e *Engine) DeleteEntitlementRecord(ctx context.Context, externalID string) key := encodeEntitlementIdentityKey(id) batch := e.db.NewBatch() defer batch.Close() + // Clean up the source-scope index entry (the only entitlement + // secondary index) so a replayed scope can't resurrect the row. + if oldVal, closer, getErr := e.db.Get(key); getErr == nil { + oldScope, scanErr := scanEntitlementSourceScopeRaw(oldVal) + closer.Close() + if scanErr != nil { + return scanErr + } + if oldScope != "" { + if err := batch.Delete(encodeEntitlementBySourceScopeIndexKey(oldScope, id), nil); err != nil { + return err + } + } + } else if !errors.Is(getErr, pebble.ErrNotFound) { + return getErr + } if err := batch.Delete(key, nil); err != nil { return err } @@ -129,8 +180,10 @@ func (e *Engine) DeleteEntitlementRecord(ctx context.Context, externalID string) return err } e.noteEntitlementKeyspaceWrite() + deleted = true return nil }) + return deleted, err } func (e *Engine) IterateEntitlements(ctx context.Context, yield func(*v3.EntitlementRecord) bool) error { diff --git a/pkg/dotc1z/engine/pebble/grants.go b/pkg/dotc1z/engine/pebble/grants.go index 1abf1d3d8..a194738b4 100644 --- a/pkg/dotc1z/engine/pebble/grants.go +++ b/pkg/dotc1z/engine/pebble/grants.go @@ -114,6 +114,11 @@ func (e *Engine) PutGrantRecords(ctx context.Context, records ...*v3.GrantRecord if r == nil { continue } + // Dense ingestion facts (one atomic load once armed; the + // annotation walk only runs until the sync's first hit). + if err := e.noteGrantRecordFacts(r); err != nil { + return err + } id, err := grantIdentityFromRecord(r) if err != nil { return err @@ -263,6 +268,12 @@ func (e *Engine) PutExpandedGrantRecords(ctx context.Context, records []*v3.Gran r.SetExpansion(prior.GetExpansion()) r.SetNeedsExpansion(prior.GetNeedsExpansion()) r.SetDiscoveredAt(prior.GetDiscoveredAt()) + // Preserve the source-cache scope stamp exactly like the + // expansion side-state: the expander rewrites existing + // direct grants to bake in Sources, and clobbering the + // stamp here would silently drop every expander-touched + // grant from the next sync's replay of its scope. + r.SetSourceScopeKey(prior.GetSourceScopeKey()) var err error idxScratch, err = e.deleteGrantIndexesScratch(idxBatch, ext, oldVal, idxScratch) if err != nil { @@ -746,14 +757,17 @@ func (e *Engine) putSynthesizedGrantContributionsBatch(ctx context.Context, reco if err != nil { return err } - // sources (field 9) is the record's highest field, so appending - // it after the base marshal matches the deterministic byte order. + // sources (field 9) is the highest field this record carries, so + // appending it after the base marshal matches the deterministic + // byte order. (source_scope_key is field 10, but synthesized + // grants never carry a scope stamp — fillSynthGrantRecord leaves + // it unset — so field 9 stays last on the wire.) val, srcScratch = appendGrantSourcesWire(val, srcScratch, rec.sources) valScratch = val if err := priBatch.Set(keyScratch, val, nil); err != nil { return err } - idxScratch, err = e.writeGrantIndexesForIdentityScratch(idxBatch, rec.id, false, idxScratch) + idxScratch, err = e.writeGrantIndexesForIdentityScratch(idxBatch, rec.id, false, "", idxScratch) if err != nil { return err } @@ -835,6 +849,10 @@ func (e *Engine) UnsafePutUniqueGrantRecords(ctx context.Context, records ...*v3 if r == nil { continue } + // No noteGrantRecordFacts here (unlike PutGrantRecords + // and PutGrantRecordsIfNewer): this is the trusted-import + // path — nothing in a bulk-import lifecycle consumes the + // external-match fact, so the hot loop skips the walk. val, err := marshalRecord(r) if err != nil { errMu.Lock() @@ -986,11 +1004,14 @@ func grantIndexKeys(r *v3.GrantRecord) [][]byte { if err != nil { return nil } - keys := make([][]byte, 0, 2) + keys := make([][]byte, 0, 3) keys = append(keys, encodeGrantByPrincipalIdentityIndexKey(id)) if r.GetNeedsExpansion() { keys = append(keys, encodeGrantByNeedsExpansionIdentityIndexKey(id)) } + if sh := r.GetSourceScopeKey(); sh != "" { + keys = append(keys, encodeGrantBySourceScopeIndexKey(sh, id)) + } return keys } @@ -1038,7 +1059,7 @@ func (e *Engine) writeGrantIndexesScratch(batch *pebble.Batch, r *v3.GrantRecord if err != nil { return scratch, err } - return e.writeGrantIndexesForIdentityScratch(batch, id, r.GetNeedsExpansion(), scratch) + return e.writeGrantIndexesForIdentityScratch(batch, id, r.GetNeedsExpansion(), r.GetSourceScopeKey(), scratch) } // markDeferredIdxPending arms the deferred by_principal rebuild, durably. @@ -1081,7 +1102,10 @@ func (e *Engine) clearDeferredIdxPending() error { // for one grant. by_principal is never written inline: it is scattered // relative to the entitlement-first write order, so it is always rebuilt as // one sorted SST at EndSync (deferredIdxPending → BuildDeferredGrantIndexes). -func (e *Engine) writeGrantIndexesForIdentityScratch(batch *pebble.Batch, id grantIdentity, needsExpansion bool, scratch []byte) ([]byte, error) { +// by_needs_expansion and by_source_scope are written inline: both share the +// entitlement-first ordering of the primary keyspace, so their writes stay +// sorted. +func (e *Engine) writeGrantIndexesForIdentityScratch(batch *pebble.Batch, id grantIdentity, needsExpansion bool, sourceScopeKey string, scratch []byte) ([]byte, error) { if err := e.markDeferredIdxPending(); err != nil { return scratch, err } @@ -1096,6 +1120,12 @@ func (e *Engine) writeGrantIndexesForIdentityScratch(batch *pebble.Batch, id gra if err := e.stageGrantDigestInvalidation(batch, id.entitlement); err != nil { return scratch, err } + if sourceScopeKey != "" { + scratch = appendGrantBySourceScopeIndexKey(scratch[:0], sourceScopeKey, id) + if err := batch.Set(scratch, nil, nil); err != nil { + return scratch, err + } + } return scratch, nil } @@ -1108,7 +1138,7 @@ func (e *Engine) writeGrantIndexesForIdentityScratch(batch *pebble.Batch, id gra // which also clears any stale entries an overwrite would have left. Returns // the (possibly grown) scratch buffer. func (e *Engine) deleteGrantIndexesScratch(batch *pebble.Batch, externalID string, value, scratch []byte) ([]byte, error) { - entRT, entRID, entID, principalRT, principalID, _, err := scanGrantIndexFieldsRaw(value) + entRT, entRID, entID, principalRT, principalID, _, sourceScopeKey, err := scanGrantIndexFieldsRaw(value) if err != nil { return scratch, err } @@ -1129,6 +1159,12 @@ func (e *Engine) deleteGrantIndexesScratch(batch *pebble.Batch, externalID strin if err := e.stageGrantDigestInvalidation(batch, id.entitlement); err != nil { return scratch, err } + if sourceScopeKey != "" { + scratch = appendGrantBySourceScopeIndexKey(scratch[:0], sourceScopeKey, id) + if err := batch.Delete(scratch, nil); err != nil { + return scratch, err + } + } return scratch, nil } diff --git a/pkg/dotc1z/engine/pebble/grants_synth_encode.go b/pkg/dotc1z/engine/pebble/grants_synth_encode.go index b398712fd..19c67eec5 100644 --- a/pkg/dotc1z/engine/pebble/grants_synth_encode.go +++ b/pkg/dotc1z/engine/pebble/grants_synth_encode.go @@ -30,11 +30,19 @@ var expandedGrantImmutableAnnotations = []*anypb.Any{expandedGrantImmutableAnnot // appendSynthGrantExternalIDWire) and sources (field 9, appended by // appendGrantSourcesWire). Expansion/NeedsExpansion stay zero for // synthesized grants. +// +// SourceScopeKey is cleared unconditionally, for two reasons: synthesized +// (expander-derived) grants are never part of a source-cache scope, and — +// load-bearing for appendGrantSourcesWire — an empty proto3 scalar emits no +// bytes, which keeps sources (field 9) the highest field on the wire so the +// hand encoder's append-after-base-marshal stays canonical. See +// TestGrantSourcesWireSchemaPin. func fillSynthGrantRecord(r *v3.GrantRecord, rec *synthesizedGrantRecord, now *timestamppb.Timestamp) { r.SetEntitlement(rec.entitlement) r.SetPrincipal(rec.principal) r.SetAnnotations(expandedGrantImmutableAnnotations) r.SetDiscoveredAt(now) + r.SetSourceScopeKey("") } // grantExternalIDFieldTag is the wire tag for GrantRecord.external_id (2). diff --git a/pkg/dotc1z/engine/pebble/grants_synth_encode_test.go b/pkg/dotc1z/engine/pebble/grants_synth_encode_test.go index a128d1746..3f4f0b9a3 100644 --- a/pkg/dotc1z/engine/pebble/grants_synth_encode_test.go +++ b/pkg/dotc1z/engine/pebble/grants_synth_encode_test.go @@ -45,8 +45,19 @@ func TestGrantSourcesWireSchemaPin(t *testing.T) { maxNum = n } } - require.Equal(t, protoreflect.FieldNumber(9), maxNum, - "GrantRecord gained a field numbered above sources(9): appendGrantSourcesWire appends sources last and would emit non-canonical field order") + // source_scope_key (10) sorts above sources (9), which would break the + // append-after-base-marshal canonical order IF it were ever populated on + // a synthesized record. It never is: fillSynthGrantRecord clears it + // unconditionally, and an empty proto3 scalar emits no bytes, so sources + // stays the highest field on the wire. Any field beyond 10 (or a + // non-scalar 10) requires revisiting appendGrantSourcesWire. + require.Equal(t, protoreflect.FieldNumber(10), maxNum, + "GrantRecord gained a field beyond source_scope_key(10): decide whether appendGrantSourcesWire's append-last invariant still holds") + ssh := fields.ByNumber(10) + require.NotNil(t, ssh) + require.Equal(t, protoreflect.Name("source_scope_key"), ssh.Name(), "field 10 is no longer source_scope_key") + require.Equal(t, protoreflect.StringKind, ssh.Kind(), + "source_scope_key is no longer a string scalar; the empty-value-emits-nothing guarantee appendGrantSourcesWire relies on may not hold") src := fields.ByName("sources") require.NotNil(t, src, "GrantRecord.sources missing") @@ -238,6 +249,12 @@ func TestFillSynthGrantRecordSchemaPin(t *testing.T) { "7:needs_expansion:bool", "8:annotations:message", "9:sources:message", + // source_scope_key is part of the synthesized write path only as an + // unconditional CLEAR (expander-derived grants are never part of a + // source-cache scope); fillSynthGrantRecord sets it to "" to honor + // the reused-struct invariant and appendGrantSourcesWire's + // highest-populated-field assumption. + "10:source_scope_key:string", }, protoFields, "GrantRecord shape changed: decide whether fillSynthGrantRecord must set the new/changed field") recType := reflect.TypeOf(synthesizedGrantRecord{}) diff --git a/pkg/dotc1z/engine/pebble/id_index_migration.go b/pkg/dotc1z/engine/pebble/id_index_migration.go index 019aa7150..c592f89fd 100644 --- a/pkg/dotc1z/engine/pebble/id_index_migration.go +++ b/pkg/dotc1z/engine/pebble/id_index_migration.go @@ -213,7 +213,7 @@ func (e *Engine) emitStructuredGrantMigration(ctx context.Context, primary *spil lastLog = now } } - entRT, entRID, entID, principalRT, principalID, _, err := scanGrantIndexFieldsRaw(iter.Value()) + entRT, entRID, entID, principalRT, principalID, _, _, err := scanGrantIndexFieldsRaw(iter.Value()) if err != nil { return rows, fmt.Errorf("id-index migration: scan grant: %w", err) } diff --git a/pkg/dotc1z/engine/pebble/if_newer.go b/pkg/dotc1z/engine/pebble/if_newer.go index 1b20dcc81..37466e376 100644 --- a/pkg/dotc1z/engine/pebble/if_newer.go +++ b/pkg/dotc1z/engine/pebble/if_newer.go @@ -51,6 +51,11 @@ func (e *Engine) PutGrantRecordsIfNewer(ctx context.Context, records ...*v3.Gran if r == nil { continue } + // Dense ingestion facts (see PutGrantRecords): partial-sync + // upserts can carry ExternalResourceMatch* annotations too. + if err := e.noteGrantRecordFacts(r); err != nil { + return err + } id, err := grantIdentityFromRecord(r) if err != nil { return err @@ -188,7 +193,16 @@ func (e *Engine) PutEntitlementRecordsIfNewer(ctx context.Context, records ...*v closer.Close() continue } + oldScope, scanErr := scanEntitlementSourceScopeRaw(oldVal) closer.Close() + if scanErr != nil { + return scanErr + } + if oldScope != "" && oldScope != r.GetSourceScopeKey() { + if err := batch.Delete(encodeEntitlementBySourceScopeIndexKey(oldScope, id), nil); err != nil { + return err + } + } case errors.Is(getErr, pebble.ErrNotFound): default: return fmt.Errorf("PutEntitlementRecordsIfNewer: get: %w", getErr) @@ -200,6 +214,11 @@ func (e *Engine) PutEntitlementRecordsIfNewer(ctx context.Context, records ...*v if err := batch.Set(key, val, nil); err != nil { return err } + if sh := r.GetSourceScopeKey(); sh != "" { + if err := batch.Set(encodeEntitlementBySourceScopeIndexKey(sh, id), nil, nil); err != nil { + return err + } + } written++ } if written == 0 { diff --git a/pkg/dotc1z/engine/pebble/ingest_facts.go b/pkg/dotc1z/engine/pebble/ingest_facts.go new file mode 100644 index 000000000..28c00630e --- /dev/null +++ b/pkg/dotc1z/engine/pebble/ingest_facts.go @@ -0,0 +1,359 @@ +package pebble + +// Ingestion facts: engine-maintained evidence about rows, consumed by the +// syncer's ingestion invariants (see +// docs/tasks/source-cache-ingestion-invariants.md). +// +// Facts are chosen by density. Sparse facts get partial indexes +// (by_needs_expansion). Dense facts — annotations that the biggest +// connectors stamp on ~100% of grants — get a MONOTONE existence bit +// (this file) or no structure at all (read from row values on demand). +// A dense inline secondary index is deliberately rejected: it is the +// cost class the engine already refuses (the reason by_principal is a +// deferred build). +// +// The existence bit mirrors the deferred-index pending marker exactly: +// an atomic bool for in-process reads, a durable engine-meta key so a +// crash/resume keeps the fact, restored at Open, cleared by +// ResetForNewSync. Monotonicity (set-once, never unset within a sync) +// makes the bit order-independent under parallel workers and mixed +// stream/replay ingestion. + +import ( + "context" + "errors" + "fmt" + "strings" + + "github.com/cockroachdb/pebble/v2" + "google.golang.org/protobuf/types/known/anypb" + + v3 "github.com/conductorone/baton-sdk/pb/c1/storage/v3" + "github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/codec" +) + +// encodeExternalMatchFactKey is the durable engine-meta marker for "this +// sync wrote at least one grant carrying an ExternalResourceMatch* +// annotation". Engine-meta (typeEngineMeta) survives ResetForNewSync's +// record-span excise, so the reset path clears it explicitly. +func encodeExternalMatchFactKey() []byte { + buf := make([]byte, 0, 2+len("fact_external_match_grants")) + buf = append(buf, versionV3, typeEngineMeta) + return codec.AppendTupleStrings(buf, "fact_external_match_grants") +} + +// markExternalMatchFact records the existence fact, durably. CAS keeps +// the durable write to one per sync, off the per-record hot path: after +// the first hit, callers pay one atomic load. +func (e *Engine) markExternalMatchFact() error { + if !e.externalMatchFact.CompareAndSwap(false, true) { + return nil + } + if err := e.db.Set(encodeExternalMatchFactKey(), nil, pebble.Sync); err != nil { + // Roll back so a retried write re-attempts the durable marker; + // an armed flag without the durable key would diverge from a + // crash/resume of the same sync (same rationale as + // markDeferredIdxPending). + e.externalMatchFact.Store(false) + return fmt.Errorf("arm external-match fact marker: %w", err) + } + return nil +} + +// HasExternalMatchGrants reports the existence fact for the open sync. +func (e *Engine) HasExternalMatchGrants() bool { + return e.externalMatchFact.Load() +} + +// restoreIngestFactMarkers reloads durable fact markers at Open (a prior +// process may have set them before crashing; a finished file carries them +// for read-side consumers). +func (e *Engine) restoreIngestFactMarkers() error { + if _, closer, err := e.db.Get(encodeExternalMatchFactKey()); err == nil { + closer.Close() + e.externalMatchFact.Store(true) + } else if !errors.Is(err, pebble.ErrNotFound) { + return err + } + return nil +} + +// clearIngestFactMarkers drops the durable markers and in-memory bits. +// Called by ResetForNewSync, whose record-span excise does not reach the +// engine-meta keyspace. +func (e *Engine) clearIngestFactMarkers() error { + e.externalMatchFact.Store(false) + return e.db.Delete(encodeExternalMatchFactKey(), pebble.Sync) +} + +// grantRecordHasExternalMatch reports whether a translated GrantRecord's +// annotations carry any ExternalResourceMatch* type. Called on the put +// path only while the fact bit is still unset, so the per-record cost +// vanishes after the first hit. +func grantRecordHasExternalMatch(anns []*anypb.Any) bool { + for _, a := range anns { + url := a.GetTypeUrl() + name := url + if i := strings.LastIndexByte(url, '/'); i >= 0 { + name = url[i+1:] + } + switch name { + case anyTypeExternalResourceMatch, anyTypeExternalResourceMatchAll, anyTypeExternalResourceMatchID: + return true + } + } + return false +} + +// noteGrantRecordFacts inspects one to-be-written grant record for dense +// facts. Cheap by construction: one atomic load when the bit is already +// set; the annotation walk only runs until the sync's first hit. +func (e *Engine) noteGrantRecordFacts(r *v3.GrantRecord) error { + if e.externalMatchFact.Load() { + return nil + } + if grantRecordHasExternalMatch(r.GetAnnotations()) { + return e.markExternalMatchFact() + } + return nil +} + +// ForEachDistinctGrantEntitlementResource visits each distinct +// (entitlement resource_type_id, resource_id) pair present in the grant +// primary keyspace, in key order. The keyspace leads with +// ent_rt | ent_rid, so distinctness is a prefix-skip: one seek per +// distinct resource — O(distinct) seeks, never O(grants). Backs the +// syncer's grant→resource referential invariant. +func (e *Engine) ForEachDistinctGrantEntitlementResource(ctx context.Context, visit func(resourceTypeID, resourceID string) error) error { + prefix := encodeGrantPrefix() + iter, err := e.db.NewIter(&pebble.IterOptions{ + LowerBound: prefix, + UpperBound: upperBoundOf(prefix), + }) + if err != nil { + return err + } + defer iter.Close() + + const headerLen = 3 // versionV3, typeGrant, separator + for valid := iter.First(); valid; { + if err := ctx.Err(); err != nil { + return err + } + key := iter.Key() + if len(key) <= headerLen { + return fmt.Errorf("distinct ent-resource scan: malformed grant key %x", key) + } + tail := key[headerLen:] + rtBytes, next, ok := codec.DecodeTupleStringAlias(tail, 0) + if !ok || next >= len(tail) { + return fmt.Errorf("distinct ent-resource scan: malformed grant key tail %x", key) + } + ridBytes, _, ok := codec.DecodeTupleStringAlias(tail, next+1) + if !ok { + return fmt.Errorf("distinct ent-resource scan: malformed grant key tail %x", key) + } + rt, rid := string(rtBytes), string(ridBytes) + if err := visit(rt, rid); err != nil { + return err + } + // Skip every remaining grant of this entitlement resource. + valid = iter.SeekGE(upperBoundOf(encodeGrantPrimaryEntitlementResourcePrefix(rt, rid))) + } + return iter.Error() +} + +// ForEachDistinctEntitlementResource visits each distinct +// (resource_type_id, resource_id) pair present in the entitlement +// primary keyspace, in key order. Same prefix-skip shape as the grant +// scan: one seek per distinct resource, never O(entitlements). Backs +// the syncer's entitlement→resource referential invariant (I7). +func (e *Engine) ForEachDistinctEntitlementResource(ctx context.Context, visit func(resourceTypeID, resourceID string) error) error { + prefix := encodeEntitlementPrefix() + iter, err := e.db.NewIter(&pebble.IterOptions{ + LowerBound: prefix, + UpperBound: upperBoundOf(prefix), + }) + if err != nil { + return err + } + defer iter.Close() + + const headerLen = 3 // versionV3, typeEntitlement, separator + for valid := iter.First(); valid; { + if err := ctx.Err(); err != nil { + return err + } + key := iter.Key() + if len(key) <= headerLen { + return fmt.Errorf("distinct entitlement-resource scan: malformed entitlement key %x", key) + } + tail := key[headerLen:] + rtBytes, next, ok := codec.DecodeTupleStringAlias(tail, 0) + if !ok || next >= len(tail) { + return fmt.Errorf("distinct entitlement-resource scan: malformed entitlement key tail %x", key) + } + ridBytes, _, ok := codec.DecodeTupleStringAlias(tail, next+1) + if !ok { + return fmt.Errorf("distinct entitlement-resource scan: malformed entitlement key tail %x", key) + } + rt, rid := string(rtBytes), string(ridBytes) + if err := visit(rt, rid); err != nil { + return err + } + valid = iter.SeekGE(upperBoundOf(encodeEntitlementPrimaryResourcePrefix(rt, rid))) + } + return iter.Error() +} + +// ForEachDanglingGrantEntitlement visits each distinct entitlement +// identity referenced by grants for which NO entitlement row exists. +// The grant primary key leads with the full entitlement identity tuple +// (which is byte-identical to the entitlement primary key's tuple), so +// the scan is one seek per distinct entitlement plus one point-Get +// probe each — O(distinct entitlements), never O(grants), and no value +// reads: the identity tuple reconstructs the external id byte-exactly +// (see identity.go). Backs the syncer's grant→entitlement referential +// invariant (I8). +func (e *Engine) ForEachDanglingGrantEntitlement(ctx context.Context, visit func(entitlementID, resourceTypeID, resourceID string) error) error { + prefix := encodeGrantPrefix() + iter, err := e.db.NewIter(&pebble.IterOptions{ + LowerBound: prefix, + UpperBound: upperBoundOf(prefix), + }) + if err != nil { + return err + } + defer iter.Close() + + const headerLen = 3 // versionV3, typeGrant, separator + for valid := iter.First(); valid; { + if err := ctx.Err(); err != nil { + return err + } + key := iter.Key() + if len(key) <= headerLen { + return fmt.Errorf("dangling grant-entitlement scan: malformed grant key %x", key) + } + tail := key[headerLen:] + var comps [4]string + off := 0 + for i := range comps { + b, next, ok := codec.DecodeTupleStringAlias(tail, off) + if !ok || (i < len(comps)-1 && next >= len(tail)) { + return fmt.Errorf("dangling grant-entitlement scan: malformed grant key tail %x", key) + } + comps[i] = string(b) + off = next + 1 + } + id := entitlementIdentity{ + resourceTypeID: comps[0], + resourceID: comps[1], + stripped: comps[2] == idFlagStripped, + tail: comps[3], + } + exists, err := e.hasEntitlementIdentity(id) + if err != nil { + return err + } + if !exists { + if err := visit(id.externalID(), id.resourceTypeID, id.resourceID); err != nil { + return err + } + } + // Skip every remaining grant of this entitlement. + valid = iter.SeekGE(upperBoundOf(encodeGrantPrimaryEntitlementIdentityPrefix(comps[0], comps[1], comps[2], comps[3]))) + } + return iter.Error() +} + +func (e *Engine) hasEntitlementIdentity(id entitlementIdentity) (bool, error) { + _, closer, err := e.db.Get(encodeEntitlementIdentityKey(id)) + if err != nil { + if errors.Is(err, pebble.ErrNotFound) { + return false, nil + } + return false, err + } + closer.Close() + return true, nil +} + +// GrantsForEntitlementAllCarryInsertFact reports whether EVERY grant +// under the given entitlement identity carries the InsertResourceGrants +// annotation — the fail-fast probe for the per-grant I8 exemption: a +// dangling entitlement whose grants all carry the fact is the +// machinery-owned shape (exempt); one non-carrying grant makes it a +// connector-owned dangling reference. Vacuously true with no grants +// (never the case for a dangling entitlement, which is only visited +// because grants reference it). Reads row values, so it is reserved for +// DANGLING referential probes — rare to zero on healthy syncs. +func (e *Engine) GrantsForEntitlementAllCarryInsertFact(ctx context.Context, entitlementID, entResourceTypeID, entResourceID string) (bool, error) { + entID := entitlementIdentityFromParts(entResourceTypeID, entResourceID, entitlementID) + prefix := encodeGrantPrimaryEntitlementIdentityPrefix( + entID.resourceTypeID, entID.resourceID, entID.flagComponent(), entID.tail) + iter, err := e.db.NewIter(&pebble.IterOptions{ + LowerBound: prefix, + UpperBound: upperBoundOf(prefix), + }) + if err != nil { + return false, err + } + defer iter.Close() + for iter.First(); iter.Valid(); iter.Next() { + if err := ctx.Err(); err != nil { + return false, err + } + flags, err := scanGrantSourceCacheFlagsRaw(iter.Value()) + if err != nil { + return false, err + } + if !flags.insertResourceGrants { + return false, nil + } + } + return true, iter.Error() +} + +// GrantsForEntResourceCarryInsertFact reports whether any grant under the +// given entitlement resource carries the InsertResourceGrants annotation. +// Reads row values, so it is reserved for DANGLING referential probes — +// rare to zero on healthy syncs — never the bulk path. +func (e *Engine) GrantsForEntResourceCarryInsertFact(ctx context.Context, resourceTypeID, resourceID string) (bool, error) { + prefix := encodeGrantPrimaryEntitlementResourcePrefix(resourceTypeID, resourceID) + iter, err := e.db.NewIter(&pebble.IterOptions{ + LowerBound: prefix, + UpperBound: upperBoundOf(prefix), + }) + if err != nil { + return false, err + } + defer iter.Close() + for iter.First(); iter.Valid(); iter.Next() { + if err := ctx.Err(); err != nil { + return false, err + } + flags, err := scanGrantSourceCacheFlagsRaw(iter.Value()) + if err != nil { + return false, err + } + if flags.insertResourceGrants { + return true, nil + } + } + return false, iter.Error() +} + +// HasResourceRecord reports whether a resource row exists — the probe +// side of the referential invariant. +func (e *Engine) HasResourceRecord(ctx context.Context, resourceTypeID, resourceID string) (bool, error) { + _, closer, err := e.db.Get(encodeResourceKey(resourceTypeID, resourceID)) + if err != nil { + if errors.Is(err, pebble.ErrNotFound) { + return false, nil + } + return false, err + } + closer.Close() + return true, nil +} diff --git a/pkg/dotc1z/engine/pebble/ingest_repair.go b/pkg/dotc1z/engine/pebble/ingest_repair.go new file mode 100644 index 000000000..6378c9226 --- /dev/null +++ b/pkg/dotc1z/engine/pebble/ingest_repair.go @@ -0,0 +1,775 @@ +package pebble + +// Dangling-reference drops: the engine mechanics behind the syncer's +// lenient-mode referential invariants (I7/I8/I9 in +// pkg/sync/ingest_invariants.go). Connectors ship magic-id construction +// bugs — grants and annotations naming entitlements/resources whose rows +// were never synced — and the platform's ingestion provably discards +// those rows, so the syncer drops them at the post-collection seam and +// reports aggregates. Policy (what to drop, exemptions, strict-mode +// failure) lives in the syncer; these methods are pure mechanics. +// +// All deletes route through the same locked helpers as interactive +// deletes, so secondary indexes and (if already built) digest +// invalidation stay consistent. Populations are small in practice +// (dangling refs are bugs, not steady state), so per-row batches are +// acceptable. + +import ( + "context" + "errors" + "fmt" + + "github.com/cockroachdb/pebble/v2" + "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" + "go.uber.org/zap" + + v3 "github.com/conductorone/baton-sdk/pb/c1/storage/v3" + "github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/codec" +) + +// EnsureGrantIndexes runs the deferred grant-index build NOW if one is +// pending, and retires the pending marker. The by_principal index is +// inline for connector-emitted grants but deferred for expansion/synth +// writes; the dangling-principal scan (I9) needs it complete. This is +// the SAME build EndSync would run, so the marker must clear here or +// EndSync repeats the whole O(grants) scan/build a second time. Rows +// dropped afterward maintain the indexes inline and stage digest +// invalidation, which EndSync's RepairMissingGrantDigests pass heals; +// grant drops also invalidate the stashed deferred grant stats +// (invalidateDeferredGrantStats) so the stats sidecar recounts instead +// of reporting dropped rows. +func (e *Engine) EnsureGrantIndexes(ctx context.Context) error { + if !e.deferredIdxPending.Load() { + return nil + } + if err := e.BuildDeferredGrantIndexes(ctx); err != nil { + return err + } + return e.clearDeferredIdxPending() +} + +// ForEachDanglingGrantPrincipal visits each distinct principal referenced +// by grants for which NO resource row exists. One seek per distinct +// principal over the by_principal index (which leads with principal +// identity) plus one point probe each — O(distinct principals), never +// O(grants). Callers must EnsureGrantIndexes first or the scan misses +// synth-written grants. +// +// matchAnnotatedOnly reports that EVERY grant of the dangling principal +// carries an ExternalResourceMatch* annotation — the legitimate carrier +// shape when no external resource file was configured (the match op +// deletes carriers when it runs, so annotated survivors always mean the +// op didn't run or didn't match). carrierGrants is the number of grant +// rows under such a principal, so callers can report per-GRANT totals +// (the mixed-principal path reports its own per-grant skip count). +// Value reads happen only for dangling principals, never on the +// healthy path. +// +// A principal whose index entries are ALL orphans (no primary rows) is +// never visited: it has no grants to judge, so it is healed in place — +// the orphan index keys are deleted — instead of being vacuously +// classified as match-annotated-only. +func (e *Engine) ForEachDanglingGrantPrincipal(ctx context.Context, visit func(principalRT, principalID string, matchAnnotatedOnly bool, carrierGrants int64) error) error { + prefix := []byte{versionV3, typeIndex, idxGrantByPrincipal} + prefix = codec.AppendTupleSeparator(prefix) + iter, err := e.db.NewIter(&pebble.IterOptions{ + LowerBound: prefix, + UpperBound: upperBoundOf(prefix), + }) + if err != nil { + return err + } + defer iter.Close() + + // Healed-orphan aggregation: one Warn per sweep (the house shape — + // never per-principal logs; a systematic writer bug could strand + // entries for thousands of principals). Orphans are always a bug + // signal, so the aggregate must be visible at default level. + var healedEntries int64 + healedPrincipals := 0 + var healedExamples []string + + for valid := iter.First(); valid; { + if err := ctx.Err(); err != nil { + return err + } + key := iter.Key() + tail := key[len(prefix):] + rtBytes, next, ok := codec.DecodeTupleStringAlias(tail, 0) + if !ok || next >= len(tail) { + return fmt.Errorf("dangling grant-principal scan: malformed index key %x", key) + } + ridBytes, _, ok := codec.DecodeTupleStringAlias(tail, next+1) + if !ok { + return fmt.Errorf("dangling grant-principal scan: malformed index key %x", key) + } + rt, rid := string(rtBytes), string(ridBytes) + exists, err := e.HasResourceRecord(ctx, rt, rid) + if err != nil { + return err + } + if !exists { + matchOnly, carrierGrants, err := e.grantsForPrincipalAllMatchAnnotated(ctx, rt, rid) + if err != nil { + return err + } + switch { + case matchOnly && carrierGrants == 0: + // Every index entry under this principal is an orphan + // (no primary row): there are no grants to judge, so + // neither the match-carrier exemption nor the fail/drop + // arms apply — the "all match-annotated" verdict was + // vacuous. Heal the index garbage instead, mirroring the + // replay pre-clear's treatment of orphan scope-index + // entries, so a writer bug that strands by_principal + // entries is scrubbed rather than vacuously exempted on + // every future sweep. + healed, err := e.healOrphanPrincipalIndexEntries(ctx, rt, rid) + if err != nil { + return err + } + if healed > 0 { + healedEntries += healed + healedPrincipals++ + if len(healedExamples) < maxHealedOrphanExamples { + healedExamples = append(healedExamples, rt+"/"+rid) + } + } + default: + if err := visit(rt, rid, matchOnly, carrierGrants); err != nil { + return err + } + } + } + // Skip every remaining grant of this principal. + valid = iter.SeekGE(upperBoundOf(encodeGrantByPrincipalPrefix(rt, rid))) + } + if err := iter.Error(); err != nil { + return err + } + if healedEntries > 0 { + ctxzap.Extract(ctx).Warn("HEALED orphan by_principal index entries (index keys with no grant row — evidence of a writer bug, not connector data)", + zap.Int64("healed_entries", healedEntries), + zap.Int("principals", healedPrincipals), + zap.Strings("principal_examples", healedExamples), + ) + } + return nil +} + +// maxHealedOrphanExamples caps the principal examples on the aggregated +// healed-orphan warning, matching the dangling-reference warnings' shape. +const maxHealedOrphanExamples = 25 + +// healOrphanPrincipalIndexEntries deletes by_principal index entries +// whose primary grant row does not exist, for one principal. Identities +// are re-collected and re-probed under the write lock, so an entry whose +// row appeared since the caller's read scan is kept. Returns the number +// of entries healed. +func (e *Engine) healOrphanPrincipalIndexEntries(ctx context.Context, principalRT, principalID string) (int64, error) { + var healed int64 + err := e.withWrite(func() error { + ids, err := e.grantIdentitiesForPrincipal(ctx, principalRT, principalID) + if err != nil { + return err + } + batch := e.db.NewBatch() + defer batch.Close() + for _, id := range ids { + if err := ctx.Err(); err != nil { + return err + } + _, closer, getErr := e.db.Get(encodeGrantIdentityKey(id)) + switch { + case errors.Is(getErr, pebble.ErrNotFound): + if err := batch.Delete(encodeGrantByPrincipalIdentityIndexKey(id), nil); err != nil { + return err + } + healed++ + case getErr != nil: + return getErr + default: + closer.Close() + } + } + if healed == 0 { + return nil + } + return batch.Commit(e.dropWriteOpts()) + }) + if err != nil { + return 0, err + } + if healed > 0 { + ctxzap.Extract(ctx).Debug("healed orphan by_principal index entries", + zap.String("principal_resource_type_id", principalRT), + zap.String("principal_resource_id", principalID), + zap.Int64("healed", healed), + ) + } + return healed, nil +} + +// grantsForPrincipalAllMatchAnnotated reports whether every grant under +// the principal carries an ExternalResourceMatch* annotation, and how +// many grant rows that is (valid only when the bool is true — the walk +// stops at the first non-annotated grant). Reserved for dangling +// principals (reads row values). +func (e *Engine) grantsForPrincipalAllMatchAnnotated(ctx context.Context, principalRT, principalID string) (bool, int64, error) { + ids, err := e.grantIdentitiesForPrincipal(ctx, principalRT, principalID) + if err != nil { + return false, 0, err + } + var carrierGrants int64 + for _, id := range ids { + rec, err := e.getGrantRecordByIdentity(id) + if err != nil { + if errors.Is(err, pebble.ErrNotFound) { + continue // index entry without a row; the scan tolerates it + } + return false, 0, err + } + if !grantRecordHasExternalMatch(rec.GetAnnotations()) { + return false, 0, nil + } + carrierGrants++ + } + return true, carrierGrants, nil +} + +// grantIdentitiesForPrincipal collects the grant identities under one +// principal from the by_principal index. Collected before any deletes so +// callers never interleave iteration with writes. +func (e *Engine) grantIdentitiesForPrincipal(ctx context.Context, principalRT, principalID string) ([]grantIdentity, error) { + prefix := encodeGrantByPrincipalPrefix(principalRT, principalID) + iter, err := e.db.NewIter(&pebble.IterOptions{ + LowerBound: prefix, + UpperBound: upperBoundOf(prefix), + }) + if err != nil { + return nil, err + } + defer iter.Close() + + var ids []grantIdentity + for iter.First(); iter.Valid(); iter.Next() { + if err := ctx.Err(); err != nil { + return nil, err + } + tail := iter.Key()[len(prefix):] + var comps [4]string + off := 0 + for i := range comps { + b, next, ok := codec.DecodeTupleStringAlias(tail, off) + if !ok { + return nil, fmt.Errorf("by_principal index: malformed key tail %x", iter.Key()) + } + comps[i] = string(b) + off = next + 1 + } + ids = append(ids, grantIdentity{ + entitlement: entitlementIdentity{ + resourceTypeID: comps[0], + resourceID: comps[1], + stripped: comps[2] == idFlagStripped, + tail: comps[3], + }, + principalTypeID: principalRT, + principalID: principalID, + }) + } + return ids, iter.Error() +} + +func (e *Engine) getGrantRecordByIdentity(id grantIdentity) (*v3.GrantRecord, error) { + val, closer, err := e.db.Get(encodeGrantIdentityKey(id)) + if err != nil { + return nil, err + } + defer closer.Close() + rec := &v3.GrantRecord{} + if err := unmarshalRecord(val, rec); err != nil { + return nil, fmt.Errorf("grant record by identity: unmarshal: %w", err) + } + return rec, nil +} + +// dropBatchRows chunks the streaming drop batches: large enough to +// amortize commit overhead, small enough to bound batch memory. Drops +// during a fresh sync ride NoSync (the seal fsyncs), matching the +// tombstone and replay write paths — WITHOUT this, a dangling +// entitlement with a large grant population would pay one fsync per +// row. +const dropBatchRows = 4096 + +// stageSourceCacheScopeInvalidationLocked stages a dropped row's +// source-scope manifest invalidation INTO the batch that carries the +// row's delete. A drop deletes rows out from under a manifest entry +// whose validator still vouches for the scope's FULL upstream row set; +// left intact, the next sync's lookup would hit, the connector would +// 304, and the replay would carry the sanitized subset forward — +// permanently, even after the dangling rows' referent reappears upstream +// (a cold sync would restore them; a warm one never would). Marking the +// entry invalidated makes the next lookup miss, so exactly the +// dropped-from scopes re-fetch cold and warm/cold equivalence is +// preserved. +// +// ORDERING IS LOAD-BEARING: the invalidation must ride the SAME batch as +// the scope's FIRST staged delete, never a later write. Batches commit +// atomically and the WAL preserves commit order (a crash loses only a +// suffix), so "a committed delete for a scope implies its committed +// invalidation" holds through any crash or mid-loop error — including a +// crash between chunked commits. Invalidating after the deletes (a +// separate write) would open the one unrecoverable window: rows durably +// gone, validator retained, and the resumed invariant pass finds nothing +// dangling to re-trigger the invalidation — the next sync 304s the +// sanitized subset forever. The inverse leak (invalidation committed, +// deletes lost or unfinished) is safe: the scope re-fetches cold once, +// and the resumed pass re-detects whatever still dangles. +// +// The entry is kept (marked, not deleted) so the scope's SURVIVING +// stamped rows don't read as an I6 orphan (lost manifest write), and the +// stale validator stays visible to the audit tooling. Scopes with no +// manifest entry are skipped: nothing vouches for them, so there is +// nothing to invalidate (and writing one would hide a real I6 orphan). +// Idempotent — re-marking an invalidated entry is a no-op — so resumed +// syncs re-reaching the invariant seam converge. staged dedupes within +// one drop call; the read sees committed state, so a scope invalidated +// by an earlier call (or an earlier committed chunk) is skipped too. +// Caller must hold the write lock (runs inside the drop's withWrite +// closure). +func (e *Engine) stageSourceCacheScopeInvalidationLocked(batch *pebble.Batch, rowKind, scopeKey string, staged map[string]struct{}) error { + if scopeKey == "" { + return nil + } + if _, ok := staged[scopeKey]; ok { + return nil + } + staged[scopeKey] = struct{}{} + key := encodeSourceCacheEntryKey(rowKind, scopeKey) + val, closer, err := e.db.Get(key) + if err != nil { + if errors.Is(err, pebble.ErrNotFound) { + return nil + } + return fmt.Errorf("invalidate source-cache scope %q: %w", scopeKey, err) + } + rec := &v3.SourceCacheEntryRecord{} + unmarshalErr := unmarshalRecord(val, rec) + closer.Close() + if unmarshalErr != nil { + return fmt.Errorf("invalidate source-cache scope %q: unmarshal: %w", scopeKey, unmarshalErr) + } + if rec.GetInvalidated() { + return nil + } + rec.SetInvalidated(true) + newVal, err := marshalRecord(rec) + if err != nil { + return fmt.Errorf("invalidate source-cache scope %q: marshal: %w", scopeKey, err) + } + if err := batch.Set(key, newVal, nil); err != nil { + return fmt.Errorf("invalidate source-cache scope %q: %w", scopeKey, err) + } + return nil +} + +// dropWriteOpts returns the write options for streaming drops. +func (e *Engine) dropWriteOpts() *pebble.WriteOptions { + if e.IsFreshSync() { + return pebble.NoSync + } + return writeOpts(e.opts.durability) +} + +// dropBatcher is the single choke point for dangling-reference row +// deletion. Every dropped row carries cross-cutting obligations — +// secondary-index cleanup, digest invalidation, source-cache manifest +// invalidation, chunked-commit crash ordering — and every one of them +// has produced a bug when a call site was left to remember it. The +// batcher makes the obligations structural: a drop arm stages rows +// through stageGrantDelete / stageEntitlementDelete and calls finish; +// it never touches a raw batch, so it cannot stage a delete without its +// obligations or commit them in an unsafe order. +// +// Crash contract (see stageSourceCacheScopeInvalidationLocked): a +// scope's manifest invalidation rides the same batch as its first +// staged delete, chunks commit in WAL order, and the test hook fires +// between chunk commits so the crash-window tests can land exactly +// there. Callers must hold the engine write lock (the batcher runs +// inside the drop's withWrite closure). +type dropBatcher struct { + e *Engine + ctx context.Context + opts *pebble.WriteOptions + batch *pebble.Batch + rowsInBatch int + stagedScopes map[string]struct{} + // committedRows counts rows whose chunk commit LANDED — consumed by + // obligations that must fire even when a later chunk errors (the + // entitlement bare-id keyspace note). + committedRows int64 +} + +func (e *Engine) newDropBatcher(ctx context.Context) *dropBatcher { + return &dropBatcher{ + e: e, + ctx: ctx, + opts: e.dropWriteOpts(), + batch: e.db.NewBatch(), + stagedScopes: map[string]struct{}{}, + } +} + +// close releases the open batch; safe after finish (callers defer it). +func (b *dropBatcher) close() { _ = b.batch.Close() } + +// stageGrantDelete stages one grant row's COMPLETE drop: manifest +// invalidation for its source scope (first occurrence, same batch), +// every secondary index + digest invalidation derived from the raw +// value, and the primary delete; then rotates the chunk if full. +func (b *dropBatcher) stageGrantDelete(priKey, rawValue []byte, sourceScopeKey string) error { + if err := b.ctx.Err(); err != nil { + return err + } + if err := b.e.stageSourceCacheScopeInvalidationLocked(b.batch, "grants", sourceScopeKey, b.stagedScopes); err != nil { + return err + } + if err := b.e.deleteGrantIndexesRaw(b.batch, "", rawValue); err != nil { + return err + } + if err := b.batch.Delete(priKey, nil); err != nil { + return err + } + return b.rotate() +} + +// stageEntitlementDelete is stageGrantDelete's entitlement twin (the +// scope entry is the only entitlement secondary index). +func (b *dropBatcher) stageEntitlementDelete(id entitlementIdentity, priKey []byte, scopeKey string) error { + if err := b.ctx.Err(); err != nil { + return err + } + if err := b.e.stageSourceCacheScopeInvalidationLocked(b.batch, "entitlements", scopeKey, b.stagedScopes); err != nil { + return err + } + if scopeKey != "" { + if err := b.batch.Delete(encodeEntitlementBySourceScopeIndexKey(scopeKey, id), nil); err != nil { + return err + } + } + if err := b.batch.Delete(priKey, nil); err != nil { + return err + } + return b.rotate() +} + +func (b *dropBatcher) rotate() error { + b.rowsInBatch++ + if b.rowsInBatch < dropBatchRows { + return nil + } + if err := b.batch.Commit(b.opts); err != nil { + return err + } + b.committedRows += int64(b.rowsInBatch) + _ = b.batch.Close() + b.batch = b.e.db.NewBatch() + b.rowsInBatch = 0 + if b.e.testDropChunkHook != nil { + return b.e.testDropChunkHook() + } + return nil +} + +// finish commits the final partial chunk. +func (b *dropBatcher) finish() error { + if err := b.batch.Commit(b.opts); err != nil { + return err + } + b.committedRows += int64(b.rowsInBatch) + b.rowsInBatch = 0 + return nil +} + +// DeleteGrantsForPrincipal deletes every grant of one principal EXCEPT +// rows carrying ExternalResourceMatch* annotations (unprocessed match +// carriers are evidence of a missing external-resource file, not bad +// data). Streams one chunked batch pass — never one commit per row. +// Dropped rows' source scopes get their manifest entries invalidated +// atomically with each scope's first staged delete (see +// stageSourceCacheScopeInvalidationLocked). Returns +// (deleted, skippedMatchAnnotated). +func (e *Engine) DeleteGrantsForPrincipal(ctx context.Context, principalRT, principalID string) (int64, int64, error) { + indexPrefix := encodeGrantByPrincipalPrefix(principalRT, principalID) + + var deleted, skipped int64 + err := e.withWrite(func() error { + iter, err := e.db.NewIter(&pebble.IterOptions{ + LowerBound: indexPrefix, + UpperBound: upperBoundOf(indexPrefix), + }) + if err != nil { + return err + } + defer iter.Close() + + b := e.newDropBatcher(ctx) + defer b.close() + // Keyed on commits that LANDED, not on success: committed drop + // chunks make any stashed deferred-grant stats stale even when a + // later chunk errors, so the sidecar must recount rather than + // report rows that were durably dropped. + defer func() { + if b.committedRows > 0 { + e.invalidateDeferredGrantStats() + } + }() + + for iter.First(); iter.Valid(); iter.Next() { + if err := ctx.Err(); err != nil { + return err + } + // Index tail: ent identity components (the principal is the + // prefix). Primary key = grant header + identity tail in + // primary order. + tail := iter.Key()[len(indexPrefix):] + var comps [4]string + off := 0 + for i := range comps { + c, next, ok := codec.DecodeTupleStringAlias(tail, off) + if !ok { + return fmt.Errorf("by_principal index: malformed key tail %x", iter.Key()) + } + comps[i] = string(c) + off = next + 1 + } + id := grantIdentity{ + entitlement: entitlementIdentity{ + resourceTypeID: comps[0], + resourceID: comps[1], + stripped: comps[2] == idFlagStripped, + tail: comps[3], + }, + principalTypeID: principalRT, + principalID: principalID, + } + priKey := encodeGrantIdentityKey(id) + val, closer, getErr := e.db.Get(priKey) + if getErr != nil { + if errors.Is(getErr, pebble.ErrNotFound) { + // Orphan index entry (no row): heal it in the same + // batch pass, mirroring the replay pre-clear, so the + // drop doesn't leave index garbage behind. + if err := b.batch.Delete(iter.Key(), nil); err != nil { + return err + } + if err := b.rotate(); err != nil { + return err + } + continue + } + return getErr + } + rec := &v3.GrantRecord{} + if err := unmarshalRecord(val, rec); err != nil { + closer.Close() + return fmt.Errorf("dangling principal drop: unmarshal: %w", err) + } + if grantRecordHasExternalMatch(rec.GetAnnotations()) { + closer.Close() + skipped++ + continue + } + stageErr := b.stageGrantDelete(priKey, val, rec.GetSourceScopeKey()) + closer.Close() + if stageErr != nil { + return stageErr + } + deleted++ + } + if err := iter.Error(); err != nil { + return err + } + return b.finish() + }) + if err != nil { + return 0, 0, err + } + return deleted, skipped, nil +} + +// DeleteGrantsForEntitlement deletes every grant row under one +// entitlement identity EXCEPT rows carrying the InsertResourceGrants +// annotation — the drop arm for grants referencing an entitlement with no +// row. The exemption is per GRANT, matching the ownership rule +// (docs/tasks/dangling-reference-drops.md): a machinery-owned IRG grant +// legitimately has no entitlement row (downstream synthesizes it), while +// a connector-owned grant under the same missing entitlement is a +// dangling reference and drops. Streams the primary prefix with values in +// hand (index cleanup derives from the value), chunked batches, no +// per-row commits. Dropped rows' source scopes get their manifest +// entries invalidated atomically with each scope's first staged delete +// (see stageSourceCacheScopeInvalidationLocked). Returns +// (deleted, skippedInsertFact). +func (e *Engine) DeleteGrantsForEntitlement(ctx context.Context, entitlementID, entResourceTypeID, entResourceID string) (int64, int64, error) { + entID := entitlementIdentityFromParts(entResourceTypeID, entResourceID, entitlementID) + prefix := encodeGrantPrimaryEntitlementIdentityPrefix( + entID.resourceTypeID, entID.resourceID, entID.flagComponent(), entID.tail) + + var deleted, skipped int64 + err := e.withWrite(func() error { + iter, err := e.db.NewIter(&pebble.IterOptions{ + LowerBound: prefix, + UpperBound: upperBoundOf(prefix), + }) + if err != nil { + return err + } + defer iter.Close() + + b := e.newDropBatcher(ctx) + defer b.close() + // Keyed on commits that LANDED, not on success — see + // DeleteGrantsForPrincipal's twin defer. + defer func() { + if b.committedRows > 0 { + e.invalidateDeferredGrantStats() + } + }() + + for iter.First(); iter.Valid(); iter.Next() { + if err := ctx.Err(); err != nil { + return err + } + flags, flagsErr := scanGrantSourceCacheFlagsRaw(iter.Value()) + if flagsErr != nil { + return fmt.Errorf("dangling entitlement drop: scan flags: %w", flagsErr) + } + if flags.insertResourceGrants { + skipped++ + continue + } + _, _, _, _, _, _, sourceScopeKey, scanErr := scanGrantIndexFieldsRaw(iter.Value()) + if scanErr != nil { + return fmt.Errorf("dangling entitlement drop: scan scope: %w", scanErr) + } + key := iter.Key() + k := make([]byte, len(key)) + copy(k, key) + if err := b.stageGrantDelete(k, iter.Value(), sourceScopeKey); err != nil { + return err + } + deleted++ + } + if err := iter.Error(); err != nil { + return err + } + return b.finish() + }) + if err != nil { + return 0, 0, err + } + return deleted, skipped, nil +} + +// DeleteEntitlementsForResource deletes every entitlement row under one +// resource — the drop arm for entitlements referencing a resource with no +// row. Dropped rows' source scopes get their manifest entries +// invalidated atomically with each scope's first staged delete (see +// stageSourceCacheScopeInvalidationLocked). Returns the count +// and up to maxIDs deleted external ids for the caller's aggregate +// report; maxIDs <= 0 collects none (callers pass a shrinking example +// budget, so the guard below doubles as the clamp). +func (e *Engine) DeleteEntitlementsForResource(ctx context.Context, resourceTypeID, resourceID string, maxIDs int) (int64, []string, error) { + prefix := encodeEntitlementPrimaryResourcePrefix(resourceTypeID, resourceID) + type entRow struct { + id entitlementIdentity + scopeKey string + primaryKy []byte + } + var rows []entRow + + iter, err := e.db.NewIter(&pebble.IterOptions{ + LowerBound: prefix, + UpperBound: upperBoundOf(prefix), + }) + if err != nil { + return 0, nil, err + } + const headerLen = 3 // versionV3, typeEntitlement, separator + for iter.First(); iter.Valid(); iter.Next() { + if err := ctx.Err(); err != nil { + iter.Close() + return 0, nil, err + } + key := iter.Key() + tail := key[headerLen:] + var comps [4]string + off := 0 + ok := true + for i := range comps { + b, next, decOK := codec.DecodeTupleStringAlias(tail, off) + if !decOK { + ok = false + break + } + comps[i] = string(b) + off = next + 1 + } + if !ok { + iter.Close() + return 0, nil, fmt.Errorf("dangling entitlement drop: malformed entitlement key %x", key) + } + scope, scanErr := scanEntitlementSourceScopeRaw(iter.Value()) + if scanErr != nil { + iter.Close() + return 0, nil, scanErr + } + k := make([]byte, len(key)) + copy(k, key) + rows = append(rows, entRow{ + id: entitlementIdentity{ + resourceTypeID: comps[0], + resourceID: comps[1], + stripped: comps[2] == idFlagStripped, + tail: comps[3], + }, + scopeKey: scope, + primaryKy: k, + }) + } + if err := iter.Error(); err != nil { + iter.Close() + return 0, nil, err + } + iter.Close() + + var deleted int64 + var ids []string + err = e.withWrite(func() error { + b := e.newDropBatcher(ctx) + defer b.close() + // Keyed on commits that LANDED, not on success: a failure after a + // partial chunk commit has already mutated the entitlement + // keyspace, so the bare-id lookup map must be invalidated even as + // the error unwinds (same contract as the replay pre-clear). + defer func() { + if b.committedRows > 0 { + e.noteEntitlementKeyspaceWrite() + } + }() + for _, row := range rows { + if err := b.stageEntitlementDelete(row.id, row.primaryKy, row.scopeKey); err != nil { + return err + } + deleted++ + if len(ids) < maxIDs { + ids = append(ids, row.id.externalID()) + } + } + return b.finish() + }) + return deleted, ids, err +} diff --git a/pkg/dotc1z/engine/pebble/ingest_repair_crash_test.go b/pkg/dotc1z/engine/pebble/ingest_repair_crash_test.go new file mode 100644 index 000000000..6105053f2 --- /dev/null +++ b/pkg/dotc1z/engine/pebble/ingest_repair_crash_test.go @@ -0,0 +1,173 @@ +package pebble + +// Deterministic crash test for the dangling-reference drops' manifest +// invalidation. The drops delete rows in chunked batches; a crash (or +// mid-loop error) between a committed chunk and the drop's completion +// must NEVER leave a scope with durably deleted rows and an intact +// manifest validator — the resumed invariant pass would find nothing +// dangling to re-trigger the invalidation, and every later sync would +// 304-replay the sanitized subset (the exact permanent-divergence bug +// manifest invalidation exists to prevent). The contract +// (stageSourceCacheScopeInvalidationLocked): the invalidation rides the +// SAME batch as the scope's first staged delete, so batch atomicity plus +// WAL prefix ordering make "committed delete ⇒ committed invalidation" +// hold through any crash point. + +import ( + "context" + "errors" + "fmt" + "testing" + + "github.com/stretchr/testify/require" + + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + "github.com/conductorone/baton-sdk/pkg/connectorstore" + "github.com/conductorone/baton-sdk/pkg/sourcecache" +) + +// TestDanglingDropCrashBetweenChunksKeepsInvalidation crashes +// DeleteGrantsForEntitlement after its FIRST committed chunk (via the +// engine's test hook — the in-process analog of a process kill at that +// seam) and asserts the half-dropped scope's manifest entry is already +// invalidated. The re-run (the resumed sync's idempotent invariant pass) +// then converges: remaining rows drop, the entry stays invalidated. +func TestDanglingDropCrashBetweenChunksKeepsInvalidation(t *testing.T) { + ctx := context.Background() + a := newAdapter(t) + _, err := a.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + e := a.PebbleEngine() + + // More than one drop chunk of grants under ONE dangling entitlement + // (scGrant's group:g1:member), all stamped with one scope that holds + // a valid manifest entry. + const total = dropBatchRows + 500 + scopedCtx := sourcecache.WithScope(ctx, scopeA) + const putChunk = 1000 + for start := 0; start < total; start += putChunk { + end := min(start+putChunk, total) + page := make([]*v2.Grant, 0, end-start) + for i := start; i < end; i++ { + page = append(page, scGrant("member", fmt.Sprintf("u%05d", i), false)) + } + require.NoError(t, a.PutGrants(scopedCtx, page...)) + } + require.NoError(t, e.PutSourceCacheEntry(ctx, "grants", scopeA, "etag-v1")) + + scopePrefix := encodeGrantBySourceScopePrefix(scopeA) + before, err := e.countSourceScopeIndexRange(ctx, scopePrefix) + require.NoError(t, err) + require.Equal(t, int64(total), before) + + // Crash after the first committed chunk. + injected := errors.New("injected crash between drop chunks") + e.testDropChunkHook = func() error { return injected } + _, _, err = e.DeleteGrantsForEntitlement(ctx, "group:g1:member", "group", "g1") + require.ErrorIs(t, err, injected) + e.testDropChunkHook = nil + + // The crash landed between a committed chunk and completion: some + // rows are durably gone, some remain. + remaining, err := e.countSourceScopeIndexRange(ctx, scopePrefix) + require.NoError(t, err) + require.Equal(t, int64(total-dropBatchRows), remaining, + "exactly the first chunk must be committed at the crash point") + + // THE CONTRACT: the scope's manifest entry must already be + // invalidated — it rode the first chunk's batch. An intact validator + // here is the permanent-sanitized-replay bug. + rec, err := e.GetSourceCacheEntry(ctx, "grants", scopeA) + require.NoError(t, err) + require.True(t, rec.GetInvalidated(), + "a committed drop chunk without a committed invalidation would 304-replay the sanitized scope forever") + require.Equal(t, "etag-v1", rec.GetCacheValidator(), + "the stale validator is kept (marked, not erased) for the audit surface") + + // Resume: the idempotent re-run converges — remaining rows drop and + // the entry stays invalidated. + deleted, skipped, err := e.DeleteGrantsForEntitlement(ctx, "group:g1:member", "group", "g1") + require.NoError(t, err) + require.Zero(t, skipped) + require.Equal(t, int64(total-dropBatchRows), deleted) + after, err := e.countSourceScopeIndexRange(ctx, scopePrefix) + require.NoError(t, err) + require.Zero(t, after) + rec, err = e.GetSourceCacheEntry(ctx, "grants", scopeA) + require.NoError(t, err) + require.True(t, rec.GetInvalidated()) +} + +// TestManifestSnapshotInvalidationIsTyped pins the snapshot's typed +// invalidation verdict: validators are OPAQUE connector strings, so a +// legitimate validator that happens to end with any in-band sentinel +// must never be misclassified as invalidated (the audit once encoded +// invalidation as a "\x00invalidated" validator suffix — this is its +// regression test), and an invalidated entry must report its verdict in +// the typed field with the original validator intact. +func TestManifestSnapshotInvalidationIsTyped(t *testing.T) { + ctx := context.Background() + a := newAdapter(t) + _, err := a.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + e := a.PebbleEngine() + + // A hostile-but-legal validator: byte-identical to the old in-band + // sentinel encoding. + trapValidator := "W/\"etag\"\x00invalidated" + require.NoError(t, e.PutSourceCacheEntry(ctx, "grants", scopeA, trapValidator)) + + // A genuinely invalidated scope, via the drop path. + require.NoError(t, a.PutGrants(sourcecache.WithScope(ctx, scopeB), + scGrant("member", "alice", false))) + require.NoError(t, e.PutSourceCacheEntry(ctx, "grants", scopeB, "etag-b")) + _, _, err = e.DeleteGrantsForEntitlement(ctx, "group:g1:member", "group", "g1") + require.NoError(t, err) + + snap, err := e.SourceCacheManifestSnapshot(ctx) + require.NoError(t, err) + + trap := snap["grants\x00"+scopeA] + require.False(t, trap.Invalidated, + "a legitimate validator matching the old sentinel encoding must not be misclassified") + require.Equal(t, trapValidator, trap.CacheValidator, + "the opaque validator must round-trip byte-exact") + + dropped := snap["grants\x00"+scopeB] + require.True(t, dropped.Invalidated, "the dropped-from scope must report its typed verdict") + require.Equal(t, "etag-b", dropped.CacheValidator, + "invalidation must not mutate the stored validator") +} + +// TestDanglingDropCleanCompletionInvalidates pins the non-crash contract +// for all three drop arms: a completed drop leaves every touched scope's +// manifest entry invalidated (and only touched scopes — a skipped +// exempt-carrier's scope keeps its validator). +func TestDanglingDropCleanCompletionInvalidates(t *testing.T) { + ctx := context.Background() + a := newAdapter(t) + _, err := a.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + e := a.PebbleEngine() + + // Scope A: dangling grants (dropped). Scope B: untouched grants under + // a different entitlement (validator must survive). + require.NoError(t, a.PutGrants(sourcecache.WithScope(ctx, scopeA), + scGrant("member", "alice", false), scGrant("member", "bob", false))) + require.NoError(t, a.PutGrants(sourcecache.WithScope(ctx, scopeB), + scGrant("owner", "carol", false))) + require.NoError(t, e.PutSourceCacheEntry(ctx, "grants", scopeA, "etag-a")) + require.NoError(t, e.PutSourceCacheEntry(ctx, "grants", scopeB, "etag-b")) + + deleted, skipped, err := e.DeleteGrantsForEntitlement(ctx, "group:g1:member", "group", "g1") + require.NoError(t, err) + require.Zero(t, skipped) + require.Equal(t, int64(2), deleted) + + recA, err := e.GetSourceCacheEntry(ctx, "grants", scopeA) + require.NoError(t, err) + require.True(t, recA.GetInvalidated(), "the dropped-from scope must be invalidated") + recB, err := e.GetSourceCacheEntry(ctx, "grants", scopeB) + require.NoError(t, err) + require.False(t, recB.GetInvalidated(), "an untouched scope's validator must survive") +} diff --git a/pkg/dotc1z/engine/pebble/ingest_repair_orphan_index_test.go b/pkg/dotc1z/engine/pebble/ingest_repair_orphan_index_test.go new file mode 100644 index 000000000..106a03487 --- /dev/null +++ b/pkg/dotc1z/engine/pebble/ingest_repair_orphan_index_test.go @@ -0,0 +1,97 @@ +package pebble + +// Pins the orphan by_principal handling in the dangling-principal sweep +// (I9's engine side): an index entry with no primary grant row must be +// HEALED (deleted), never vacuously classified. Before the fix, a +// principal whose entries were all orphans came back as "all +// match-annotated" with zero carriers — skipping the fail/drop arms, +// emitting no signal, and leaving the garbage for every future sweep — +// and the drop path kept orphan keys behind the rows it deleted. + +import ( + "context" + "fmt" + "testing" + + "github.com/cockroachdb/pebble/v2" + "github.com/stretchr/testify/require" + + "github.com/conductorone/baton-sdk/pkg/connectorstore" +) + +// plantOrphanPrincipalIndexEntry writes ONLY a by_principal index key — +// no primary row — simulating a writer bug that stranded the entry. +func plantOrphanPrincipalIndexEntry(t *testing.T, e *Engine, principalID, entTail string) grantIdentity { + t.Helper() + id := grantIdentity{ + entitlement: entitlementIdentityFromParts("group", "g1", "group:g1:"+entTail), + principalTypeID: "user", + principalID: principalID, + } + require.NoError(t, e.db.Set(encodeGrantByPrincipalIdentityIndexKey(id), nil, nil)) + return id +} + +func countPrincipalIndexEntries(t *testing.T, e *Engine, principalRT, principalID string) int { + t.Helper() + ids, err := e.grantIdentitiesForPrincipal(context.Background(), principalRT, principalID) + require.NoError(t, err) + return len(ids) +} + +func TestDanglingPrincipalSweepHealsAllOrphanIndexEntries(t *testing.T) { + ctx := context.Background() + a := newAdapter(t) + _, err := a.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + e := a.PebbleEngine() + + // "ghost" has index entries but no grant rows and no resource row: + // the all-orphan shape. Before the fix this visited as + // (matchAnnotatedOnly=true, carrierGrants=0) and nothing was cleaned. + for i := 0; i < 3; i++ { + plantOrphanPrincipalIndexEntry(t, e, "ghost", fmt.Sprintf("member%d", i)) + } + + visited := 0 + require.NoError(t, e.ForEachDanglingGrantPrincipal(ctx, func(rt, rid string, matchOnly bool, carriers int64) error { + visited++ + return nil + })) + require.Zero(t, visited, "an all-orphan principal has no grants to judge and must not be visited") + require.Zero(t, countPrincipalIndexEntries(t, e, "user", "ghost"), + "orphan index entries must be healed, not tolerated forever") + + // Idempotent: a re-run sees a clean index. + require.NoError(t, e.ForEachDanglingGrantPrincipal(ctx, func(rt, rid string, matchOnly bool, carriers int64) error { + visited++ + return nil + })) + require.Zero(t, visited) +} + +func TestDeleteGrantsForPrincipalHealsOrphanEntries(t *testing.T) { + ctx := context.Background() + a := newAdapter(t) + _, err := a.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + e := a.PebbleEngine() + + // Mixed population: one REAL dangling grant row (drops) plus one + // orphan index entry (must be healed by the same pass, not left + // behind the dropped rows). + g := mkV2Grant("g-real", "group:g1:member", "user", "mixed") + require.NoError(t, a.PutGrants(ctx, g)) + plantOrphanPrincipalIndexEntry(t, e, "mixed", "owner") + require.Equal(t, 2, countPrincipalIndexEntries(t, e, "user", "mixed")) + + deleted, skipped, err := e.DeleteGrantsForPrincipal(ctx, "user", "mixed") + require.NoError(t, err) + require.Equal(t, int64(1), deleted, "the real dangling row drops") + require.Zero(t, skipped) + require.Zero(t, countPrincipalIndexEntries(t, e, "user", "mixed"), + "the orphan entry must be healed in the same pass") + + _, err = e.GetGrantRecord(ctx, g.GetId()) + require.ErrorIs(t, err, pebble.ErrNotFound) +} diff --git a/pkg/dotc1z/engine/pebble/inspect_diff_test.go b/pkg/dotc1z/engine/pebble/inspect_diff_test.go index c31aa4117..39484c8b3 100644 --- a/pkg/dotc1z/engine/pebble/inspect_diff_test.go +++ b/pkg/dotc1z/engine/pebble/inspect_diff_test.go @@ -543,7 +543,7 @@ func grantIdentityFromByEntitlementKey(key []byte, layout string) (string, bool) } func grantIdentityFromLegacyGrantValue(value []byte) (string, bool, error) { - entRT, entRID, entID, principalRT, principalID, _, err := scanGrantIndexFieldsRaw(value) + entRT, entRID, entID, principalRT, principalID, _, _, err := scanGrantIndexFieldsRaw(value) if err != nil { return "", false, err } diff --git a/pkg/dotc1z/engine/pebble/keys.go b/pkg/dotc1z/engine/pebble/keys.go index 440793d5e..bd6a1b973 100644 --- a/pkg/dotc1z/engine/pebble/keys.go +++ b/pkg/dotc1z/engine/pebble/keys.go @@ -64,7 +64,16 @@ const ( typeCounter byte = 0x08 typeSession byte = 0x09 typeDigest byte = 0x0A - typeEngineMeta byte = 0xFF + // typeSourceCache holds one SourceCacheEntryRecord per + // (row_kind, scope_key) — the source-cache replay manifest (see + // proto/c1/connector/v2/annotation_source_cache.proto). Placed + // after typeSession so the clone path's counter/session excise + // span ([typeCounter, upperBoundOf(typeSession))) leaves it in the + // clone: a cloned sync artifact stays usable as a replay source. + // ResetForNewSync's [typeResourceType, typeEngineMeta) span wipes + // it with the rest of the sync-scoped data. + typeSourceCache byte = 0x0B + typeEngineMeta byte = 0xFF ) // Index-discriminator bytes (second byte after typeIndex). One byte @@ -86,6 +95,13 @@ const ( // (typeDigest) folds over; built ONLY by the seal-time deferred // pass, never maintained inline. See digest.go and grant_digest.go. idxGrantByEntitlementPrincipalHash byte = 0x08 + // by_source_scope families: partial indexes over records whose + // source_scope_key is non-empty (source-cache replay). Tails are + // identity tuples per the keys.go convention, so replay derives + // each primary key from the index key without touching the record. + idxGrantBySourceScope byte = 0x09 + idxEntitlementBySourceScope byte = 0x0A + idxResourceBySourceScope byte = 0x0B ) // --- Grant --- @@ -213,6 +229,19 @@ func encodeGrantPrimaryEntitlementResourcePrefix(resourceTypeID, resourceID stri return codec.AppendTupleSeparator(buf) } +// encodeGrantPrimaryEntitlementIdentityPrefix is the grant primary-key +// prefix covering every grant of one entitlement identity: the four +// leading tuple components (rt, rid, flag, tail) with a trailing +// separator. Components are passed raw (as decoded from a key) and +// re-escaped by the tuple codec, so the round trip is byte-exact. +func encodeGrantPrimaryEntitlementIdentityPrefix(resourceTypeID, resourceID, flag, tail string) []byte { + buf := make([]byte, 0, 96) + buf = append(buf, versionV3, typeGrant) + buf = codec.AppendTupleSeparator(buf) + buf = codec.AppendTupleStrings(buf, resourceTypeID, resourceID, flag, tail) + return codec.AppendTupleSeparator(buf) +} + // encodeGrantByNeedsExpansionIndexKey: index of grants whose // NeedsExpansion flag is true. Pebble equivalent of the SQLite // partial index `WHERE needs_expansion = 1`. The grant is added to @@ -399,6 +428,139 @@ func DigestUpperBound() []byte { return upperBoundOf(DigestLowerBound()) } +// --- Source-cache (by_source_scope indexes + entry keyspace) --- + +// encodeGrantBySourceScopeIndexKey is the partial by_source_scope index +// over grants whose SourceScopeKey is non-empty: +// +// v3 | typeIndex | idxGrantBySourceScope | 0x00 | +// scope_key | 0x00 | +// ent_rt | 0x00 | ent_rid | 0x00 | ent_flag | 0x00 | ent_tail | 0x00 | +// principal_rt | 0x00 | principal_id +// +// The tail after scope_key is the grant identity tuple — byte-identical +// to the grant primary key's tail — so replay derives the primary key from +// the index key alone. Paired with encodeGrantBySourceScopePrefix +// (by-value prefix, with trailing sep). +func encodeGrantBySourceScopeIndexKey(scopeKey string, id grantIdentity) []byte { + return appendGrantBySourceScopeIndexKey(make([]byte, 0, 128), scopeKey, id) +} + +func appendGrantBySourceScopeIndexKey(dst []byte, scopeKey string, id grantIdentity) []byte { + dst = append(dst, versionV3, typeIndex, idxGrantBySourceScope) + dst = codec.AppendTupleSeparator(dst) + return codec.AppendTupleStrings( + dst, + scopeKey, + id.entitlement.resourceTypeID, + id.entitlement.resourceID, + id.entitlement.flagComponent(), + id.entitlement.tail, + id.principalTypeID, + id.principalID, + ) +} + +// encodeGrantBySourceScopePrefix is the by-value prefix for "all grants +// stamped with this scope key". Trailing sep is load-bearing — see the +// keys.go convention doc. +func encodeGrantBySourceScopePrefix(scopeKey string) []byte { + buf := make([]byte, 0, 32+len(scopeKey)) + buf = append(buf, versionV3, typeIndex, idxGrantBySourceScope) + buf = codec.AppendTupleSeparator(buf) + buf = codec.AppendTupleStrings(buf, scopeKey) + return codec.AppendTupleSeparator(buf) +} + +// encodeEntitlementBySourceScopeIndexKey: +// +// v3 | typeIndex | idxEntitlementBySourceScope | 0x00 | +// scope_key | 0x00 | rt | 0x00 | rid | 0x00 | flag | 0x00 | tail +// +// Tail is the entitlement identity tuple, byte-identical to the +// entitlement primary key's tail. Paired with +// encodeEntitlementBySourceScopePrefix. +func encodeEntitlementBySourceScopeIndexKey(scopeKey string, id entitlementIdentity) []byte { + return appendEntitlementBySourceScopeIndexKey(make([]byte, 0, 128), scopeKey, id) +} + +func appendEntitlementBySourceScopeIndexKey(dst []byte, scopeKey string, id entitlementIdentity) []byte { + dst = append(dst, versionV3, typeIndex, idxEntitlementBySourceScope) + dst = codec.AppendTupleSeparator(dst) + return codec.AppendTupleStrings(dst, scopeKey, id.resourceTypeID, id.resourceID, id.flagComponent(), id.tail) +} + +func encodeEntitlementBySourceScopePrefix(scopeKey string) []byte { + buf := make([]byte, 0, 32+len(scopeKey)) + buf = append(buf, versionV3, typeIndex, idxEntitlementBySourceScope) + buf = codec.AppendTupleSeparator(buf) + buf = codec.AppendTupleStrings(buf, scopeKey) + return codec.AppendTupleSeparator(buf) +} + +// encodeResourceBySourceScopeIndexKey: +// +// v3 | typeIndex | idxResourceBySourceScope | 0x00 | +// scope_key | 0x00 | resource_type_id | 0x00 | resource_id +// +// Tail is the resource primary tuple. Paired with +// encodeResourceBySourceScopePrefix. +func encodeResourceBySourceScopeIndexKey(scopeKey, resourceTypeID, resourceID string) []byte { + return appendResourceBySourceScopeIndexKey(make([]byte, 0, 96), scopeKey, resourceTypeID, resourceID) +} + +func appendResourceBySourceScopeIndexKey(dst []byte, scopeKey, resourceTypeID, resourceID string) []byte { + dst = append(dst, versionV3, typeIndex, idxResourceBySourceScope) + dst = codec.AppendTupleSeparator(dst) + return codec.AppendTupleStrings(dst, scopeKey, resourceTypeID, resourceID) +} + +func encodeResourceBySourceScopePrefix(scopeKey string) []byte { + buf := make([]byte, 0, 32+len(scopeKey)) + buf = append(buf, versionV3, typeIndex, idxResourceBySourceScope) + buf = codec.AppendTupleSeparator(buf) + buf = codec.AppendTupleStrings(buf, scopeKey) + return codec.AppendTupleSeparator(buf) +} + +// encodeSourceCacheEntryKey is the primary key for one source-cache +// manifest entry: +// +// v3 | typeSourceCache | 0x00 | row_kind | 0x00 | scope_key +// +// Paired with encodeSourceCachePrefix (by-type prefix). +func encodeSourceCacheEntryKey(rowKind, scopeKey string) []byte { + buf := make([]byte, 0, 32+len(rowKind)+len(scopeKey)) + buf = append(buf, versionV3, typeSourceCache) + buf = codec.AppendTupleSeparator(buf) + return codec.AppendTupleStrings(buf, rowKind, scopeKey) +} + +// encodeSourceCachePrefix is the by-type prefix for ALL source-cache +// state (manifest entries and the compat record). ClearSourceCacheEntries +// and ResetForNewSync cover this whole prefix. +func encodeSourceCachePrefix() []byte { + return []byte{versionV3, typeSourceCache} +} + +// encodeSourceCacheManifestPrefix is the sub-prefix for (row_kind, +// scope_key) MANIFEST entries only — the 0x00 discriminator every +// encodeSourceCacheEntryKey starts with. Manifest iterators (snapshot, +// clear-check) bound here so the compat record (0x01 discriminator) +// never unmarshals as a manifest entry. +func encodeSourceCacheManifestPrefix() []byte { + return []byte{versionV3, typeSourceCache, 0x00} +} + +// encodeSourceCacheCompatKey is the singleton replay-compatibility key +// record for the sync (see SourceCacheCompatRecord): one key under the +// source-cache prefix with a 0x01 discriminator, so the fold's +// ClearSourceCacheEntries and ResetForNewSync wipe it with the manifest +// while manifest iterators never see it. +func encodeSourceCacheCompatKey() []byte { + return []byte{versionV3, typeSourceCache, 0x01, 'c', 'o', 'm', 'p', 'a', 't'} +} + // --- ResourceType --- // encodeResourceTypeKey returns the primary key for a resource_type: @@ -627,6 +789,28 @@ func GrantByEntitlementResourceUpperBound() []byte { return upperBoundOf(GrantByEntitlementResourceLowerBound()) } +func GrantBySourceScopeLowerBound() []byte { + return []byte{versionV3, typeIndex, idxGrantBySourceScope} +} +func GrantBySourceScopeUpperBound() []byte { return upperBoundOf(GrantBySourceScopeLowerBound()) } + +func EntitlementBySourceScopeLowerBound() []byte { + return []byte{versionV3, typeIndex, idxEntitlementBySourceScope} +} +func EntitlementBySourceScopeUpperBound() []byte { + return upperBoundOf(EntitlementBySourceScopeLowerBound()) +} + +func ResourceBySourceScopeLowerBound() []byte { + return []byte{versionV3, typeIndex, idxResourceBySourceScope} +} +func ResourceBySourceScopeUpperBound() []byte { + return upperBoundOf(ResourceBySourceScopeLowerBound()) +} + +func SourceCacheEntryLowerBound() []byte { return encodeSourceCachePrefix() } +func SourceCacheEntryUpperBound() []byte { return upperBoundOf(encodeSourceCachePrefix()) } + func AssetLowerBound() []byte { return encodeAssetPrefix() } func AssetUpperBound() []byte { return upperBoundOf(encodeAssetPrefix()) } diff --git a/pkg/dotc1z/engine/pebble/lookup.go b/pkg/dotc1z/engine/pebble/lookup.go index 2b70e59fe..1058b7b0b 100644 --- a/pkg/dotc1z/engine/pebble/lookup.go +++ b/pkg/dotc1z/engine/pebble/lookup.go @@ -265,6 +265,29 @@ func (e *Engine) grantPrimaryPrefixNonEmpty(prefix []byte) (bool, error) { // the row instead of the concat). Exactly one hit wins; zero is // pebble.ErrNotFound; several is ErrAmbiguousExternalID. func (e *Engine) resolveGrantIdentityByExternalID(ctx context.Context, grantID string) (grantIdentity, error) { + id, err := e.resolveGrantIdentityByCandidates(ctx, grantID) + if err == nil || !errors.Is(err, errNoGrantCandidateHits) { + return id, err + } + // Candidate probing proved nothing either way: the id may still be a + // connector-custom STORED external id (with or without colons), which + // only the O(all grants) scan can find. Only interactive edges reach + // this line; bulk paths (source-cache tombstones) resolve rows via + // the scope index instead and never pay the scan. + return e.scanGrantIdentityByStoredExternalID(ctx, grantID) +} + +// errNoGrantCandidateHits reports that combinatorial candidate probing +// completed without a hit — distinct from pebble.ErrNotFound, which the +// full resolution reserves for "provably absent after the scan of last +// resort". Internal to the resolution layer. +var errNoGrantCandidateHits = errors.New("pebble: no grant candidate parse hit") + +// resolveGrantIdentityByCandidates is the bounded stage of grant-id +// resolution: combinatorial concat splits probed with point Gets, no +// keyspace scan. Returns errNoGrantCandidateHits when the shape yields no +// candidates (fewer than two colons) or every candidate missed. +func (e *Engine) resolveGrantIdentityByCandidates(ctx context.Context, grantID string) (grantIdentity, error) { var colons []int for i := 0; i < len(grantID); i++ { if grantID[i] == ':' { @@ -272,10 +295,8 @@ func (e *Engine) resolveGrantIdentityByExternalID(ctx context.Context, grantID s } } if len(colons) < 2 { - // No concat shape to split: connector-custom ids (SQLite keyed rows - // by these, and provisioner revokes address grants with them) are - // findable only by their STORED external id. - return e.scanGrantIdentityByStoredExternalID(ctx, grantID) + // No concat shape to split: findable only by STORED external id. + return grantIdentity{}, errNoGrantCandidateHits } if len(colons) > maxBareIDColons { return grantIdentity{}, fmt.Errorf("%w: grant id has %d colons; too complex to resolve safely by string", ErrAmbiguousExternalID, len(colons)) @@ -378,9 +399,7 @@ func (e *Engine) resolveGrantIdentityByExternalID(ctx context.Context, grantID s } switch len(hits) { case 0: - // Every concat split missed: the id may still be a connector-custom - // STORED external id that merely contains colons. - return e.scanGrantIdentityByStoredExternalID(ctx, grantID) + return grantIdentity{}, errNoGrantCandidateHits case 1: return hits[0], nil default: diff --git a/pkg/dotc1z/engine/pebble/lookup_test.go b/pkg/dotc1z/engine/pebble/lookup_test.go index cffafc464..ab67f6f9d 100644 --- a/pkg/dotc1z/engine/pebble/lookup_test.go +++ b/pkg/dotc1z/engine/pebble/lookup_test.go @@ -68,7 +68,7 @@ func TestBareIDEntitlementLookupExactlyOne(t *testing.T) { // Two matches → explicit ambiguity, for reads AND deletes. _, err = e.GetEntitlementRecord(ctx, "shared-id") require.ErrorIs(t, err, ErrAmbiguousExternalID) - err = e.DeleteEntitlementRecord(ctx, "shared-id") + _, err = e.DeleteEntitlementRecord(ctx, "shared-id") require.ErrorIs(t, err, ErrAmbiguousExternalID, "an ambiguous string must never guess a delete") // Both ambiguous rows are intact. @@ -101,7 +101,9 @@ func TestBareIDEntitlementLookupInvalidation(t *testing.T) { require.NoError(t, err, "write must invalidate the cached map") require.Equal(t, "second", got.GetExternalId()) - require.NoError(t, e.DeleteEntitlementRecord(ctx, "first")) + deletedFirst, err := e.DeleteEntitlementRecord(ctx, "first") + require.NoError(t, err) + require.True(t, deletedFirst) _, err = e.GetEntitlementRecord(ctx, "first") require.ErrorIs(t, err, pebble.ErrNotFound, "delete must invalidate the cached map") } diff --git a/pkg/dotc1z/engine/pebble/mutation_paths.go b/pkg/dotc1z/engine/pebble/mutation_paths.go new file mode 100644 index 000000000..9b36f5211 --- /dev/null +++ b/pkg/dotc1z/engine/pebble/mutation_paths.go @@ -0,0 +1,195 @@ +package pebble + +// The mutation-path registry. +// +// THE LAW THIS FILE ENCODES: bug count scales with (paths × obligations) +// when cross-cutting write obligations are enforced by call-site +// convention, and with (paths + obligations) when they are derived at +// choke points. Three generations of the same bug family prove it here: +// the replay ingestion path missed the fresh path's side effects (the +// I1–I4 invariants exist because of it), the deletion paths missed +// index/manifest/dirty obligations (the dropBatcher and the +// invalidation-with-first-delete contract exist because of it), and each +// was found by review after shipping. This registry is the +// sideEffectAnnotationCoverage trick applied to code paths: every file +// in this package that touches a RAW Pebble write primitive must declare +// what it writes, how it satisfies the cross-cutting obligations, and +// (where they exist in-package) the tests that pin its contracts. +// mutation_paths_test.go enforces both directions mechanically — an +// undeclared raw-writing file and a stale declaration both fail — so a +// NEW mutation path cannot ship without having this conversation in the +// PR that adds it. +// +// THE OBLIGATIONS every declaration must account for (explicitly or as +// n/a-with-reason): +// +// - secondary indexes: by_principal / by_source_scope / +// needs_expansion / by_parent entries staged with the row mutation +// (or deliberately deferred to the seal build). +// - digest invalidation: post-seal grant mutations must stage +// per-entitlement digest invalidation (present-means-exact). +// - source-cache manifest: deleting rows out from under a scope's +// validator must invalidate the entry IN THE SAME BATCH as the +// scope's first delete (see stageSourceCacheScopeInvalidationLocked). +// - envelope save: mutations must survive Close — the wrapper dirty +// bit plus the Engine.Mutated latch (withWriteAllowSealed) backstop. +// - entitlement keyspace note: entitlement-keyspace mutations must +// invalidate the bare-id lookup map, keyed on commits that LANDED. +// - crash ordering: multi-batch operations must leave every WAL prefix +// safe (durable markers, idempotent re-runs, obligation-first +// batching). +// +// SYNCER-LEVEL MUTATION PATHS (they all funnel into the files below; +// listed here so the inventory has one home): +// +// 1. fresh response pages (Put*) — oracle: the replay- +// equivalence harness's cold legs + model-derived expected sets. +// 2. source-cache replay copy — oracle: warm ≡ cold +// (source_cache_equivalence_test.go) + halt-point sweep. +// 3. deletion family (tombstones, dangling drops, replay pre-clear) +// — oracle: the dirty +// scenario family + drop crash tests; all drops via dropBatcher. +// 4. raw-SST ingestion (ToPebble conversion, compactor merge) +// — oracle: byte-equivalence +// vs the canonical write path (TestToPebbleBulkImportEquivalence, +// pkg/dotc1z). +// 5. expansion/synth writes — deferred index + fused +// digest build at seal; durable pending markers. +// 6. external-match transform (delete carriers + write transformed) +// — composite path over +// grants.go primitives; deliberately does NOT invalidate manifests +// (transformed grants replace carriers under the SAME scope, so +// warm/cold converge — pinned by the harness's external-churn +// divergence trap). +// 7. I3 repair copy (CopyResourceRowFrom) — single-row ingestion +// from the previous artifact. +// +// PRODUCT NON-GOALS the registry records so they are not re-opened by +// accident: SQLite artifacts are never source-cache replay sources +// (SourceCacheStore is Pebble-only; SQLite rows carry no scope stamps), +// and compacted artifacts are never replay sources (keep-newer merges +// that no input's validators describe; triple-gated by the compacted +// flag, manifest clearing, and token provenance). + +// rawWriteDeclaration documents one file's raw write surface. +type rawWriteDeclaration struct { + // Writes summarizes what the file mutates and on which path(s). + Writes string + // Obligations states how the cross-cutting obligations are satisfied + // (or why they do not apply). + Obligations string + // PinnedBy lists IN-PACKAGE test functions pinning the file's write + // contracts; existence is enforced. Cross-package oracles are named + // in Obligations text instead. + PinnedBy []string +} + +// rawWriteRegistry: one entry per non-test file in this package that +// invokes a raw Pebble write primitive (NewBatch, db.Set, db.Delete, +// DeleteRange, db.Ingest, IngestAndExcise, db.Apply). Enforced by +// TestRawWriteRegistryCoversAllRawWritingFiles. +var rawWriteRegistry = map[string]rawWriteDeclaration{ + "assets.go": { + Writes: "asset rows (put/delete) on the fresh ingestion path", + Obligations: "no secondary indexes, digests, or scope stamps; envelope save via store wrappers + Engine.Mutated backstop", + }, + "bulk_import.go": { + Writes: "raw-SST bulk ingestion (ToPebble conversion from sorted SQLite sources)", + Obligations: "maintains grantIndexFamilies inline (incl. by_source_scope; SQLite sources carry no scope stamps — SQLite replay is a non-goal); " + + "digests deliberately absent (derived at seal); byte-equivalence vs the canonical write path pinned by TestToPebbleBulkImportEquivalence (pkg/dotc1z)", + }, + "deferred_index.go": { + Writes: "seal-time by_principal index rebuild + primary-grant keyspace rebuild via IngestAndExcise", + Obligations: "crash ordering via the durable deferred-index pending marker (interrupted build re-runs at resumed EndSync); " + + "stats stash invalidated by drops (invalidateDeferredGrantStats)", + }, + "digest.go": { + Writes: "grant digest nodes and invalidation ranges", + Obligations: "present-means-exact: mutations stage invalidation for touched entitlements; build/repair own reconstruction", + }, + "entitlements.go": { + Writes: "entitlement primary rows + by_source_scope index (fresh, replay overlay, tombstones)", + Obligations: "scope index staged with the row; bare-id lookup map invalidated via noteEntitlementKeyspaceWrite keyed on landed commits", + }, + "grant_digest.go": { + Writes: "digest keyspace drops (wholesale invalidation)", + Obligations: "drop-first recovery: every crash converts to digests-absent, which present-means-exact treats as safe", + }, + "grant_digest_build.go": { + Writes: "seal-time by_entitlement_principal_hash index + digest construction (fused with the deferred pass)", + Obligations: "crash ordering via the durable digest-build pending marker (crash mid-build drops digest state at next open/repair " + + "instead of trusting partial roots)", + }, + "grant_digest_repair.go": { + Writes: "digest repair for entitlements missing roots (post-seal mutation healing)", + Obligations: "runs under the write lock end-to-end so no grant write interleaves mid-partition; trusts only present roots", + }, + "grants.go": { + Writes: "grant primary rows + inline indexes (needs_expansion, by_source_scope), synth-layer SST ingestion, deferred-index marker arming", + Obligations: "by_principal deferred to seal (durable marker); post-seal mutations stage digest invalidation; " + + "scope index staged with the row (replay equivalence depends on it)", + }, + "id_index_format.go": { + Writes: "engine-meta markers for the id-index layout (deferred-index pending, digest-build pending)", + Obligations: "markers ARE the crash-ordering mechanism for the seal builds; durable (survive restart) by design", + }, + "id_index_migration.go": { + Writes: "on-open id-index layout migration via IngestAndExcise", + Obligations: "runs before the store serves reads; migrated-on-open forces an envelope save so the migration is never repaid", + }, + "index_migrations.go": { + Writes: "on-open secondary-index backfills", + Obligations: "same on-open contract as the id-index migration: complete before serving, dirty-on-migrate", + }, + "if_newer.go": { + Writes: "keep-newer merge primitive (compaction fold path)", + Obligations: "fold outputs are never replay sources (manifest cleared + compacted flag + token provenance — see the non-goals above)", + }, + "ingest_facts.go": { + Writes: "existence-bit fact markers (external-match grants seen, etc.)", + Obligations: "facts are monotone hints re-derived from the store at consuming seams (I2 repair); losing one is repaired, never silent", + }, + "ingest_repair.go": { + Writes: "dangling-reference drops (I7/I8/I9 arms) — ALL row deletion flows through dropBatcher", + Obligations: "the choke point: secondary indexes + digest invalidation derived from raw values; manifest invalidation rides the SAME batch " + + "as each scope's first delete; chunked commits with the crash hook between chunks; entitlement keyspace note keyed on landed commits", + PinnedBy: []string{ + "TestDanglingDropCrashBetweenChunksKeepsInvalidation", + "TestDanglingDropCleanCompletionInvalidates", + "TestManifestSnapshotInvalidationIsTyped", + }, + }, + "keyspace_version.go": { + Writes: "keyspace version stamp (engine-meta)", + Obligations: "written at store creation/migration boundaries only; no row obligations", + }, + "resource_types.go": { + Writes: "resource type rows (fresh ingestion)", + Obligations: "no secondary indexes or scope stamps; envelope save via wrappers + Mutated backstop", + }, + "resources.go": { + Writes: "resource primary rows + by_parent and by_source_scope indexes (fresh, replay, tombstones, I3 repair copy)", + Obligations: "indexes staged with the row; repair copy reuses the row's own index derivation", + }, + "session_store.go": { + Writes: "session key/value rows", + Obligations: "sync-scoped scratch state; no cross-cutting obligations beyond the envelope save", + }, + "source_cache.go": { + Writes: "source-cache manifest entries, replay copies (pre-clear + copy + recount), delta tombstones", + Obligations: "replay copy is the scope's first writer (resume pre-clear restores that precondition; ResumedRowsCleared marks the store dirty); " + + "post-copy recount fails partial copies before a manifest entry can seal them; tombstones delete rows + indexes together", + PinnedBy: []string{ + "TestSourceCacheReplayResumeAfterOverlayCommit", + "TestDeleteGrantsByPrincipalsInScope", + }, + }, + "sync_runs.go": { + Writes: "sync run records (start/end/compacted/token stamps)", + Obligations: "run records are the replay metadata gate (UsableAsReplaySource); compacted stamping is part of the fold's refusal contract", + }, + "sync_stats_sidecar.go": { + Writes: "stats sidecar records", + Obligations: "advisory (readers fall back to O(N) iteration); drop paths invalidate the deferred stash so counts never include dropped rows", + }, +} diff --git a/pkg/dotc1z/engine/pebble/mutation_paths_test.go b/pkg/dotc1z/engine/pebble/mutation_paths_test.go new file mode 100644 index 000000000..c88e0c96f --- /dev/null +++ b/pkg/dotc1z/engine/pebble/mutation_paths_test.go @@ -0,0 +1,111 @@ +package pebble + +// Enforcement for the mutation-path registry (mutation_paths.go): every +// non-test file in this package that touches a raw Pebble write +// primitive must carry a declaration, and every declaration must +// describe a file that still raw-writes. This is how a NEW mutation +// path is forced to state its obligations in the PR that adds it — the +// same mechanism sideEffectAnnotationCoverage uses for side-effecting +// annotations. + +import ( + "os" + "regexp" + "sort" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +// rawWriteSignals are the textual markers of raw Pebble write usage. +// Deliberately simple substring checks over source text: false +// positives just require a (cheap, documentation-bearing) declaration, +// while a regex clever enough to dodge comments would be a second +// implementation to keep correct. +var rawWriteSignals = []string{ + ".NewBatch(", + ".db.Set(", + ".db.Delete(", + "DeleteRange(", + ".db.Ingest(", + "IngestAndExcise(", + ".db.Apply(", +} + +func TestRawWriteRegistryCoversAllRawWritingFiles(t *testing.T) { + entries, err := os.ReadDir(".") + require.NoError(t, err) + + detected := map[string]bool{} + for _, entry := range entries { + name := entry.Name() + if entry.IsDir() || !strings.HasSuffix(name, ".go") || strings.HasSuffix(name, "_test.go") { + continue + } + src, err := os.ReadFile(name) + require.NoError(t, err) + for _, signal := range rawWriteSignals { + if strings.Contains(string(src), signal) { + detected[name] = true + break + } + } + } + + for _, name := range sortedNames(detected) { + _, declared := rawWriteRegistry[name] + require.True(t, declared, + "%s uses raw Pebble write primitives but has NO mutation-path declaration.\n"+ + "Add an entry to rawWriteRegistry (mutation_paths.go) stating what it writes and how it satisfies\n"+ + "the cross-cutting obligations (secondary indexes, digest invalidation, source-cache manifest,\n"+ + "envelope save, entitlement keyspace note, crash ordering). This is deliberate friction:\n"+ + "every undeclared mutation path in this package's history has shipped the same bug family.", + name) + } + for name := range rawWriteRegistry { + require.True(t, detected[name], + "rawWriteRegistry declares %s but it no longer uses raw write primitives (or was renamed); remove or update the stale entry", name) + } + + // Declarations must actually declare. + for name, decl := range rawWriteRegistry { + require.NotEmpty(t, decl.Writes, "registry entry %s: Writes must describe the mutation path", name) + require.NotEmpty(t, decl.Obligations, "registry entry %s: Obligations must state how (or why not) the cross-cutting obligations apply", name) + } +} + +func TestRawWriteRegistryPinnedByTestsExist(t *testing.T) { + testFuncs := map[string]bool{} + entries, err := os.ReadDir(".") + require.NoError(t, err) + testFuncRe := regexp.MustCompile(`(?m)^func (Test\w+)\(`) + for _, entry := range entries { + name := entry.Name() + if entry.IsDir() || !strings.HasSuffix(name, "_test.go") { + continue + } + src, err := os.ReadFile(name) + require.NoError(t, err) + for _, m := range testFuncRe.FindAllStringSubmatch(string(src), -1) { + testFuncs[m[1]] = true + } + } + + for name, decl := range rawWriteRegistry { + for _, pinned := range decl.PinnedBy { + require.True(t, testFuncs[pinned], + "registry entry %s names pinning test %q, which does not exist in this package; fix the name or move the pin into Obligations text (cross-package oracles)", + name, pinned) + } + } +} + +func sortedNames(set map[string]bool) []string { + out := make([]string, 0, len(set)) + for k := range set { + out = append(out, k) + } + sort.Strings(out) + return out +} diff --git a/pkg/dotc1z/engine/pebble/obligations_on_failure_test.go b/pkg/dotc1z/engine/pebble/obligations_on_failure_test.go new file mode 100644 index 000000000..036704cc2 --- /dev/null +++ b/pkg/dotc1z/engine/pebble/obligations_on_failure_test.go @@ -0,0 +1,253 @@ +package pebble + +// Table-driven harness for the recurring bug class of this branch: +// POST-MUTATION OBLIGATIONS SKIPPED ON THE ERROR PATH. Every mutation +// path that commits intermediate batches owes side effects the moment +// any commit lands — empty-keyspace fast-path disarm, bare-id lookup +// invalidation, manifest invalidation, stats-stash invalidation — and +// each has independently shipped a bug where the obligation only fired +// on full success (the replay pre-clear, the partial replay copy, the +// deferred-grant-stats stash). +// +// One table row per mutation path: force a failure AFTER the first +// landed chunk (via the replay batch var or the drop chunk hook) and +// assert every declared obligation fired anyway. A new mutation path +// gets a table row, not a bespoke test. + +import ( + "context" + "errors" + "fmt" + "testing" + + "github.com/stretchr/testify/require" + + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + "github.com/conductorone/baton-sdk/pkg/connectorstore" + "github.com/conductorone/baton-sdk/pkg/sourcecache" +) + +// obligationsCase drives one mutation path to a mid-flight failure. +// run must return the injected error (asserted non-nil); obligations +// assert the engine state the committed prefix demands. +type obligationsCase struct { + name string + run func(t *testing.T, ctx context.Context, cur, prev *Adapter, syncID string) error + obligations func(t *testing.T, e *Engine, syncID string) +} + +// plantDanglingGrants puts enough grants under one missing-referent +// shape to span at least one full drop chunk, all stamped with scopeA +// under a live manifest entry, so the chunk hook can fire mid-drop. +// varyEnt=true varies the entitlement per grant (one shared principal — +// the by_principal drop shape); varyEnt=false keeps one entitlement +// (the by_entitlement drop shape). +func plantDanglingGrants(t *testing.T, ctx context.Context, a *Adapter, varyEnt bool) { + t.Helper() + const total = dropBatchRows + 500 + scoped := sourcecache.WithScope(ctx, scopeA) + const putChunk = 1000 + for start := 0; start < total; start += putChunk { + end := min(start+putChunk, total) + page := make([]*v2.Grant, 0, end-start) + for i := start; i < end; i++ { + if varyEnt { + page = append(page, scGrant(fmt.Sprintf("member%05d", i), "dangler", false)) + } else { + page = append(page, scGrant("member", fmt.Sprintf("u%05d", i), false)) + } + } + require.NoError(t, a.PutGrants(scoped, page...)) + } + require.NoError(t, a.PebbleEngine().PutSourceCacheEntry(ctx, "grants", scopeA, "etag-v1")) +} + +func requireStashInvalidated(t *testing.T, e *Engine, syncID string) { + t.Helper() + require.Nil(t, e.takeDeferredGrantStats(syncID), + "committed drop chunks make the stashed grant counts stale; the stash must be invalidated even when the drop fails") +} + +func requireScopeAInvalidated(t *testing.T, e *Engine) { + t.Helper() + rec, err := e.GetSourceCacheEntry(context.Background(), "grants", scopeA) + require.NoError(t, err) + require.True(t, rec.GetInvalidated(), + "the scope losing rows must be invalidated with the first committed chunk") +} + +func TestFailedMutationPathsFireObligations(t *testing.T) { + injected := errors.New("injected mid-mutation failure") + + cases := []obligationsCase{ + { + // Replay pre-clear: crashed-attempt residue deleted, then the + // copy is cancelled. The empty-keyspace fast path must be + // disarmed by the landed deletes alone. + name: "replay-grants-preclear", + run: func(t *testing.T, ctx context.Context, cur, prev *Adapter, _ string) error { + e := cur.PebbleEngine() + for i := 0; i < 3; i++ { + plantScopedGrant(t, e, scopeA, fmt.Sprintf("u%d", i)) + } + cctx := &failAfterNChecks{Context: ctx, remaining: 1} + _, err := e.ReplaySourceCacheGrants(cctx, prev.PebbleEngine(), scopeA) + return err + }, + obligations: func(t *testing.T, e *Engine, _ string) { + require.False(t, e.takeFreshGrantsEmpty(), + "landed pre-clear deletes must disarm the grants fast path") + }, + }, + { + name: "replay-entitlements-preclear", + run: func(t *testing.T, ctx context.Context, cur, prev *Adapter, _ string) error { + e := cur.PebbleEngine() + for i := 0; i < 3; i++ { + plantScopedEntitlement(t, e, scopeA, fmt.Sprintf("g%d", i)) + } + cctx := &failAfterNChecks{Context: ctx, remaining: 1} + _, err := e.ReplaySourceCacheEntitlements(cctx, prev.PebbleEngine(), scopeA) + return err + }, + obligations: func(t *testing.T, e *Engine, _ string) { + require.False(t, e.takeFreshEntitlementsEmpty(), + "landed pre-clear deletes must disarm the entitlements fast path (and bump the bare-id generation)") + }, + }, + { + name: "replay-resources-preclear", + run: func(t *testing.T, ctx context.Context, cur, prev *Adapter, _ string) error { + e := cur.PebbleEngine() + for i := 0; i < 3; i++ { + plantScopedResource(t, e, scopeA, fmt.Sprintf("u%d", i)) + } + cctx := &failAfterNChecks{Context: ctx, remaining: 1} + _, err := e.ReplaySourceCacheResources(cctx, prev.PebbleEngine(), scopeA) + return err + }, + obligations: func(t *testing.T, e *Engine, _ string) { + require.False(t, e.takeFreshResourcesEmpty(), + "landed pre-clear deletes must disarm the resources fast path") + }, + }, + { + // I8's drop arm: one entitlement, > one chunk of grants. Fail + // after the first committed chunk. + name: "drop-grants-for-entitlement", + run: func(t *testing.T, ctx context.Context, cur, prev *Adapter, syncID string) error { + e := cur.PebbleEngine() + plantDanglingGrants(t, ctx, cur, false) + e.stashDeferredGrantStats(&deferredGrantStats{syncID: syncID, grants: 999_999}) + e.testDropChunkHook = func() error { return injected } + t.Cleanup(func() { e.testDropChunkHook = nil }) + _, _, err := e.DeleteGrantsForEntitlement(ctx, "group:g1:member", "group", "g1") + return err + }, + obligations: func(t *testing.T, e *Engine, syncID string) { + requireScopeAInvalidated(t, e) + requireStashInvalidated(t, e, syncID) + }, + }, + { + // I9's drop arm: one principal, > one chunk of grants. + name: "drop-grants-for-principal", + run: func(t *testing.T, ctx context.Context, cur, prev *Adapter, syncID string) error { + e := cur.PebbleEngine() + plantDanglingGrants(t, ctx, cur, true) + e.stashDeferredGrantStats(&deferredGrantStats{syncID: syncID, grants: 999_999}) + e.testDropChunkHook = func() error { return injected } + t.Cleanup(func() { e.testDropChunkHook = nil }) + _, _, err := e.DeleteGrantsForPrincipal(ctx, "user", "dangler") + return err + }, + obligations: func(t *testing.T, e *Engine, syncID string) { + requireScopeAInvalidated(t, e) + requireStashInvalidated(t, e, syncID) + }, + }, + { + // H1 residual: endSyncFinalize stamps the in-memory record + // (existing.SetEndedAt) BEFORE PutSyncRunRecord commits it. If + // that commit fails, the stamp must leak nowhere — the stored + // record stays unstamped and the sync stays discoverable as + // unfinished. Safe today because `existing` is a local copy + // unmarshalled per EndSync call; this row keeps it safe against + // a future record cache. + name: "endsync-stamp-commit-failure", + run: func(t *testing.T, ctx context.Context, cur, prev *Adapter, _ string) error { + e := cur.PebbleEngine() + require.NoError(t, cur.PutGrants(ctx, mkV2Grant("g1", "group:g1:member", "user", "alice"))) + e.testEndSyncStampHook = func() error { return injected } + t.Cleanup(func() { e.testEndSyncStampHook = nil }) + return cur.EndSync(ctx) + }, + obligations: func(t *testing.T, e *Engine, syncID string) { + rec, err := e.GetSyncRunRecord(context.Background(), syncID) + require.NoError(t, err) + require.Nil(t, rec.GetEndedAt(), + "a failed stamp commit must not surface ended_at; the in-memory stamp on the local record must be unreachable") + unfinished, err := e.LatestUnfinishedSyncRecord(context.Background(), nil) + require.NoError(t, err) + require.NotNil(t, unfinished, + "the sync must stay discoverable as unfinished after a failed stamp commit") + require.Equal(t, syncID, unfinished.GetSyncId()) + }, + }, + { + // I7's drop arm: entitlements under one resource. Obligation: + // the bare-id keyspace note (observable via the fresh flag — + // the delete mutated the keyspace, so the flag must be gone). + name: "drop-entitlements-for-resource", + run: func(t *testing.T, ctx context.Context, cur, prev *Adapter, _ string) error { + e := cur.PebbleEngine() + const total = dropBatchRows + 500 + const putChunk = 1000 + for start := 0; start < total; start += putChunk { + end := min(start+putChunk, total) + page := make([]*v2.Entitlement, 0, end-start) + for i := start; i < end; i++ { + page = append(page, v2.Entitlement_builder{ + Id: fmt.Sprintf("group:g1:member%05d", i), + Resource: v2.Resource_builder{ + Id: v2.ResourceId_builder{ResourceType: "group", Resource: "g1"}.Build(), + }.Build(), + }.Build()) + } + require.NoError(t, cur.PutEntitlements(sourcecache.WithScope(ctx, scopeA), page...)) + } + require.NoError(t, e.PutSourceCacheEntry(ctx, "entitlements", scopeA, "etag-v1")) + e.testDropChunkHook = func() error { return injected } + t.Cleanup(func() { e.testDropChunkHook = nil }) + _, _, err := e.DeleteEntitlementsForResource(ctx, "group", "g1", 5) + return err + }, + obligations: func(t *testing.T, e *Engine, _ string) { + rec, err := e.GetSourceCacheEntry(context.Background(), "entitlements", scopeA) + require.NoError(t, err) + require.True(t, rec.GetInvalidated(), + "the scope losing entitlement rows must be invalidated with the first committed chunk") + }, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + ctx := context.Background() + oldBatch := replayBatchRows + replayBatchRows = 1 + t.Cleanup(func() { replayBatchRows = oldBatch }) + + prev := newAdapter(t) + _, err := prev.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + cur := newAdapter(t) + syncID, err := cur.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + + err = tc.run(t, ctx, cur, prev, syncID) + require.Error(t, err, "the case must fail mid-mutation; a clean run proves nothing about error-path obligations") + tc.obligations(t, cur.PebbleEngine(), syncID) + }) + } +} diff --git a/pkg/dotc1z/engine/pebble/raw_records.go b/pkg/dotc1z/engine/pebble/raw_records.go index b56c4c439..307172cd8 100644 --- a/pkg/dotc1z/engine/pebble/raw_records.go +++ b/pkg/dotc1z/engine/pebble/raw_records.go @@ -3,11 +3,15 @@ package pebble import ( "fmt" "math" + "strings" "time" "github.com/cockroachdb/pebble/v2" "google.golang.org/protobuf/encoding/protowire" + "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/timestamppb" + + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" ) const ( @@ -110,18 +114,25 @@ func rawTimestampNanos(value []byte) (int64, error) { } func (e *Engine) deleteResourceIndexesRaw(batch *pebble.Batch, resourceTypeID string, resourceID string, value []byte) error { - parentRT, parentID, err := scanResourceParentRaw(value) + parentRT, parentID, sourceScopeKey, err := scanResourceIndexFieldsRaw(value) if err != nil { return err } - if parentID == "" { - return nil + if parentID != "" { + if err := batch.Delete(encodeResourceByParentIndexKey(parentRT, parentID, resourceTypeID, resourceID), nil); err != nil { + return err + } } - return batch.Delete(encodeResourceByParentIndexKey(parentRT, parentID, resourceTypeID, resourceID), nil) + if sourceScopeKey != "" { + if err := batch.Delete(encodeResourceBySourceScopeIndexKey(sourceScopeKey, resourceTypeID, resourceID), nil); err != nil { + return err + } + } + return nil } func (e *Engine) deleteGrantIndexesRaw(batch *pebble.Batch, externalID string, value []byte) error { - entRT, entRID, entID, principalRT, principalID, _, err := scanGrantIndexFieldsRaw(value) + entRT, entRID, entID, principalRT, principalID, _, sourceScopeKey, err := scanGrantIndexFieldsRaw(value) if err != nil { return err } @@ -142,6 +153,11 @@ func (e *Engine) deleteGrantIndexesRaw(batch *pebble.Batch, externalID string, v if err := e.stageGrantDigestInvalidation(batch, id.entitlement); err != nil { return err } + if sourceScopeKey != "" { + if err := batch.Delete(encodeGrantBySourceScopeIndexKey(sourceScopeKey, id), nil); err != nil { + return err + } + } return batch.Delete(encodeGrantByNeedsExpansionIdentityIndexKey(id), nil) } @@ -319,37 +335,83 @@ func scanEntitlementResourceTypeRaw(value []byte) ([]byte, error) { // occurrence of the target field, matching scanGrantIndexFieldsRaw and // approximating proto merge semantics. Values written by this SDK carry // at most one occurrence, so this only matters for foreign writers. -func scanResourceParentRaw(value []byte) (string, string, error) { - var rt, id string +// scanResourceIndexFieldsRaw extracts the parent ref (field 6) and +// source_scope_key (field 9) from a marshaled ResourceRecord — the +// fields that key resource secondary indexes. +func scanResourceIndexFieldsRaw(value []byte) (string, string, string, error) { + var rt, id, sourceScopeKey string for len(value) > 0 { num, typ, n := protowire.ConsumeTag(value) if n < 0 { - return "", "", protowire.ParseError(n) + return "", "", "", protowire.ParseError(n) + } + value = value[n:] + switch num { + case 6: + if typ != protowire.BytesType { + return "", "", "", fmt.Errorf("raw record: resource parent has wire type %v", typ) + } + msg, n := protowire.ConsumeBytes(value) + if n < 0 { + return "", "", "", protowire.ParseError(n) + } + var err error + rt, id, err = scanResourceRefRaw(msg) + if err != nil { + return "", "", "", err + } + value = value[n:] + case 9: + if typ != protowire.BytesType { + return "", "", "", fmt.Errorf("raw record: resource source_scope_key has wire type %v", typ) + } + s, n := protowire.ConsumeBytes(value) + if n < 0 { + return "", "", "", protowire.ParseError(n) + } + sourceScopeKey = string(s) + value = value[n:] + default: + n = protowire.ConsumeFieldValue(num, typ, value) + if n < 0 { + return "", "", "", protowire.ParseError(n) + } + value = value[n:] + } + } + return rt, id, sourceScopeKey, nil +} + +// scanEntitlementSourceScopeRaw extracts source_scope_key (field 11) +// from a marshaled EntitlementRecord. Keys the entitlement +// by_source_scope index — the only entitlement secondary index. +func scanEntitlementSourceScopeRaw(value []byte) (string, error) { + var sourceScopeKey string + for len(value) > 0 { + num, typ, n := protowire.ConsumeTag(value) + if n < 0 { + return "", protowire.ParseError(n) } value = value[n:] - if num != 6 { + if num != 11 { n = protowire.ConsumeFieldValue(num, typ, value) if n < 0 { - return "", "", protowire.ParseError(n) + return "", protowire.ParseError(n) } value = value[n:] continue } if typ != protowire.BytesType { - return "", "", fmt.Errorf("raw record: resource parent has wire type %v", typ) + return "", fmt.Errorf("raw record: entitlement source_scope_key has wire type %v", typ) } - msg, n := protowire.ConsumeBytes(value) + s, n := protowire.ConsumeBytes(value) if n < 0 { - return "", "", protowire.ParseError(n) - } - var err error - rt, id, err = scanResourceRefRaw(msg) - if err != nil { - return "", "", err + return "", protowire.ParseError(n) } + sourceScopeKey = string(s) value = value[n:] } - return rt, id, nil + return sourceScopeKey, nil } func scanEntitlementResourceRaw(value []byte) (string, string, error) { @@ -429,63 +491,73 @@ func scanEntitlementIdentityFieldsRaw(value []byte) (string, string, string, err return rt, id, externalID, nil } -func scanGrantIndexFieldsRaw(value []byte) (string, string, string, string, string, bool, error) { - var entRT, entRID, entID, principalRT, principalID string +func scanGrantIndexFieldsRaw(value []byte) (string, string, string, string, string, bool, string, error) { + var entRT, entRID, entID, principalRT, principalID, sourceScopeKey string var needsExpansion bool for len(value) > 0 { num, typ, n := protowire.ConsumeTag(value) if n < 0 { - return "", "", "", "", "", false, protowire.ParseError(n) + return "", "", "", "", "", false, "", protowire.ParseError(n) } value = value[n:] switch num { case 3: if typ != protowire.BytesType { - return "", "", "", "", "", false, fmt.Errorf("raw record: grant entitlement has wire type %v", typ) + return "", "", "", "", "", false, "", fmt.Errorf("raw record: grant entitlement has wire type %v", typ) } msg, n := protowire.ConsumeBytes(value) if n < 0 { - return "", "", "", "", "", false, protowire.ParseError(n) + return "", "", "", "", "", false, "", protowire.ParseError(n) } var err error entRT, entRID, entID, err = scanEntitlementRefRaw(msg) if err != nil { - return "", "", "", "", "", false, err + return "", "", "", "", "", false, "", err } value = value[n:] case 4: if typ != protowire.BytesType { - return "", "", "", "", "", false, fmt.Errorf("raw record: grant principal has wire type %v", typ) + return "", "", "", "", "", false, "", fmt.Errorf("raw record: grant principal has wire type %v", typ) } msg, n := protowire.ConsumeBytes(value) if n < 0 { - return "", "", "", "", "", false, protowire.ParseError(n) + return "", "", "", "", "", false, "", protowire.ParseError(n) } var err error principalRT, principalID, err = scanPrincipalRefRaw(msg) if err != nil { - return "", "", "", "", "", false, err + return "", "", "", "", "", false, "", err } value = value[n:] case 7: if typ != protowire.VarintType { - return "", "", "", "", "", false, fmt.Errorf("raw record: grant needs_expansion has wire type %v", typ) + return "", "", "", "", "", false, "", fmt.Errorf("raw record: grant needs_expansion has wire type %v", typ) } v, n := protowire.ConsumeVarint(value) if n < 0 { - return "", "", "", "", "", false, protowire.ParseError(n) + return "", "", "", "", "", false, "", protowire.ParseError(n) } needsExpansion = v != 0 value = value[n:] + case 10: + if typ != protowire.BytesType { + return "", "", "", "", "", false, "", fmt.Errorf("raw record: grant source_scope_key has wire type %v", typ) + } + s, n := protowire.ConsumeBytes(value) + if n < 0 { + return "", "", "", "", "", false, "", protowire.ParseError(n) + } + sourceScopeKey = string(s) + value = value[n:] default: n = protowire.ConsumeFieldValue(num, typ, value) if n < 0 { - return "", "", "", "", "", false, protowire.ParseError(n) + return "", "", "", "", "", false, "", protowire.ParseError(n) } value = value[n:] } } - return entRT, entRID, entID, principalRT, principalID, needsExpansion, nil + return entRT, entRID, entID, principalRT, principalID, needsExpansion, sourceScopeKey, nil } // scanGrantNeedsExpansionRaw extracts only the needs_expansion flag @@ -592,3 +664,132 @@ func scanResourceRefRawBytes(value []byte, set func(protowire.Number, []byte)) e } return nil } + +// Annotation message names (the tail of an Any type_url) that drive +// replay side effects. Matched on the segment after the last '/' so +// non-default type-url prefixes still resolve. +const ( + anyTypeChildResourceType = "c1.connector.v2.ChildResourceType" + anyTypeInsertResourceGrants = "c1.connector.v2.InsertResourceGrants" + anyTypeExternalResourceMatch = "c1.connector.v2.ExternalResourceMatch" + anyTypeExternalResourceMatchAll = "c1.connector.v2.ExternalResourceMatchAll" + anyTypeExternalResourceMatchID = "c1.connector.v2.ExternalResourceMatchID" +) + +// scanAnnotationAnysRaw walks a record's repeated-Any annotations field +// (fieldNum) without unmarshaling the record, invoking visit with each +// Any's message type name and value bytes. Used by the source-cache +// replay paths to detect side-effect annotations (ChildResourceType, +// InsertResourceGrants, ExternalResourceMatch*) on raw-copied rows. +func scanAnnotationAnysRaw(value []byte, fieldNum protowire.Number, visit func(typeName string, payload []byte) error) error { + for len(value) > 0 { + num, typ, n := protowire.ConsumeTag(value) + if n < 0 { + return protowire.ParseError(n) + } + value = value[n:] + if num != fieldNum { + n = protowire.ConsumeFieldValue(num, typ, value) + if n < 0 { + return protowire.ParseError(n) + } + value = value[n:] + continue + } + if typ != protowire.BytesType { + return fmt.Errorf("raw record: annotations field has wire type %v", typ) + } + anyMsg, n := protowire.ConsumeBytes(value) + if n < 0 { + return protowire.ParseError(n) + } + value = value[n:] + + var typeURL string + var payload []byte + rest := anyMsg + for len(rest) > 0 { + anum, atyp, an := protowire.ConsumeTag(rest) + if an < 0 { + return protowire.ParseError(an) + } + rest = rest[an:] + if atyp != protowire.BytesType { + an = protowire.ConsumeFieldValue(anum, atyp, rest) + if an < 0 { + return protowire.ParseError(an) + } + rest = rest[an:] + continue + } + b, an := protowire.ConsumeBytes(rest) + if an < 0 { + return protowire.ParseError(an) + } + switch anum { + case 1: + typeURL = string(b) + case 2: + payload = b + default: + } + rest = rest[an:] + } + typeName := typeURL + if i := strings.LastIndexByte(typeURL, '/'); i >= 0 { + typeName = typeURL[i+1:] + } + if typeName == "" { + continue + } + if err := visit(typeName, payload); err != nil { + return err + } + } + return nil +} + +// scanResourceChildTypeIDsRaw returns the resource_type_ids named by +// ChildResourceType annotations on a marshaled ResourceRecord (field 7). +// Nil for the common case of a resource with no child types. +func scanResourceChildTypeIDsRaw(value []byte) ([]string, error) { + var out []string + err := scanAnnotationAnysRaw(value, 7, func(typeName string, payload []byte) error { + if typeName != anyTypeChildResourceType { + return nil + } + crt := &v2.ChildResourceType{} + if err := proto.Unmarshal(payload, crt); err != nil { + return fmt.Errorf("raw record: unmarshal ChildResourceType annotation: %w", err) + } + if id := crt.GetResourceTypeId(); id != "" { + out = append(out, id) + } + return nil + }) + if err != nil { + return nil, err + } + return out, nil +} + +// grantSourceCacheFlags reports which replay side-effect annotations a +// marshaled GrantRecord carries (annotations field 8). +type grantSourceCacheFlags struct { + insertResourceGrants bool + externalResourceMatch bool +} + +func scanGrantSourceCacheFlagsRaw(value []byte) (grantSourceCacheFlags, error) { + var f grantSourceCacheFlags + err := scanAnnotationAnysRaw(value, 8, func(typeName string, _ []byte) error { + switch typeName { + case anyTypeInsertResourceGrants: + f.insertResourceGrants = true + case anyTypeExternalResourceMatch, anyTypeExternalResourceMatchAll, anyTypeExternalResourceMatchID: + f.externalResourceMatch = true + } + return nil + }) + return f, err +} diff --git a/pkg/dotc1z/engine/pebble/resources.go b/pkg/dotc1z/engine/pebble/resources.go index 564a0430a..6a73356d6 100644 --- a/pkg/dotc1z/engine/pebble/resources.go +++ b/pkg/dotc1z/engine/pebble/resources.go @@ -124,8 +124,12 @@ func (e *Engine) GetResourceRecord(ctx context.Context, resourceTypeID, resource return r, nil } -func (e *Engine) DeleteResourceRecord(ctx context.Context, resourceTypeID, resourceID string) error { - return e.withWrite(func() error { +// DeleteResourceRecord deletes one resource row and its index entries. +// Returns whether a row existed (a missing row is a no-op, not an error — +// tombstone semantics). +func (e *Engine) DeleteResourceRecord(ctx context.Context, resourceTypeID, resourceID string) (bool, error) { + deleted := false + err := e.withWrite(func() error { key := encodeResourceKey(resourceTypeID, resourceID) batch := e.db.NewBatch() @@ -146,20 +150,32 @@ func (e *Engine) DeleteResourceRecord(ctx context.Context, resourceTypeID, resou if err := batch.Delete(key, nil); err != nil { return err } - return batch.Commit(writeOpts(e.opts.durability)) + if err := batch.Commit(writeOpts(e.opts.durability)); err != nil { + return err + } + deleted = true + return nil }) + return deleted, err } func (e *Engine) writeResourceIndexes(batch *pebble.Batch, r *v3.ResourceRecord) error { - parent := r.GetParent() - if parent == nil || parent.GetResourceId() == "" { - return nil + if parent := r.GetParent(); parent != nil && parent.GetResourceId() != "" { + k := encodeResourceByParentIndexKey( + parent.GetResourceTypeId(), parent.GetResourceId(), + r.GetResourceTypeId(), r.GetResourceId(), + ) + if err := batch.Set(k, nil, nil); err != nil { + return err + } + } + if sh := r.GetSourceScopeKey(); sh != "" { + k := encodeResourceBySourceScopeIndexKey(sh, r.GetResourceTypeId(), r.GetResourceId()) + if err := batch.Set(k, nil, nil); err != nil { + return err + } } - k := encodeResourceByParentIndexKey( - parent.GetResourceTypeId(), parent.GetResourceId(), - r.GetResourceTypeId(), r.GetResourceId(), - ) - return batch.Set(k, nil, nil) + return nil } func (e *Engine) IterateResources(ctx context.Context, yield func(*v3.ResourceRecord) bool) error { diff --git a/pkg/dotc1z/engine/pebble/source_cache.go b/pkg/dotc1z/engine/pebble/source_cache.go new file mode 100644 index 000000000..3f450a1db --- /dev/null +++ b/pkg/dotc1z/engine/pebble/source_cache.go @@ -0,0 +1,1525 @@ +package pebble + +import ( + "context" + "errors" + "fmt" + "sort" + + "github.com/cockroachdb/pebble/v2" + "google.golang.org/protobuf/encoding/protowire" + "google.golang.org/protobuf/types/known/timestamppb" + + v3 "github.com/conductorone/baton-sdk/pb/c1/storage/v3" + "github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/codec" +) + +// Source-cache replay, engine side. +// +// The typeSourceCache keyspace holds one SourceCacheEntryRecord per +// (row_kind, scope_key): the opaque upstream validator (etag / delta +// token) the sync recorded for that scope. Rows produced under a scope +// are stamped with source_scope_key and indexed under the +// by_source_scope families, whose tails are identity tuples — so a +// replay derives every primary key from the index key alone and copies +// raw values across files without a proto unmarshal. +// +// The previous sync lives in a separate read-only engine (a Pebble c1z +// holds exactly one sync); replay copies from prev into the receiver. + +// replayBatchRows bounds how many rows accumulate in one pebble.Batch +// before an intermediate commit. Replay of a delta-query collection can +// be the whole previous row set, so the batch must not grow unbounded. +// A var (not const) so tests can shrink it to exercise the +// partial-commit failure windows deterministically. +var replayBatchRows = 10_000 + +// ReplayedParentResource identifies one replayed resource whose +// annotations carry ChildResourceType — the syncer must schedule child +// SyncResources actions for it, since a replayed page returns no rows +// and the response-row path that normally schedules children never +// sees these parents. +type ReplayedParentResource struct { + ResourceTypeID string + ResourceID string + ChildTypeIDs []string +} + +// SourceCacheReplayResult reports what one scope's replay copied. +// +// NOTE on side effects: replayed rows contribute to the syncer's +// side-effect state through STORE-DERIVED mechanisms, not through fields +// here — the needs_expansion index and the external-match existence bit +// are maintained for replayed rows exactly as for fresh ones, and the +// syncer's ingestion invariants read the store (see +// docs/tasks/source-cache-ingestion-invariants.md). The two fields that +// remain (ChildResources, RelatedResourceRows) are MECHANISMS, not +// evidence: child scheduling and related-resource recreation cannot be +// derived from the store after the fact, so replay must carry them. +type SourceCacheReplayResult struct { + Rows int64 + // StaleSkipped counts index entries under the scope that did NOT + // yield a copied row: the primary was missing, or its value stamp + // named a different scope. This is the discriminator between "scope + // legitimately empty" (index prefix empty, StaleSkipped == 0) and + // "scope's rows were clobbered without index cleanup" (index says + // rows existed, none survived the stamp check). The syncer fails a + // replay that copies zero rows while StaleSkipped > 0. + StaleSkipped int64 + // ChildResources lists replayed parents carrying ChildResourceType + // annotations (resources replay only). See ReplayedParentResource. + ChildResources []ReplayedParentResource + // RelatedResourceRows counts resource rows copied from prev because + // a replayed grant carried InsertResourceGrants (grants replay + // only). Those resources exist ONLY as grant-driven insertions — + // no ListResources response ever returns them — so the replay must + // recreate them or the current sync holds grants whose resources + // do not exist. + RelatedResourceRows int64 + // ResumedRowsCleared counts destination rows deleted before the copy + // because they were already stamped with the scope — evidence of a + // resumed re-run over a crashed attempt's committed replay/overlay + // rows (see clearReplayScopeRowsLocked). Always zero on a page's + // first run. + ResumedRowsCleared int64 +} + +// SourceCacheManifestEntry is one manifest entry as reported by +// SourceCacheManifestSnapshot. Typed on purpose: the validator is an +// OPAQUE connector string, so invalidation must be a separate field — +// any in-band sentinel encoding could collide with a legitimate +// validator and misclassify it. +type SourceCacheManifestEntry struct { + // CacheValidator is the stored upstream validator (kept even on + // invalidated entries, for the audit surface). + CacheValidator string + // Invalidated reports the dangling-reference drops marked this scope + // (see stageSourceCacheScopeInvalidationLocked): lookups miss it and + // the scope re-fetches cold next sync. + Invalidated bool +} + +// SourceCacheManifestSnapshot returns every manifest entry as +// "row_kind\x00scope_key" -> SourceCacheManifestEntry. Inspection +// surface: the replay-equivalence harness compares a warm sync's +// manifest against a cold sync's (they must be identical — same scopes, +// same validators, same invalidation verdicts), catching phantom or +// missing entries that are invisible through the v2 read API but poison +// FUTURE syncs' replays. +func (e *Engine) SourceCacheManifestSnapshot(ctx context.Context) (map[string]SourceCacheManifestEntry, error) { + iter, err := e.db.NewIter(&pebble.IterOptions{ + LowerBound: encodeSourceCacheManifestPrefix(), + UpperBound: upperBoundOf(encodeSourceCacheManifestPrefix()), + }) + if err != nil { + return nil, err + } + defer iter.Close() + out := map[string]SourceCacheManifestEntry{} + for iter.First(); iter.Valid(); iter.Next() { + if err := ctx.Err(); err != nil { + return nil, err + } + rec := &v3.SourceCacheEntryRecord{} + if err := unmarshalRecord(iter.Value(), rec); err != nil { + return nil, fmt.Errorf("SourceCacheManifestSnapshot: unmarshal: %w", err) + } + out[rec.GetRowKind()+"\x00"+rec.GetScopeKey()] = SourceCacheManifestEntry{ + CacheValidator: rec.GetCacheValidator(), + Invalidated: rec.GetInvalidated(), + } + } + return out, iter.Error() +} + +// SourceScopeIndexSnapshot returns, per row kind, scope_key -> the +// number of by_source_scope index entries. Same inspection rationale as +// SourceCacheManifestSnapshot: stamp/index divergence between a warm and +// a cold sync is invisible to v2 reads until a later sync replays from +// the damaged file. +func (e *Engine) SourceScopeIndexSnapshot(ctx context.Context) (map[string]map[string]int, error) { + out := map[string]map[string]int{} + families := []struct { + kind string + lower []byte + }{ + {"resources", ResourceBySourceScopeLowerBound()}, + {"entitlements", EntitlementBySourceScopeLowerBound()}, + {"grants", GrantBySourceScopeLowerBound()}, + } + for _, fam := range families { + prefix := codec.AppendTupleSeparator(append([]byte{}, fam.lower...)) + iter, err := e.db.NewIter(&pebble.IterOptions{ + LowerBound: prefix, + UpperBound: upperBoundOf(fam.lower), + }) + if err != nil { + return nil, err + } + counts := map[string]int{} + for iter.First(); iter.Valid(); iter.Next() { + if err := ctx.Err(); err != nil { + iter.Close() + return nil, err + } + tail := iter.Key()[len(prefix):] + scope, _, ok := codec.DecodeTupleStringAlias(tail, 0) + if !ok { + iter.Close() + return nil, fmt.Errorf("SourceScopeIndexSnapshot: malformed %s index key %x", fam.kind, iter.Key()) + } + counts[string(scope)]++ + } + if err := iter.Error(); err != nil { + iter.Close() + return nil, err + } + iter.Close() + out[fam.kind] = counts + } + return out, nil +} + +// countSourceScopeIndexRange counts the index entries under one scope's +// by_source_scope prefix — a contiguous key-only scan, no value reads. +func (e *Engine) countSourceScopeIndexRange(ctx context.Context, prefix []byte) (int64, error) { + iter, err := e.db.NewIter(&pebble.IterOptions{ + LowerBound: prefix, + UpperBound: upperBoundOf(prefix), + }) + if err != nil { + return 0, err + } + defer iter.Close() + var n int64 + for iter.First(); iter.Valid(); iter.Next() { + if err := ctx.Err(); err != nil { + return 0, err + } + n++ + } + return n, iter.Error() +} + +// verifyReplayedScopeCount is the replay-time count check: immediately +// after a replay's final commit, the destination's scope-index range must +// hold exactly the rows the copy claims to have written. Sound because +// the copy is the scope's first writer in the sync — fresh rows for a +// replayed scope arrive only via the overlay, after this, and a resumed +// re-run first clears whatever a crashed attempt committed under the +// scope (clearReplayScopeRowsLocked), including overlay rows whose +// identities the re-copy would not cover. A mismatch means a partial +// copy landed — the silent-bad-data class — so it fails the replay +// rather than letting the manifest entry seal it as complete. +func (e *Engine) verifyReplayedScopeCount(ctx context.Context, kind, scopeKey string, prefix []byte, copied int64) error { + got, err := e.countSourceScopeIndexRange(ctx, prefix) + if err != nil { + return fmt.Errorf("source cache replay: recount %s scope %q: %w", kind, scopeKey, err) + } + if got != copied { + return fmt.Errorf( + "source cache replay: %s scope %q index holds %d entries after copy but %d rows were copied (partial replay copy)", + kind, scopeKey, got, copied) + } + return nil +} + +// SourceCacheOrphanScopes returns, per row kind, every scope key present +// in a by_source_scope index that has NO manifest entry. At a sealed +// sync's quiesce point this is an invariant violation: every stamped row +// was written under a scope whose completion writes the manifest entry, +// and post-processing (external match, expansion) only re-stamps rows +// with scopes that already completed. An orphan means a manifest write +// was lost or a stamp leaked — either would poison the NEXT sync's +// replay. One skip-scan per distinct scope; O(distinct scopes) seeks. +func (e *Engine) SourceCacheOrphanScopes(ctx context.Context) (map[string][]string, error) { + out := map[string][]string{} + families := []struct { + kind string + lower []byte + prefix func(scope string) []byte + }{ + {"resources", ResourceBySourceScopeLowerBound(), encodeResourceBySourceScopePrefix}, + {"entitlements", EntitlementBySourceScopeLowerBound(), encodeEntitlementBySourceScopePrefix}, + {"grants", GrantBySourceScopeLowerBound(), encodeGrantBySourceScopePrefix}, + } + for _, fam := range families { + famPrefix := codec.AppendTupleSeparator(append([]byte{}, fam.lower...)) + iter, err := e.db.NewIter(&pebble.IterOptions{ + LowerBound: famPrefix, + UpperBound: upperBoundOf(fam.lower), + }) + if err != nil { + return nil, err + } + for valid := iter.First(); valid; { + if err := ctx.Err(); err != nil { + iter.Close() + return nil, err + } + tail := iter.Key()[len(famPrefix):] + scopeBytes, _, ok := codec.DecodeTupleStringAlias(tail, 0) + if !ok { + iter.Close() + return nil, fmt.Errorf("SourceCacheOrphanScopes: malformed %s index key %x", fam.kind, iter.Key()) + } + scope := string(scopeBytes) + if _, closer, getErr := e.db.Get(encodeSourceCacheEntryKey(fam.kind, scope)); getErr == nil { + closer.Close() + } else if errors.Is(getErr, pebble.ErrNotFound) { + out[fam.kind] = append(out[fam.kind], scope) + } else { + iter.Close() + return nil, getErr + } + // Skip the rest of this scope's entries. + valid = iter.SeekGE(upperBoundOf(fam.prefix(scope))) + } + if err := iter.Error(); err != nil { + iter.Close() + return nil, err + } + iter.Close() + } + if len(out) == 0 { + return nil, nil + } + for _, scopes := range out { + sort.Strings(scopes) + } + return out, nil +} + +// ClearSourceCacheEntries removes every source-cache manifest entry AND +// the replay-compatibility record (the range covers the whole +// typeSourceCache prefix). Compaction calls this on fold outputs: a +// keep-newer upsert merge is not a faithful snapshot of ANY input sync +// (base rows the newer sync deleted survive), so no input's validator +// may be allowed to 304-validate the merged row set. With the manifest +// empty, every lookup against the artifact misses and the next sync +// fetches fresh — cold-correct. The scope stamps and scope-index entries +// on rows are left in place: replay only reads them after a manifest +// hit, which can no longer happen. +func (e *Engine) ClearSourceCacheEntries(ctx context.Context) error { + return e.withWrite(func() error { + if err := e.requireCurrentSync(); err != nil { + return err + } + return e.db.DeleteRange(SourceCacheEntryLowerBound(), SourceCacheEntryUpperBound(), writeOpts(e.opts.durability)) + }) +} + +// PutSourceCacheEntry writes the manifest entry for (rowKind, scopeKey). +// Zero-row scopes still get entries — the validator must survive to the +// next sync even when the scope produced no rows. +func (e *Engine) PutSourceCacheEntry(ctx context.Context, rowKind, scopeKey, cacheValidator string) error { + return e.withWrite(func() error { + if err := e.requireCurrentSync(); err != nil { + return err + } + rec := &v3.SourceCacheEntryRecord{} + rec.SetRowKind(rowKind) + rec.SetScopeKey(scopeKey) + rec.SetCacheValidator(cacheValidator) + rec.SetDiscoveredAt(timestamppb.Now()) + val, err := marshalRecord(rec) + if err != nil { + return err + } + opts := writeOpts(e.opts.durability) + if e.IsFreshSync() { + opts = pebble.NoSync + } + return e.db.Set(encodeSourceCacheEntryKey(rowKind, scopeKey), val, opts) + }) +} + +// PutSourceCacheCompat writes the sync's replay-compatibility key record +// (singleton; see SourceCacheCompatRecord). Written when the source +// cache's write side enables — BEFORE any manifest entry — so an +// artifact can never hold validators without the key that scopes their +// validity. Idempotent overwrite on resume. +func (e *Engine) PutSourceCacheCompat(ctx context.Context, rec *v3.SourceCacheCompatRecord) error { + return e.withWrite(func() error { + if err := e.requireCurrentSync(); err != nil { + return err + } + rec.SetId("compat") + val, err := marshalRecord(rec) + if err != nil { + return err + } + opts := writeOpts(e.opts.durability) + if e.IsFreshSync() { + opts = pebble.NoSync + } + return e.db.Set(encodeSourceCacheCompatKey(), val, opts) + }) +} + +// GetSourceCacheCompat returns the sync's replay-compatibility key +// record, or pebble.ErrNotFound (pre-key artifacts, folds). +func (e *Engine) GetSourceCacheCompat(ctx context.Context) (*v3.SourceCacheCompatRecord, error) { + val, closer, err := e.db.Get(encodeSourceCacheCompatKey()) + if err != nil { + return nil, err + } + defer closer.Close() + rec := &v3.SourceCacheCompatRecord{} + if err := unmarshalRecord(val, rec); err != nil { + return nil, fmt.Errorf("GetSourceCacheCompat: unmarshal: %w", err) + } + return rec, nil +} + +// GetSourceCacheEntry returns the manifest entry for (rowKind, scopeKey), +// or pebble.ErrNotFound. +func (e *Engine) GetSourceCacheEntry(ctx context.Context, rowKind, scopeKey string) (*v3.SourceCacheEntryRecord, error) { + val, closer, err := e.db.Get(encodeSourceCacheEntryKey(rowKind, scopeKey)) + if err != nil { + return nil, err + } + defer closer.Close() + rec := &v3.SourceCacheEntryRecord{} + if err := unmarshalRecord(val, rec); err != nil { + return nil, fmt.Errorf("GetSourceCacheEntry: unmarshal: %w", err) + } + return rec, nil +} + +// DeleteGrantsByPrincipalsInScope deletes every grant row in the CURRENT +// store stamped with scopeKey whose principal id is in principalIDs — +// the engine side of principal-scoped delta tombstones +// (SourceCacheRecord.deleted_principal_ids). +// +// One prefix scan of the scope's by_source_scope index resolves +// everything: the index tail IS the grant identity, so the primary key +// and every secondary index key for a match are constructible from the +// index key alone — no value reads, no string resolution, no guessing. +// A principal with no rows in the scope is a no-op (providers tombstone +// objects the client never synced). Deleting a missing secondary index +// entry is a pebble no-op, which covers the mixed inline/deferred +// by_principal state mid-sync. +// +// Complexity: O(scope size) tuple-walks per call regardless of tombstone +// count — callers batch a page's tombstones into one call. +func (e *Engine) DeleteGrantsByPrincipalsInScope(ctx context.Context, scopeKey string, principalIDs map[string]struct{}) (int64, error) { + if len(principalIDs) == 0 { + return 0, nil + } + prefix := encodeGrantBySourceScopePrefix(scopeKey) + var deleted int64 + + err := e.withWrite(func() error { + if err := e.requireCurrentSync(); err != nil { + return err + } + iter, err := e.db.NewIter(&pebble.IterOptions{ + LowerBound: prefix, + UpperBound: upperBoundOf(prefix), + }) + if err != nil { + return err + } + defer iter.Close() + + opts := writeOpts(e.opts.durability) + if e.IsFreshSync() { + opts = pebble.NoSync + } + batch := e.db.NewBatch() + defer func() { _ = batch.Close() }() + + for iter.First(); iter.Valid(); iter.Next() { + if err := ctx.Err(); err != nil { + return err + } + key := iter.Key() + tail := key[len(prefix):] + // Tail layout: ent_rt | ent_rid | flag | ent_tail | prin_rt | prin_id + // (identical to the grant primary key tail; decoder shared with + // the primary-prefix scan paths in grants.go). + id, ok := decodeGrantIdentityTail(key, prefix) + if !ok { + continue // malformed index key — defensive skip + } + if _, hit := principalIDs[id.principalID]; !hit { + continue + } + // Primary key = grant header + the identity tail verbatim. + priKey := make([]byte, 0, 3+len(tail)) + priKey = append(priKey, versionV3, typeGrant) + priKey = codec.AppendTupleSeparator(priKey) + priKey = append(priKey, tail...) + if err := batch.Delete(priKey, nil); err != nil { + return err + } + if err := batch.Delete(encodeGrantByPrincipalIdentityIndexKey(id), nil); err != nil { + return err + } + if err := batch.Delete(encodeGrantByNeedsExpansionIdentityIndexKey(id), nil); err != nil { + return err + } + // The scope index entry itself (the key under the iterator — + // safe: the iterator reads a snapshot). + if err := batch.Delete(key, nil); err != nil { + return err + } + deleted++ + } + if err := iter.Error(); err != nil { + return err + } + if err := batch.Commit(opts); err != nil { + return err + } + return nil + }) + if err != nil { + return 0, err + } + return deleted, nil +} + +// DeleteGrantsByExternalIDsInScope deletes every grant row in the CURRENT +// store stamped with scopeKey whose STORED grant id (external id, which +// may be a connector-custom shape) is in ids. One scan of the scope's +// index, loading each candidate's primary row to compare the stored id — +// bounded by the scope's row count, never the whole keyspace. This is the +// tombstone path for connectors with custom grant ids whose scopes span +// multiple resources (so principal-scoped deletes would over-delete). +func (e *Engine) DeleteGrantsByExternalIDsInScope(ctx context.Context, scopeKey string, ids map[string]struct{}) (int64, error) { + if len(ids) == 0 { + return 0, nil + } + prefix := encodeGrantBySourceScopePrefix(scopeKey) + var deleted int64 + + err := e.withWrite(func() error { + if err := e.requireCurrentSync(); err != nil { + return err + } + iter, err := e.db.NewIter(&pebble.IterOptions{ + LowerBound: prefix, + UpperBound: upperBoundOf(prefix), + }) + if err != nil { + return err + } + defer iter.Close() + + opts := writeOpts(e.opts.durability) + if e.IsFreshSync() { + opts = pebble.NoSync + } + batch := e.db.NewBatch() + defer func() { _ = batch.Close() }() + + for iter.First(); iter.Valid(); iter.Next() { + if err := ctx.Err(); err != nil { + return err + } + key := iter.Key() + tail := key[len(prefix):] + id, ok := decodeGrantIdentityTail(key, prefix) + if !ok { + continue // malformed index key — defensive skip + } + // Primary key = grant header + the identity tail verbatim. + priKey := make([]byte, 0, 3+len(tail)) + priKey = append(priKey, versionV3, typeGrant) + priKey = codec.AppendTupleSeparator(priKey) + priKey = append(priKey, tail...) + + val, closer, err := e.db.Get(priKey) + if err != nil { + if errors.Is(err, pebble.ErrNotFound) { + continue // index ahead of primary — defensive skip + } + return err + } + rec := &v3.GrantRecord{} + uerr := unmarshalRecord(val, rec) + _ = closer.Close() + if uerr != nil { + return fmt.Errorf("DeleteGrantsByExternalIDsInScope: unmarshal: %w", uerr) + } + if _, hit := ids[rec.GetExternalId()]; !hit { + continue + } + if err := batch.Delete(priKey, nil); err != nil { + return err + } + if err := batch.Delete(encodeGrantByPrincipalIdentityIndexKey(id), nil); err != nil { + return err + } + if err := batch.Delete(encodeGrantByNeedsExpansionIdentityIndexKey(id), nil); err != nil { + return err + } + if err := batch.Delete(key, nil); err != nil { + return err + } + deleted++ + } + if err := iter.Error(); err != nil { + return err + } + if err := batch.Commit(opts); err != nil { + return err + } + return nil + }) + if err != nil { + return 0, err + } + return deleted, nil +} + +// DeleteResourcesByIDsInScope deletes every resource row in the CURRENT +// store stamped with scopeKey whose resource id is in resourceIDs (any +// resource type) — principal-scoped tombstones for RowKindResources. +func (e *Engine) DeleteResourcesByIDsInScope(ctx context.Context, scopeKey string, resourceIDs map[string]struct{}) (int64, error) { + if len(resourceIDs) == 0 { + return 0, nil + } + prefix := encodeResourceBySourceScopePrefix(scopeKey) + var deleted int64 + + err := e.withWrite(func() error { + if err := e.requireCurrentSync(); err != nil { + return err + } + iter, err := e.db.NewIter(&pebble.IterOptions{ + LowerBound: prefix, + UpperBound: upperBoundOf(prefix), + }) + if err != nil { + return err + } + defer iter.Close() + + opts := writeOpts(e.opts.durability) + if e.IsFreshSync() { + opts = pebble.NoSync + } + batch := e.db.NewBatch() + defer func() { _ = batch.Close() }() + + for iter.First(); iter.Valid(); iter.Next() { + if err := ctx.Err(); err != nil { + return err + } + key := iter.Key() + tail := key[len(prefix):] + // Tail layout: resource_type_id | resource_id. + rtBytes, next, ok := codec.DecodeTupleStringAlias(tail, 0) + if !ok || next >= len(tail) { + continue + } + ridBytes, _, ok := codec.DecodeTupleStringAlias(tail, next+1) + if !ok { + continue + } + if _, hit := resourceIDs[string(ridBytes)]; !hit { + continue + } + rt, rid := string(rtBytes), string(ridBytes) + priKey := make([]byte, 0, 3+len(tail)) + priKey = append(priKey, versionV3, typeResource) + priKey = codec.AppendTupleSeparator(priKey) + priKey = append(priKey, tail...) + // by_parent cleanup needs the parent ref from the value. + if val, closer, getErr := e.db.Get(priKey); getErr == nil { + err := e.deleteResourceIndexesRaw(batch, rt, rid, val) + closer.Close() + if err != nil { + return err + } + } else if !errors.Is(getErr, pebble.ErrNotFound) { + return getErr + } + if err := batch.Delete(priKey, nil); err != nil { + return err + } + // deleteResourceIndexesRaw already covered the scope entry + // (source_scope_key is in the value), but delete the iterated + // key too in case the value read missed (orphan entry). + if err := batch.Delete(key, nil); err != nil { + return err + } + deleted++ + } + if err := iter.Error(); err != nil { + return err + } + return batch.Commit(opts) + }) + if err != nil { + return 0, err + } + return deleted, nil +} + +// grantValueHasSourcesRaw reports whether a marshaled GrantRecord carries +// at least one sources entry (field 9), without unmarshaling. +func grantValueHasSourcesRaw(value []byte) (bool, error) { + for len(value) > 0 { + num, typ, n := protowire.ConsumeTag(value) + if n < 0 { + return false, protowire.ParseError(n) + } + value = value[n:] + if num == 9 { + return true, nil + } + n = protowire.ConsumeFieldValue(num, typ, value) + if n < 0 { + return false, protowire.ParseError(n) + } + value = value[n:] + } + return false, nil +} + +// stripExpanderSourcesRaw clears a replayed grant's Sources map when it is +// expander-written, so the current sync's expansion recomputes it from +// true state instead of inheriting contributions that may have been +// removed upstream. Classification mirrors RollbackExpansion: a Sources +// map containing a self-source entry (keyed by the grant's own entitlement +// id) was written by the expander; one without a self-source is +// connector-set public data and is preserved. Returns (newValue, true) when +// the record was rewritten, (nil, false) when the original bytes should be +// copied verbatim. +func stripExpanderSourcesRaw(value []byte, ownEntitlementID string) ([]byte, bool, error) { + r := &v3.GrantRecord{} + if err := unmarshalRecord(value, r); err != nil { + return nil, false, fmt.Errorf("source cache replay: unmarshal grant for sources strip: %w", err) + } + sources := r.GetSources() + if len(sources) == 0 { + return nil, false, nil + } + if _, hasSelf := sources[ownEntitlementID]; !hasSelf { + // No self-source: connector-set Sources. Preserve verbatim. + return nil, false, nil + } + r.SetSources(nil) + stripped, err := marshalRecord(r) + if err != nil { + return nil, false, fmt.Errorf("source cache replay: re-marshal grant after sources strip: %w", err) + } + return stripped, true, nil +} + +// decodeResourcePrimaryTail decodes (resource_type_id, resource_id) +// from a resource primary key (v3 | typeResource | 0x00 | rt | 0x00 | rid). +func decodeResourcePrimaryTail(priKey []byte) (string, string, error) { + const headerLen = 3 // versionV3, typeResource, separator + if len(priKey) <= headerLen { + return "", "", fmt.Errorf("source cache replay: malformed resource primary key %x", priKey) + } + tail := priKey[headerLen:] + rtBytes, next, err := codec.DecodeTupleStringTo(nil, tail, 0) + if err != nil { + return "", "", err + } + if next >= len(tail) { + return "", "", fmt.Errorf("source cache replay: resource primary key missing resource_id: %x", priKey) + } + ridBytes, _, err := codec.DecodeTupleStringTo(nil, tail, next+1) + if err != nil { + return "", "", err + } + return string(rtBytes), string(ridBytes), nil +} + +// replayPrimaryFromIndexKey derives a record's primary key from its +// by_source_scope index key. The index prefix (header|0x00|scope|0x00) +// is followed by exactly the identity tuple that forms the primary +// key's tail, so the primary is header' + 0x00-separated remainder. +func replayPrimaryFromIndexKey(indexKey, indexPrefix []byte, primaryHeader [2]byte) ([]byte, error) { + if len(indexKey) <= len(indexPrefix) { + return nil, fmt.Errorf("source cache replay: malformed index key %x", indexKey) + } + tail := indexKey[len(indexPrefix):] + key := make([]byte, 0, 3+len(tail)) + key = append(key, primaryHeader[0], primaryHeader[1], 0x00) + return append(key, tail...), nil +} + +// clearReplayScopeRowsLocked deletes every row in the CURRENT store whose +// by_source_scope index entry lives under scopePrefix, restoring the +// replay copy's "first writer for the scope" precondition when a resumed +// action re-runs a replay page. +// +// Why this must exist: a crashed earlier attempt of the same page may +// have committed the page's own overlay rows — net-new identities stamped +// with this scope — AFTER its replay copy landed. Those identities are +// not in the previous file, so a re-run's copy neither overwrites nor +// removes them, and verifyReplayedScopeCount's post-copy recount would +// find copied-base + overlay entries and fail the replay even though no +// data was lost. Deleting up front is safe because the resumed page +// re-delivers everything the crashed attempt wrote: the copy re-copies +// the base and the page's own rows are re-put after the copy. +// +// Rows whose VALUE stamp names a different scope are left in place (they +// belong to that other scope); only their orphaned index entry under this +// scope is removed. On the first run of a page the scope's index range is +// empty and this is a single bounded no-op scan. +// +// stampOf extracts the row's source-scope stamp from its raw value; +// deleteRowIndexes stages the row's secondary-index deletes derived from +// its raw value (including its scope entry, when the kind has more +// indexes than the scope entry itself). The caller must hold the write +// lock (this runs inside the replay's withWrite closure). +func (e *Engine) clearReplayScopeRowsLocked( + ctx context.Context, + scopeKey string, + scopePrefix []byte, + primaryHeader [2]byte, + stampOf func(val []byte) (string, error), + deleteRowIndexes func(batch *pebble.Batch, priKey []byte, val []byte) error, +) (int64, error) { + iter, err := e.db.NewIter(&pebble.IterOptions{ + LowerBound: scopePrefix, + UpperBound: upperBoundOf(scopePrefix), + }) + if err != nil { + return 0, err + } + defer iter.Close() + + opts := writeOpts(e.opts.durability) + if e.IsFreshSync() { + opts = pebble.NoSync + } + batch := e.db.NewBatch() + defer func() { _ = batch.Close() }() + + var deleted int64 + stagedInBatch := 0 + for iter.First(); iter.Valid(); iter.Next() { + if err := ctx.Err(); err != nil { + return deleted, err + } + priKey, err := replayPrimaryFromIndexKey(iter.Key(), scopePrefix, primaryHeader) + if err != nil { + return deleted, err + } + val, closer, getErr := e.db.Get(priKey) + switch { + case errors.Is(getErr, pebble.ErrNotFound): + // Orphan index entry with no row: drop the entry alone. + if err := batch.Delete(iter.Key(), nil); err != nil { + return deleted, err + } + stagedInBatch++ + case getErr != nil: + return deleted, fmt.Errorf("source cache replay: pre-clear get: %w", getErr) + default: + stamp, stampErr := stampOf(val) + if stampErr != nil { + closer.Close() + return deleted, stampErr + } + if stamp != scopeKey { + // The row was rewritten under another scope; this index + // entry is stale. Remove the entry, keep the row. + closer.Close() + if err := batch.Delete(iter.Key(), nil); err != nil { + return deleted, err + } + stagedInBatch++ + break + } + if err := deleteRowIndexes(batch, priKey, val); err != nil { + closer.Close() + return deleted, err + } + closer.Close() + if err := batch.Delete(priKey, nil); err != nil { + return deleted, err + } + // deleteRowIndexes covers the scope entry via the value's + // stamp; delete the iterated key too so the range is clean + // even if the derivations ever disagree (deleting a missing + // key is a pebble no-op). + if err := batch.Delete(iter.Key(), nil); err != nil { + return deleted, err + } + deleted++ + stagedInBatch++ + } + if stagedInBatch >= replayBatchRows { + if err := batch.Commit(opts); err != nil { + return deleted, err + } + _ = batch.Close() + batch = e.db.NewBatch() + stagedInBatch = 0 + } + } + if err := iter.Error(); err != nil { + return deleted, err + } + if err := batch.Commit(opts); err != nil { + return deleted, err + } + return deleted, nil +} + +// ReplaySourceCacheGrants copies every grant stamped with scopeKey from +// prev into the receiver: raw primary copy plus index synthesis from the +// raw value (principal, needs_expansion, source-scope families). Mirrors +// PutGrantRecords' read-before-write index cleanup when the receiver +// already holds a record at the same identity. +func (e *Engine) ReplaySourceCacheGrants(ctx context.Context, prev *Engine, scopeKey string) (SourceCacheReplayResult, error) { + var res SourceCacheReplayResult + prefix := encodeGrantBySourceScopePrefix(scopeKey) + primaryHeader := [2]byte{versionV3, typeGrant} + + err := e.withWrite(func() error { + if err := e.requireCurrentSync(); err != nil { + return err + } + // Consumed in a defer keyed on rows whose commit LANDED, not on + // success: a replay that fails after an intermediate commit + // (large scope, error or cancellation mid-copy) has already + // populated the keyspace, and while the failing action unwinds, + // concurrently draining workers can still write — a first + // PutGrantRecords taking the empty-keyspace fast path over + // partially replayed identities would skip index cleanup. + // + // clearedRows counts pre-clear deletes the same way: the + // pre-clear commits intermediate batches too, so a pre-clear + // that fails partway has already mutated the keyspace. + // Registered BEFORE the pre-clear so its partial-failure path is + // covered. + // + // sawRelatedResource: InsertResourceGrants copies write resource + // rows too, so the resources empty-keyspace fast path must also + // be disarmed. Related copies always ride a batch that carries + // at least their own grant row, so committedRows > 0 covers + // "any batch that may hold a resource write landed". + committedRows := 0 + clearedRows := int64(0) + sawRelatedResource := false + defer func() { + if committedRows > 0 || clearedRows > 0 { + _ = e.takeFreshGrantsEmpty() + if sawRelatedResource { + _ = e.takeFreshResourcesEmpty() + } + } + }() + + // Resume idempotency: drop any destination rows a crashed earlier + // attempt already committed under this scope (replay copy and/or + // overlay rows), so the copy below is the scope's first writer + // again and the post-copy recount stays exact. + cleared, err := e.clearReplayScopeRowsLocked(ctx, scopeKey, prefix, primaryHeader, + func(val []byte) (string, error) { + _, _, _, _, _, _, srcScope, scanErr := scanGrantIndexFieldsRaw(val) + return srcScope, scanErr + }, + func(batch *pebble.Batch, priKey []byte, val []byte) error { + return e.deleteGrantIndexesRaw(batch, "", val) + }, + ) + clearedRows = cleared + if err != nil { + return fmt.Errorf("source cache replay: pre-clear grants scope %q: %w", scopeKey, err) + } + res.ResumedRowsCleared = cleared + + iter, err := prev.db.NewIter(&pebble.IterOptions{ + LowerBound: prefix, + UpperBound: upperBoundOf(prefix), + }) + if err != nil { + return err + } + defer iter.Close() + + opts := writeOpts(e.opts.durability) + if e.IsFreshSync() { + opts = pebble.NoSync + } + batch := e.db.NewBatch() + defer func() { _ = batch.Close() }() + rowsInBatch := 0 + // relatedCopied dedupes InsertResourceGrants resource copies: + // many grants in one scope typically reference few resources. + var relatedCopied map[string]struct{} + + for iter.First(); iter.Valid(); iter.Next() { + if err := ctx.Err(); err != nil { + return err + } + priKey, err := replayPrimaryFromIndexKey(iter.Key(), prefix, primaryHeader) + if err != nil { + return err + } + val, closer, getErr := prev.db.Get(priKey) + if getErr != nil { + if errors.Is(getErr, pebble.ErrNotFound) { + // Orphan index entry in the previous file — skip, + // matching the defensive-skip semantic of the other + // index read paths. + res.StaleSkipped++ + continue + } + return fmt.Errorf("source cache replay: get prev grant: %w", getErr) + } + + entRT, entRID, entID, principalRT, principalID, needsExpansion, srcScope, scanErr := scanGrantIndexFieldsRaw(val) + if scanErr != nil { + closer.Close() + return scanErr + } + // Stale-index defense: only copy rows whose VALUE stamp matches + // the queried scope. An index entry pointing at a row stamped + // differently (or not at all) is left over from a path that + // replaced the row without cleaning the index — e.g. a fold + // compaction predating the source-cache bucket plans, or an + // in-sync same-identity rewrite under a different scope. Copying + // it would inject rows upstream never returned for this scope. + if srcScope != scopeKey { + closer.Close() + res.StaleSkipped++ + continue + } + + // Clean up index entries for any record the current sync + // already wrote at this identity (same discipline as + // PutGrantRecords' read-before-write). + if oldVal, oldCloser, oldErr := e.db.Get(priKey); oldErr == nil { + if err := e.deleteGrantIndexesRaw(batch, "", oldVal); err != nil { + oldCloser.Close() + closer.Close() + return err + } + oldCloser.Close() + } else if !errors.Is(oldErr, pebble.ErrNotFound) { + closer.Close() + return fmt.Errorf("source cache replay: get current grant: %w", oldErr) + } + + // Replay-equivalence: a cached sync must reproduce what a full + // resync would produce. The one field where a verbatim copy + // diverges is expander-written Sources — the previous sync's + // expansion baked contributions into direct grants, and + // re-expansion only ADDS, so a contribution removed this sync + // (via a delta tombstone or a refetched page) would survive + // forever. Strip expander-written Sources so the current sync's + // expansion recomputes them from true state; connector-set + // Sources (no self-source entry — same classification as + // RollbackExpansion) are connector data and are preserved + // verbatim. The probe is a cheap protowire scan; the vast + // majority of rows carry no Sources and stay on the raw-copy + // path. + writeVal := val + if hasSources, probeErr := grantValueHasSourcesRaw(val); probeErr != nil { + closer.Close() + return probeErr + } else if hasSources { + stripped, strippedOK, stripErr := stripExpanderSourcesRaw(val, entID) + if stripErr != nil { + closer.Close() + return stripErr + } + if strippedOK { + writeVal = stripped + } + } + // Dense ingestion facts for replayed rows: same evidence the + // put path maintains, so the syncer's store-derived seams + // (I2) see replayed and fresh rows identically. + flags, flagsErr := scanGrantSourceCacheFlagsRaw(val) + if flagsErr != nil { + closer.Close() + return flagsErr + } + if flags.externalResourceMatch { + if err := e.markExternalMatchFact(); err != nil { + closer.Close() + return err + } + } + if flags.insertResourceGrants && entRT != "" && entRID != "" { + resKey := encodeResourceKey(entRT, entRID) + if _, done := relatedCopied[string(resKey)]; !done { + if relatedCopied == nil { + relatedCopied = make(map[string]struct{}) + } + relatedCopied[string(resKey)] = struct{}{} + copied, relErr := e.replayRelatedResourceRaw(batch, prev, resKey, entRT, entRID) + if relErr != nil { + closer.Close() + return relErr + } + if copied { + res.RelatedResourceRows++ + sawRelatedResource = true + } + } + } + + if err := batch.Set(priKey, writeVal, nil); err != nil { + closer.Close() + return err + } + closer.Close() + + if entID != "" && entRT != "" && entRID != "" && principalRT != "" && principalID != "" { + id := grantIdentity{ + entitlement: entitlementIdentityFromParts(entRT, entRID, entID), + principalTypeID: principalRT, + principalID: principalID, + } + if _, err := e.writeGrantIndexesForIdentityScratch(batch, id, needsExpansion, srcScope, nil); err != nil { + return err + } + } + res.Rows++ + rowsInBatch++ + if rowsInBatch >= replayBatchRows { + if err := batch.Commit(opts); err != nil { + return err + } + committedRows += rowsInBatch + _ = batch.Close() + batch = e.db.NewBatch() + rowsInBatch = 0 + } + } + if err := iter.Error(); err != nil { + return err + } + if err := batch.Commit(opts); err != nil { + return err + } + // Replay populated the fresh sync's grant keyspace directly. The + // first overlay PutGrantRecords must therefore perform its normal + // read-before-write index cleanup, rather than claiming the + // keyspace is still empty and leaving replayed index entries stale + // (consumed by the defer above). + committedRows += rowsInBatch + return e.verifyReplayedScopeCount(ctx, "grants", scopeKey, prefix, res.Rows) + }) + if err != nil { + return SourceCacheReplayResult{}, err + } + return res, nil +} + +// CopyResourceRowFrom copies one resource row (primary + secondary +// indexes) from prev into the receiver, byte-verbatim. The repair side of +// ingestion invariant I3: when a grant-inserted resource row was lost by +// a replay-path failure, the previous sync's row is the full-fidelity +// source of truth (it was cold-materialized from the wire grant's +// embedded resource, which grant STORAGE does not retain). Returns false +// when prev has no such row — repair impossible. +func (e *Engine) CopyResourceRowFrom(ctx context.Context, prev *Engine, resourceTypeID, resourceID string) (bool, error) { + copied := false + err := e.withWrite(func() error { + if err := e.requireCurrentSync(); err != nil { + return err + } + batch := e.db.NewBatch() + defer batch.Close() + ok, err := e.replayRelatedResourceRaw(batch, prev, encodeResourceKey(resourceTypeID, resourceID), resourceTypeID, resourceID) + if err != nil { + return err + } + if !ok { + return nil + } + if err := batch.Commit(writeOpts(e.opts.durability)); err != nil { + return err + } + _ = e.takeFreshResourcesEmpty() + copied = true + return nil + }) + return copied, err +} + +// replayRelatedResourceRaw copies one resource row from prev into batch +// (primary + secondary indexes), byte-verbatim — the engine side of +// recreating InsertResourceGrants-inserted resources when their grants +// page replays. Returns false when prev holds no such row: the previous +// sync's output didn't have it either, so there is nothing to preserve. +func (e *Engine) replayRelatedResourceRaw(batch *pebble.Batch, prev *Engine, priKey []byte, rt, rid string) (bool, error) { + val, closer, err := prev.db.Get(priKey) + if err != nil { + if errors.Is(err, pebble.ErrNotFound) { + return false, nil + } + return false, fmt.Errorf("source cache replay: get prev related resource: %w", err) + } + defer closer.Close() + + parentRT, parentID, srcScope, err := scanResourceIndexFieldsRaw(val) + if err != nil { + return false, err + } + // Clean up index entries for any record the current sync already + // wrote at this identity (same discipline as the resource replay). + if oldVal, oldCloser, oldErr := e.db.Get(priKey); oldErr == nil { + err := e.deleteResourceIndexesRaw(batch, rt, rid, oldVal) + oldCloser.Close() + if err != nil { + return false, err + } + } else if !errors.Is(oldErr, pebble.ErrNotFound) { + return false, fmt.Errorf("source cache replay: get current related resource: %w", oldErr) + } + if err := batch.Set(priKey, val, nil); err != nil { + return false, err + } + if parentID != "" { + if err := batch.Set(encodeResourceByParentIndexKey(parentRT, parentID, rt, rid), nil, nil); err != nil { + return false, err + } + } + if srcScope != "" { + if err := batch.Set(encodeResourceBySourceScopeIndexKey(srcScope, rt, rid), nil, nil); err != nil { + return false, err + } + } + return true, nil +} + +// ReplaySourceCacheEntitlements copies every entitlement stamped with +// scopeKey from prev into the receiver. +func (e *Engine) ReplaySourceCacheEntitlements(ctx context.Context, prev *Engine, scopeKey string) (SourceCacheReplayResult, error) { + var res SourceCacheReplayResult + prefix := encodeEntitlementBySourceScopePrefix(scopeKey) + primaryHeader := [2]byte{versionV3, typeEntitlement} + + err := e.withWrite(func() error { + if err := e.requireCurrentSync(); err != nil { + return err + } + // Keyed on rows whose commit LANDED, not on success (see the + // grants replay): a partial replay has already mutated the + // entitlement keyspace, so the bare-id lookup map must be + // invalidated and the empty-keyspace fast path disarmed even when + // the replay itself fails — draining workers can still resolve + // tombstones and write entitlements while the failure unwinds. + // Pre-clear deletes mutate the keyspace the same way, so they + // arm the invalidation too — registered BEFORE the pre-clear so + // a pre-clear that commits a batch and then fails is covered + // (clearedRows is assigned even on the error path). + committedRows := 0 + clearedRows := int64(0) + defer func() { + if committedRows > 0 || clearedRows > 0 { + e.noteEntitlementKeyspaceWrite() + _ = e.takeFreshEntitlementsEmpty() + } + }() + + // Resume idempotency: see the grants replay. The entitlement + // bare-id lookup map is invalidated for the deletes exactly as + // for the copies (the defer above keys on either landing). + cleared, err := e.clearReplayScopeRowsLocked(ctx, scopeKey, prefix, primaryHeader, + scanEntitlementSourceScopeRaw, + func(batch *pebble.Batch, priKey []byte, val []byte) error { + // The scope entry is the only entitlement secondary + // index, and the caller deletes the iterated entry. + return nil + }, + ) + clearedRows = cleared + if err != nil { + return fmt.Errorf("source cache replay: pre-clear entitlements scope %q: %w", scopeKey, err) + } + res.ResumedRowsCleared = cleared + + iter, err := prev.db.NewIter(&pebble.IterOptions{ + LowerBound: prefix, + UpperBound: upperBoundOf(prefix), + }) + if err != nil { + return err + } + defer iter.Close() + + opts := writeOpts(e.opts.durability) + if e.IsFreshSync() { + opts = pebble.NoSync + } + batch := e.db.NewBatch() + defer func() { _ = batch.Close() }() + rowsInBatch := 0 + + for iter.First(); iter.Valid(); iter.Next() { + if err := ctx.Err(); err != nil { + return err + } + priKey, err := replayPrimaryFromIndexKey(iter.Key(), prefix, primaryHeader) + if err != nil { + return err + } + val, closer, getErr := prev.db.Get(priKey) + if getErr != nil { + if errors.Is(getErr, pebble.ErrNotFound) { + res.StaleSkipped++ + continue + } + return fmt.Errorf("source cache replay: get prev entitlement: %w", getErr) + } + // Stale-index defense: only copy rows whose VALUE stamp matches + // the queried scope (see the grants replay for rationale). + prevScope, prevScanErr := scanEntitlementSourceScopeRaw(val) + if prevScanErr != nil { + closer.Close() + return prevScanErr + } + if prevScope != scopeKey { + closer.Close() + res.StaleSkipped++ + continue + } + // The replayed row's source-scope index key is byte-identical + // to prev's — the only entitlement secondary index. Clean up a + // differing stamp on any record the current sync already wrote + // at this identity. + if oldVal, oldCloser, oldErr := e.db.Get(priKey); oldErr == nil { + oldScope, scanErr := scanEntitlementSourceScopeRaw(oldVal) + oldCloser.Close() + if scanErr != nil { + closer.Close() + return scanErr + } + if oldScope != "" && oldScope != scopeKey { + // The old index key's tail equals the primary key's + // tail (identity tuple), so rebuild it byte-wise. + oldIdxKey := make([]byte, 0, 8+len(oldScope)+len(priKey)) + oldIdxKey = append(oldIdxKey, versionV3, typeIndex, idxEntitlementBySourceScope) + oldIdxKey = codec.AppendTupleSeparator(oldIdxKey) + oldIdxKey = codec.AppendTupleStrings(oldIdxKey, oldScope) + oldIdxKey = codec.AppendTupleSeparator(oldIdxKey) + oldIdxKey = append(oldIdxKey, priKey[3:]...) + if err := batch.Delete(oldIdxKey, nil); err != nil { + closer.Close() + return err + } + } + } else if !errors.Is(oldErr, pebble.ErrNotFound) { + closer.Close() + return fmt.Errorf("source cache replay: get current entitlement: %w", oldErr) + } + if err := batch.Set(priKey, val, nil); err != nil { + closer.Close() + return err + } + closer.Close() + if err := batch.Set(iter.Key(), nil, nil); err != nil { + return err + } + res.Rows++ + rowsInBatch++ + if rowsInBatch >= replayBatchRows { + if err := batch.Commit(opts); err != nil { + return err + } + committedRows += rowsInBatch + _ = batch.Close() + batch = e.db.NewBatch() + rowsInBatch = 0 + } + } + if err := iter.Error(); err != nil { + return err + } + if err := batch.Commit(opts); err != nil { + return err + } + // The defer above invalidates the lazy bare-id lookup map (see + // lookup.go — later tombstones would silently miss replayed rows + // otherwise) and disarms the empty-keyspace fast path for the + // first overlay PutEntitlementRecords. The bump happens strictly + // AFTER the last commit that landed: a map build racing the + // replay records the pre-bump generation and is correctly + // invalidated; bumping earlier would let such a build record the + // fresh generation against partially-replayed state. + committedRows += rowsInBatch + return e.verifyReplayedScopeCount(ctx, "entitlements", scopeKey, prefix, res.Rows) + }) + if err != nil { + return SourceCacheReplayResult{}, err + } + return res, nil +} + +// ReplaySourceCacheResources copies every resource stamped with scopeKey +// from prev into the receiver, synthesizing by_parent and by_source_scope +// index entries from the raw value. +func (e *Engine) ReplaySourceCacheResources(ctx context.Context, prev *Engine, scopeKey string) (SourceCacheReplayResult, error) { + var res SourceCacheReplayResult + prefix := encodeResourceBySourceScopePrefix(scopeKey) + primaryHeader := [2]byte{versionV3, typeResource} + + err := e.withWrite(func() error { + if err := e.requireCurrentSync(); err != nil { + return err + } + // Keyed on rows whose commit LANDED, not on success — see the + // grants replay for rationale. Registered BEFORE the pre-clear + // and keyed on its deletes too (clearedRows is assigned even on + // the error path). + committedRows := 0 + clearedRows := int64(0) + defer func() { + if committedRows > 0 || clearedRows > 0 { + _ = e.takeFreshResourcesEmpty() + } + }() + + // Resume idempotency: see the grants replay. + cleared, err := e.clearReplayScopeRowsLocked(ctx, scopeKey, prefix, primaryHeader, + func(val []byte) (string, error) { + _, _, srcScope, scanErr := scanResourceIndexFieldsRaw(val) + return srcScope, scanErr + }, + func(batch *pebble.Batch, priKey []byte, val []byte) error { + rt, rid, decodeErr := decodeResourcePrimaryTail(priKey) + if decodeErr != nil { + return decodeErr + } + return e.deleteResourceIndexesRaw(batch, rt, rid, val) + }, + ) + clearedRows = cleared + if err != nil { + return fmt.Errorf("source cache replay: pre-clear resources scope %q: %w", scopeKey, err) + } + res.ResumedRowsCleared = cleared + + iter, err := prev.db.NewIter(&pebble.IterOptions{ + LowerBound: prefix, + UpperBound: upperBoundOf(prefix), + }) + if err != nil { + return err + } + defer iter.Close() + + opts := writeOpts(e.opts.durability) + if e.IsFreshSync() { + opts = pebble.NoSync + } + batch := e.db.NewBatch() + defer func() { _ = batch.Close() }() + rowsInBatch := 0 + + for iter.First(); iter.Valid(); iter.Next() { + if err := ctx.Err(); err != nil { + return err + } + priKey, err := replayPrimaryFromIndexKey(iter.Key(), prefix, primaryHeader) + if err != nil { + return err + } + val, closer, getErr := prev.db.Get(priKey) + if getErr != nil { + if errors.Is(getErr, pebble.ErrNotFound) { + res.StaleSkipped++ + continue + } + return fmt.Errorf("source cache replay: get prev resource: %w", getErr) + } + + rt, rid, decodeErr := decodeResourcePrimaryTail(priKey) + if decodeErr != nil { + closer.Close() + return decodeErr + } + parentRT, parentID, srcScope, scanErr := scanResourceIndexFieldsRaw(val) + if scanErr != nil { + closer.Close() + return scanErr + } + // Stale-index defense: only copy rows whose VALUE stamp matches + // the queried scope (see the grants replay for rationale). + if srcScope != scopeKey { + closer.Close() + res.StaleSkipped++ + continue + } + // Child scheduling is response-row-driven in the syncer; a + // replayed parent never reaches that path, so surface its + // child types for the syncer to schedule. + childTypeIDs, childErr := scanResourceChildTypeIDsRaw(val) + if childErr != nil { + closer.Close() + return childErr + } + if len(childTypeIDs) > 0 { + res.ChildResources = append(res.ChildResources, ReplayedParentResource{ + ResourceTypeID: rt, + ResourceID: rid, + ChildTypeIDs: childTypeIDs, + }) + } + if oldVal, oldCloser, oldErr := e.db.Get(priKey); oldErr == nil { + if err := e.deleteResourceIndexesRaw(batch, rt, rid, oldVal); err != nil { + oldCloser.Close() + closer.Close() + return err + } + oldCloser.Close() + } else if !errors.Is(oldErr, pebble.ErrNotFound) { + closer.Close() + return fmt.Errorf("source cache replay: get current resource: %w", oldErr) + } + + if err := batch.Set(priKey, val, nil); err != nil { + closer.Close() + return err + } + closer.Close() + + if parentID != "" { + if err := batch.Set(encodeResourceByParentIndexKey(parentRT, parentID, rt, rid), nil, nil); err != nil { + return err + } + } + if srcScope != "" { + if err := batch.Set(encodeResourceBySourceScopeIndexKey(srcScope, rt, rid), nil, nil); err != nil { + return err + } + } + res.Rows++ + rowsInBatch++ + if rowsInBatch >= replayBatchRows { + if err := batch.Commit(opts); err != nil { + return err + } + committedRows += rowsInBatch + _ = batch.Close() + batch = e.db.NewBatch() + rowsInBatch = 0 + } + } + if err := iter.Error(); err != nil { + return err + } + if err := batch.Commit(opts); err != nil { + return err + } + // See grant replay above: direct replay writes mean the first + // overlay PutResourceRecords must not use the empty-keyspace + // fast-path, or old by_parent/by_source_scope entries survive + // (consumed by the defer above). + committedRows += rowsInBatch + return e.verifyReplayedScopeCount(ctx, "resources", scopeKey, prefix, res.Rows) + }) + if err != nil { + return SourceCacheReplayResult{}, err + } + return res, nil +} diff --git a/pkg/dotc1z/engine/pebble/source_cache_engine_test.go b/pkg/dotc1z/engine/pebble/source_cache_engine_test.go new file mode 100644 index 000000000..aeffaab12 --- /dev/null +++ b/pkg/dotc1z/engine/pebble/source_cache_engine_test.go @@ -0,0 +1,556 @@ +package pebble + +import ( + "context" + "testing" + + "github.com/cockroachdb/pebble/v2" + "github.com/stretchr/testify/require" + + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + v3 "github.com/conductorone/baton-sdk/pb/c1/storage/v3" + "github.com/conductorone/baton-sdk/pkg/annotations" + "github.com/conductorone/baton-sdk/pkg/connectorstore" + "github.com/conductorone/baton-sdk/pkg/sourcecache" + batonGrant "github.com/conductorone/baton-sdk/pkg/types/grant" +) + +var ( + scopeA = sourcecache.HashScope("https://example.test/teams/1/members?page=1") + scopeB = sourcecache.HashScope("https://example.test/teams/2/members?page=1") +) + +func scGrant(entName, principalID string, expandable bool) *v2.Grant { + // Public ids are the raw ":"-join (an external contract, never parsed + // for identity — see identity.go). + canonicalEntID := "group:g1:" + entName + ent := v2.Entitlement_builder{ + Id: canonicalEntID, + Resource: v2.Resource_builder{ + Id: v2.ResourceId_builder{ResourceType: "group", Resource: "g1"}.Build(), + }.Build(), + }.Build() + principal := v2.Resource_builder{ + Id: v2.ResourceId_builder{ResourceType: "user", Resource: principalID}.Build(), + }.Build() + var annos annotations.Annotations + if expandable { + annos.Update(v2.GrantExpandable_builder{EntitlementIds: []string{canonicalEntID}}.Build()) + } + return v2.Grant_builder{ + Id: batonGrant.NewGrantID(principal, ent), + Entitlement: ent, + Principal: principal, + Annotations: annos, + }.Build() +} + +func TestSourceCacheEntryCRUD(t *testing.T) { + ctx := context.Background() + a := newAdapter(t) + _, err := a.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + + e := a.PebbleEngine() + _, err = e.GetSourceCacheEntry(ctx, "grants", scopeA) + require.ErrorIs(t, err, pebble.ErrNotFound) + + require.NoError(t, e.PutSourceCacheEntry(ctx, "grants", scopeA, `W/"etag-1"`)) + rec, err := e.GetSourceCacheEntry(ctx, "grants", scopeA) + require.NoError(t, err) + require.Equal(t, `W/"etag-1"`, rec.GetCacheValidator()) + require.NotNil(t, rec.GetDiscoveredAt()) + + // Row-kind partition: the same scope key under a different kind misses. + _, err = e.GetSourceCacheEntry(ctx, "resources", scopeA) + require.ErrorIs(t, err, pebble.ErrNotFound) + + // Rotation overwrites in place (delta tokens rotate every round). + require.NoError(t, e.PutSourceCacheEntry(ctx, "grants", scopeA, "delta-token-2")) + rec, err = e.GetSourceCacheEntry(ctx, "grants", scopeA) + require.NoError(t, err) + require.Equal(t, "delta-token-2", rec.GetCacheValidator()) +} + +// TestSourceCacheReplayGrantsAcrossEngines is the core replay contract: +// grants stamped with a scope in file A are copied — rows, indexes, and +// needs_expansion reporting — into file B, and unstamped/other-scope rows +// stay behind. +func TestSourceCacheReplayGrantsAcrossEngines(t *testing.T) { + ctx := context.Background() + + prev := newAdapter(t) + _, err := prev.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + + gExpandable := scGrant("member", "alice", true) + gPlain := scGrant("member", "bob", false) + gOtherScope := scGrant("member", "carol", false) + gUnstamped := scGrant("member", "dave", false) + + scopedCtx := sourcecache.WithScope(ctx, scopeA) + require.NoError(t, prev.PutGrants(scopedCtx, gExpandable, gPlain)) + require.NoError(t, prev.PutGrants(sourcecache.WithScope(ctx, scopeB), gOtherScope)) + require.NoError(t, prev.PutGrants(ctx, gUnstamped)) + require.NoError(t, prev.PebbleEngine().PutSourceCacheEntry(ctx, "grants", scopeA, "etag-a")) + + cur := newAdapter(t) + _, err = cur.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + + res, err := cur.PebbleEngine().ReplaySourceCacheGrants(ctx, prev.PebbleEngine(), scopeA) + require.NoError(t, err) + require.Equal(t, int64(2), res.Rows) + // Expansion arming is store-derived (the by_needs_expansion index the + // replay synthesizes below), no longer reported on the result. + + // Replayed rows are readable by ID and carry their expansion flag. + got, err := cur.PebbleEngine().GetGrantRecord(ctx, gExpandable.GetId()) + require.NoError(t, err) + require.True(t, got.GetNeedsExpansion()) + require.Equal(t, scopeA, got.GetSourceScopeKey()) + _, err = cur.PebbleEngine().GetGrantRecord(ctx, gPlain.GetId()) + require.NoError(t, err) + + // Rows outside the scope are not copied. + _, err = cur.PebbleEngine().GetGrantRecord(ctx, gOtherScope.GetId()) + require.ErrorIs(t, err, pebble.ErrNotFound) + _, err = cur.PebbleEngine().GetGrantRecord(ctx, gUnstamped.GetId()) + require.ErrorIs(t, err, pebble.ErrNotFound) + + // The by_source_scope index came across too: a second replay FROM the + // current store (as next sync's previous) finds the same rows. + next := newAdapter(t) + _, err = next.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + res2, err := next.PebbleEngine().ReplaySourceCacheGrants(ctx, cur.PebbleEngine(), scopeA) + require.NoError(t, err) + require.Equal(t, int64(2), res2.Rows) +} + +// TestSourceCacheOrphanScopes pins invariant I6's evidence surface: +// scope-index entries whose scope has no manifest entry are reported as +// orphans (a lost manifest write or stamp leak would poison a future +// sync's replay); writing the entry clears the report. Zero-row manifest +// entries (the reverse direction) are legal and never reported. +func TestSourceCacheOrphanScopes(t *testing.T) { + ctx := context.Background() + a := newAdapter(t) + _, err := a.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + e := a.PebbleEngine() + + // Stamped rows in two scopes; a manifest entry for only one. + require.NoError(t, a.PutGrants(sourcecache.WithScope(ctx, scopeA), scGrant("member", "alice", false))) + require.NoError(t, a.PutGrants(sourcecache.WithScope(ctx, scopeB), scGrant("member", "bob", false))) + require.NoError(t, e.PutSourceCacheEntry(ctx, "grants", scopeA, "etag-a")) + + orphans, err := e.SourceCacheOrphanScopes(ctx) + require.NoError(t, err) + require.Equal(t, map[string][]string{"grants": {scopeB}}, orphans) + + // Writing the missing entry clears the violation. + require.NoError(t, e.PutSourceCacheEntry(ctx, "grants", scopeB, "etag-b")) + orphans, err = e.SourceCacheOrphanScopes(ctx) + require.NoError(t, err) + require.Empty(t, orphans) + + // A zero-row scope (manifest entry, no index entries) is legal. + require.NoError(t, e.PutSourceCacheEntry(ctx, "resources", scopeA, "etag-r")) + orphans, err = e.SourceCacheOrphanScopes(ctx) + require.NoError(t, err) + require.Empty(t, orphans) + + // Row-kind partitions are independent: entitlement stamps under a + // scope key that only has a GRANTS manifest entry are orphans. + ent := v2.Entitlement_builder{ + Id: "group:g1:member", + Resource: v2.Resource_builder{ + Id: v2.ResourceId_builder{ResourceType: "group", Resource: "g1"}.Build(), + }.Build(), + }.Build() + require.NoError(t, a.PutEntitlements(sourcecache.WithScope(ctx, scopeB), ent)) + orphans, err = e.SourceCacheOrphanScopes(ctx) + require.NoError(t, err) + require.Equal(t, map[string][]string{"entitlements": {scopeB}}, orphans) +} + +// TestSourceCacheReplayIsIdempotent pins double-execution safety: a +// resumed sync can re-run a replay page whose first execution already +// committed (the checkpoint races the crash), so replaying the same scope +// twice into the same store must converge to the same state — same rows, +// same count reported, no error. +func TestSourceCacheReplayIsIdempotent(t *testing.T) { + ctx := context.Background() + + prev := newAdapter(t) + _, err := prev.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + g1 := scGrant("member", "alice", true) + g2 := scGrant("member", "bob", false) + require.NoError(t, prev.PutGrants(sourcecache.WithScope(ctx, scopeA), g1, g2)) + + cur := newAdapter(t) + _, err = cur.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + + res1, err := cur.PebbleEngine().ReplaySourceCacheGrants(ctx, prev.PebbleEngine(), scopeA) + require.NoError(t, err) + require.Equal(t, int64(2), res1.Rows) + + res2, err := cur.PebbleEngine().ReplaySourceCacheGrants(ctx, prev.PebbleEngine(), scopeA) + require.NoError(t, err) + require.Equal(t, int64(2), res2.Rows, "second replay converges, not errors") + _ = res1 + + // State after double replay is exactly the two rows, once each. + var seen int + require.NoError(t, cur.PebbleEngine().IterateGrants(ctx, func(*v3.GrantRecord) bool { + seen++ + return true + })) + require.Equal(t, 2, seen) +} + +// TestSourceCacheReplayIgnoresStaleIndexEntries pins the stale-index +// defense: an index entry whose target row is stamped with a DIFFERENT +// scope (left behind by a path that replaced rows without cleaning the +// index — e.g. a fold compaction predating the source-cache bucket plans) +// must not be copied. Replaying it would inject rows upstream never +// returned for the queried scope. +func TestSourceCacheReplayIgnoresStaleIndexEntries(t *testing.T) { + ctx := context.Background() + + prev := newAdapter(t) + _, err := prev.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + + // Row genuinely stamped with scopeB... + g := scGrant("member", "alice", false) + require.NoError(t, prev.PutGrants(sourcecache.WithScope(ctx, scopeB), g)) + + // ...plus a stale index entry claiming it belongs to scopeA. + rec, err := prev.PebbleEngine().GetGrantRecord(ctx, g.GetId()) + require.NoError(t, err) + staleIdx := replayTestGrantScopeIndexKey(t, scopeA, rec) + require.NoError(t, prev.PebbleEngine().DB().Set(staleIdx, nil, nil)) + + cur := newAdapter(t) + _, err = cur.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + + res, err := cur.PebbleEngine().ReplaySourceCacheGrants(ctx, prev.PebbleEngine(), scopeA) + require.NoError(t, err) + require.Zero(t, res.Rows, "a row stamped with a different scope must not be replayed via a stale index entry") + require.Equal(t, int64(1), res.StaleSkipped, + "the skipped stale entry must be reported — it is the syncer's only way to tell 'legitimately empty scope' from 'scope contents clobbered without index cleanup'") + _, err = cur.PebbleEngine().GetGrantRecord(ctx, g.GetId()) + require.ErrorIs(t, err, pebble.ErrNotFound) + + // The genuine scope still replays, with no staleness reported. + res, err = cur.PebbleEngine().ReplaySourceCacheGrants(ctx, prev.PebbleEngine(), scopeB) + require.NoError(t, err) + require.Equal(t, int64(1), res.Rows) + require.Zero(t, res.StaleSkipped) +} + +// TestEntitlementRestampCleansOldScopeIndex pins the read-before-write +// cleanup on the entitlement put path: rewriting the SAME entitlement +// identity under a NEW scope must delete the old scope's by_source_scope +// entry. Without the cleanup, the old scope's index still claims the row +// while its value stamp names the new scope — the next sync's replay of +// the old scope then copies zero rows against a manifest entry that +// promised content (Rows == 0, StaleSkipped > 0), which the syncer treats +// as data loss and fails. +func TestEntitlementRestampCleansOldScopeIndex(t *testing.T) { + ctx := context.Background() + + prev := newAdapter(t) + _, err := prev.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + + res1 := v2.Resource_builder{ + Id: v2.ResourceId_builder{ResourceType: "group", Resource: "g1"}.Build(), + }.Build() + entID := "group:g1:member" + ent1 := v2.Entitlement_builder{Id: entID, Resource: res1, DisplayName: "member"}.Build() + + // First put stamps scopeA (and consumes the fresh-empty fast path); + // the rewrite stamps scopeB and must clean scopeA's index entry. + require.NoError(t, prev.PutEntitlements(sourcecache.WithScope(ctx, scopeA), ent1)) + ent1b := v2.Entitlement_builder{Id: entID, Resource: res1, DisplayName: "member (renamed)"}.Build() + require.NoError(t, prev.PutEntitlements(sourcecache.WithScope(ctx, scopeB), ent1b)) + + cur := newAdapter(t) + _, err = cur.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + + // Old scope: legitimately empty now — no rows AND no stale entries. + resA, err := cur.PebbleEngine().ReplaySourceCacheEntitlements(ctx, prev.PebbleEngine(), scopeA) + require.NoError(t, err) + require.Zero(t, resA.Rows) + require.Zero(t, resA.StaleSkipped, "re-stamp must delete the old scope's index entry, not leave it stale") + + // New scope: exactly the rewritten row. + resB, err := cur.PebbleEngine().ReplaySourceCacheEntitlements(ctx, prev.PebbleEngine(), scopeB) + require.NoError(t, err) + require.Equal(t, int64(1), resB.Rows) + require.Zero(t, resB.StaleSkipped) + got, err := cur.PebbleEngine().GetEntitlementRecord(ctx, entID) + require.NoError(t, err) + require.Equal(t, scopeB, got.GetSourceScopeKey()) + require.Equal(t, "member (renamed)", got.GetDisplayName()) +} + +// replayTestGrantScopeIndexKey builds a by_source_scope index key for the +// given scope pointing at rec's identity, bypassing the write path — the +// test plants it as a stale entry. +func replayTestGrantScopeIndexKey(t *testing.T, scopeKey string, rec *v3.GrantRecord) []byte { + t.Helper() + id, err := grantIdentityFromRecord(rec) + require.NoError(t, err) + return encodeGrantBySourceScopeIndexKey(scopeKey, id) +} + +// TestDeleteGrantsByPrincipalsInScope pins the principal-scoped tombstone +// path: bare principal ids kill exactly their rows within the scope — +// other scopes' rows for the same principal survive, unknown principals +// no-op, and the deleted rows' index entries (including the scope index) +// go with them. +func TestDeleteGrantsByPrincipalsInScope(t *testing.T) { + ctx := context.Background() + a := newAdapter(t) + _, err := a.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + + gAlice := scGrant("member", "alice", false) + gBob := scGrant("member", "bob", true) + require.NoError(t, a.PutGrants(sourcecache.WithScope(ctx, scopeA), gAlice, gBob)) + // Same principal, different scope — must survive an A-scoped delete. + gAliceOther := scGrant("owner", "alice", false) + require.NoError(t, a.PutGrants(sourcecache.WithScope(ctx, scopeB), gAliceOther)) + + deleted, err := a.PebbleEngine().DeleteGrantsByPrincipalsInScope(ctx, scopeA, map[string]struct{}{ + "alice": {}, + "bob": {}, + "unknown": {}, // tombstone for a principal never synced — no-op + }) + require.NoError(t, err) + require.Equal(t, int64(2), deleted) + + _, err = a.PebbleEngine().GetGrantRecord(ctx, gAlice.GetId()) + require.ErrorIs(t, err, pebble.ErrNotFound) + _, err = a.PebbleEngine().GetGrantRecord(ctx, gBob.GetId()) + require.ErrorIs(t, err, pebble.ErrNotFound) + _, err = a.PebbleEngine().GetGrantRecord(ctx, gAliceOther.GetId()) + require.NoError(t, err, "same principal in a different scope must survive") + + // Scope index entries went with the rows: a replay of scopeA from this + // store copies nothing. + cur := newAdapter(t) + _, err = cur.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + res, err := cur.PebbleEngine().ReplaySourceCacheGrants(ctx, a.PebbleEngine(), scopeA) + require.NoError(t, err) + require.Zero(t, res.Rows) +} + +// TestDeleteResourcesByIDsInScope pins the resources analog: bare object +// ids, any resource type, scope-relative. +func TestDeleteResourcesByIDsInScope(t *testing.T) { + ctx := context.Background() + a := newAdapter(t) + _, err := a.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + + u1 := v2.Resource_builder{Id: v2.ResourceId_builder{ResourceType: "user", Resource: "u1"}.Build(), DisplayName: "U1"}.Build() + u2 := v2.Resource_builder{Id: v2.ResourceId_builder{ResourceType: "user", Resource: "u2"}.Build(), DisplayName: "U2"}.Build() + require.NoError(t, a.PutResources(sourcecache.WithScope(ctx, scopeA), u1, u2)) + u3 := v2.Resource_builder{Id: v2.ResourceId_builder{ResourceType: "user", Resource: "u3"}.Build(), DisplayName: "U3"}.Build() + require.NoError(t, a.PutResources(sourcecache.WithScope(ctx, scopeB), u3)) + + deleted, err := a.PebbleEngine().DeleteResourcesByIDsInScope(ctx, scopeA, map[string]struct{}{ + "u1": {}, "u3": {}, // u3 is in scope B — must not die from an A-scoped tombstone + }) + require.NoError(t, err) + require.Equal(t, int64(1), deleted) + + _, err = a.PebbleEngine().GetResourceRecord(ctx, "user", "u1") + require.ErrorIs(t, err, pebble.ErrNotFound) + _, err = a.PebbleEngine().GetResourceRecord(ctx, "user", "u2") + require.NoError(t, err) + _, err = a.PebbleEngine().GetResourceRecord(ctx, "user", "u3") + require.NoError(t, err, "tombstone must be scope-relative") +} + +func TestSourceCacheReplayResourcesAndEntitlements(t *testing.T) { + ctx := context.Background() + + prev := newAdapter(t) + _, err := prev.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + + res1 := v2.Resource_builder{ + Id: v2.ResourceId_builder{ResourceType: "group", Resource: "g1"}.Build(), + DisplayName: "Group One", + }.Build() + entID := "group:g1:member" + ent1 := v2.Entitlement_builder{Id: entID, Resource: res1, DisplayName: "member"}.Build() + + scoped := sourcecache.WithScope(ctx, scopeA) + require.NoError(t, prev.PutResources(scoped, res1)) + require.NoError(t, prev.PutEntitlements(scoped, ent1)) + + cur := newAdapter(t) + _, err = cur.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + + rres, err := cur.PebbleEngine().ReplaySourceCacheResources(ctx, prev.PebbleEngine(), scopeA) + require.NoError(t, err) + require.Equal(t, int64(1), rres.Rows) + eres, err := cur.PebbleEngine().ReplaySourceCacheEntitlements(ctx, prev.PebbleEngine(), scopeA) + require.NoError(t, err) + require.Equal(t, int64(1), eres.Rows) + + gotRes, err := cur.PebbleEngine().GetResourceRecord(ctx, "group", "g1") + require.NoError(t, err) + require.Equal(t, "Group One", gotRes.GetDisplayName()) + require.Equal(t, scopeA, gotRes.GetSourceScopeKey()) + + gotEnt, err := cur.PebbleEngine().GetEntitlementRecord(ctx, entID) + require.NoError(t, err) + require.Equal(t, scopeA, gotEnt.GetSourceScopeKey()) +} + +// TestReplayInvalidatesEntitlementIDLookup pins the keyspace-generation +// contract for the replay writer: replay mutates the entitlement primary +// keyspace, so a bare-id lookup map built BEFORE the replay must be +// invalidated by it. If replay skips noteEntitlementKeyspaceWrite, the +// stale map hides replayed rows from resolveEntitlementIdentityByExternalID +// and a delta tombstone (DeleteEntitlementRecord) silently no-ops — a row +// upstream reported deleted survives the sync. +func TestReplayInvalidatesEntitlementIDLookup(t *testing.T) { + ctx := context.Background() + + prev := newAdapter(t) + _, err := prev.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + + res1 := v2.Resource_builder{ + Id: v2.ResourceId_builder{ResourceType: "group", Resource: "g1"}.Build(), + DisplayName: "Group One", + }.Build() + entID := "group:g1:member" + ent1 := v2.Entitlement_builder{Id: entID, Resource: res1, DisplayName: "member"}.Build() + require.NoError(t, prev.PutEntitlements(sourcecache.WithScope(ctx, scopeA), ent1)) + + cur := newAdapter(t) + _, err = cur.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + + // Force the lazy bare-id map to build BEFORE the replay (empty + // keyspace → entID resolves to nothing). + _, err = cur.PebbleEngine().GetEntitlementRecord(ctx, entID) + require.Error(t, err, "sanity: entitlement must not resolve before replay") + + rres, err := cur.PebbleEngine().ReplaySourceCacheEntitlements(ctx, prev.PebbleEngine(), scopeA) + require.NoError(t, err) + require.Equal(t, int64(1), rres.Rows) + + // The replayed row must be visible through the bare-id lookup (the + // map built above is stale and must have been invalidated) ... + gotEnt, err := cur.PebbleEngine().GetEntitlementRecord(ctx, entID) + require.NoError(t, err, "replayed entitlement must resolve by external id after a pre-replay map build") + require.Equal(t, scopeA, gotEnt.GetSourceScopeKey()) + + // ... and, the actual production stake: a delta tombstone against the + // replayed row must DELETE it, not silently no-op. + tombstoned, err := cur.PebbleEngine().DeleteEntitlementRecord(ctx, entID) + require.NoError(t, err) + require.True(t, tombstoned, "the tombstone must delete the replayed row, not silently no-op") + _, err = cur.PebbleEngine().GetEntitlementRecord(ctx, entID) + require.Error(t, err, "tombstoned replayed entitlement must be gone") +} + +// TestStoreExpandedGrantsPreservesSourceScope pins the write-path contract +// that keeps replay correct across expansion: when the expander rewrites an +// existing direct grant (to bake in Sources), the record's source scope +// stamp — and therefore its membership in the next sync's replay of that +// scope — must survive, exactly like expansion/needs_expansion do. +func TestStoreExpandedGrantsPreservesSourceScope(t *testing.T) { + ctx := context.Background() + + a := newAdapter(t) + _, err := a.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + + g := scGrant("member", "alice", true) + require.NoError(t, a.PutGrants(sourcecache.WithScope(ctx, scopeA), g)) + + // Expander-style rewrite: same grant, Sources baked in, no scope ctx. + rewrite := scGrant("member", "alice", false) + rewrite.SetSources(v2.GrantSources_builder{ + Sources: map[string]*v2.GrantSources_GrantSource{ + g.GetEntitlement().GetId(): {}, + }, + }.Build()) + require.NoError(t, a.Grants().StoreExpandedGrants(ctx, rewrite)) + + got, err := a.PebbleEngine().GetGrantRecord(ctx, g.GetId()) + require.NoError(t, err) + require.Equal(t, scopeA, got.GetSourceScopeKey(), "expander rewrite clobbered the source scope stamp") + require.True(t, got.GetNeedsExpansion(), "expansion side-state must be preserved too") + + // And the index survives: a replay from this store still finds the row. + cur := newAdapter(t) + _, err = cur.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + res, err := cur.PebbleEngine().ReplaySourceCacheGrants(ctx, a.PebbleEngine(), scopeA) + require.NoError(t, err) + require.Equal(t, int64(1), res.Rows) + replayed, err := cur.PebbleEngine().GetGrantRecord(ctx, g.GetId()) + require.NoError(t, err) + // Replay-equivalence: expander-written Sources (self-source present) + // are stripped so the current sync's expansion recomputes them from + // true state — a full resync would produce Sources reflecting current + // membership, so a cached sync must too. needs_expansion survived the + // copy (asserted via the replay result above), so expansion re-adds + // them. + require.Empty(t, replayed.GetSources(), "expander-written Sources must be stripped on replay for full-resync equivalence") + require.True(t, replayed.GetNeedsExpansion()) +} + +// TestSourceCacheReplayPreservesConnectorSetSources pins the other half of +// the sources-strip classification: a Sources map WITHOUT a self-source +// entry is connector-set public data (the connector emitted it on the +// ordinary path) and must survive replay byte-for-byte — never stripped. +func TestSourceCacheReplayPreservesConnectorSetSources(t *testing.T) { + ctx := context.Background() + + prev := newAdapter(t) + _, err := prev.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + + g := scGrant("member", "alice", false) + // Connector-set provenance: keyed by a FOREIGN entitlement, no + // self-source. This is public v2.Grant data, not expander output. + g.SetSources(v2.GrantSources_builder{ + Sources: map[string]*v2.GrantSources_GrantSource{ + "group:other:member": {}, + }, + }.Build()) + require.NoError(t, prev.PutGrants(sourcecache.WithScope(ctx, scopeA), g)) + + cur := newAdapter(t) + _, err = cur.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + res, err := cur.PebbleEngine().ReplaySourceCacheGrants(ctx, prev.PebbleEngine(), scopeA) + require.NoError(t, err) + require.Equal(t, int64(1), res.Rows) + + replayed, err := cur.PebbleEngine().GetGrantRecord(ctx, g.GetId()) + require.NoError(t, err) + require.Len(t, replayed.GetSources(), 1, "connector-set Sources must survive replay") + _, ok := replayed.GetSources()["group:other:member"] + require.True(t, ok) +} diff --git a/pkg/dotc1z/engine/pebble/source_cache_preclear_test.go b/pkg/dotc1z/engine/pebble/source_cache_preclear_test.go new file mode 100644 index 000000000..79b7b98a5 --- /dev/null +++ b/pkg/dotc1z/engine/pebble/source_cache_preclear_test.go @@ -0,0 +1,166 @@ +package pebble + +// Pins the pre-clear failure window of the replay functions: the +// resume-idempotency pre-clear commits intermediate batches, so a +// pre-clear that fails partway (error or cancellation) has already +// mutated the keyspace — the empty-keyspace fast-path flags must be +// disarmed and (for entitlements) the bare-id lookup map invalidated +// even though the replay itself returned an error. Before the fix, the +// entitlement invalidation defer was registered AFTER the pre-clear's +// error return, and the grants/resources defers ignored pre-clear +// deletes entirely. + +import ( + "context" + "fmt" + "testing" + + "github.com/stretchr/testify/require" + + v3 "github.com/conductorone/baton-sdk/pb/c1/storage/v3" + "github.com/conductorone/baton-sdk/pkg/connectorstore" +) + +// failAfterNChecks is a context whose Err() reports Canceled after the +// first n calls succeed — a deterministic mid-loop cancellation for +// paths that poll ctx.Err() once per iteration. +type failAfterNChecks struct { + context.Context + remaining int +} + +func (c *failAfterNChecks) Err() error { + if c.remaining <= 0 { + return context.Canceled + } + c.remaining-- + return nil +} + +// plantScopedEntitlement writes a scope-stamped entitlement row and its +// scope index key DIRECTLY into the db, bypassing PutEntitlementRecords — +// simulating rows a crashed earlier replay attempt committed, without +// consuming the fresh-empty flag the way a live writer would. +func plantScopedEntitlement(t *testing.T, e *Engine, scope, rid string) { + t.Helper() + rec := v3.EntitlementRecord_builder{ + ExternalId: "group:" + rid + ":member", + Resource: v3.ResourceRef_builder{ResourceTypeId: "group", ResourceId: rid}.Build(), + SourceScopeKey: scope, + }.Build() + id, err := entitlementIdentityFromRecord(rec) + require.NoError(t, err) + val, err := marshalRecord(rec) + require.NoError(t, err) + require.NoError(t, e.db.Set(encodeEntitlementIdentityKey(id), val, nil)) + require.NoError(t, e.db.Set(encodeEntitlementBySourceScopeIndexKey(scope, id), nil, nil)) +} + +func plantScopedGrant(t *testing.T, e *Engine, scope, principalID string) { + t.Helper() + rec := v3.GrantRecord_builder{ + ExternalId: "group:g1:member:user:" + principalID, + Entitlement: v3.EntitlementRef_builder{ + ResourceTypeId: "group", ResourceId: "g1", EntitlementId: "group:g1:member", + }.Build(), + Principal: v3.PrincipalRef_builder{ResourceTypeId: "user", ResourceId: principalID}.Build(), + SourceScopeKey: scope, + }.Build() + id, err := grantIdentityFromRecord(rec) + require.NoError(t, err) + val, err := marshalRecord(rec) + require.NoError(t, err) + require.NoError(t, e.db.Set(encodeGrantIdentityKey(id), val, nil)) + require.NoError(t, e.db.Set(encodeGrantBySourceScopeIndexKey(scope, id), nil, nil)) +} + +func plantScopedResource(t *testing.T, e *Engine, scope, rid string) { + t.Helper() + rec := v3.ResourceRecord_builder{ + ResourceTypeId: "user", + ResourceId: rid, + SourceScopeKey: scope, + }.Build() + val, err := marshalRecord(rec) + require.NoError(t, err) + require.NoError(t, e.db.Set(encodeResourceKey("user", rid), val, nil)) + require.NoError(t, e.db.Set(encodeResourceBySourceScopeIndexKey(scope, "user", rid), nil, nil)) +} + +func TestReplayPreClearPartialFailureDisarmsFastPaths(t *testing.T) { + ctx := context.Background() + old := replayBatchRows + replayBatchRows = 1 + t.Cleanup(func() { replayBatchRows = old }) + + scope := "scope-preclear-test" + + // One shared, empty previous file: the copy loop has nothing to do; + // everything under test happens in the pre-clear. + prev := newAdapter(t) + _, err := prev.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + + t.Run("entitlements", func(t *testing.T) { + cur := newAdapter(t) + _, err := cur.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + e := cur.PebbleEngine() + for i := 0; i < 3; i++ { + plantScopedEntitlement(t, e, scope, fmt.Sprintf("g%d", i)) + } + // One iteration succeeds (its single-row batch commits), the + // second sees the cancellation: a partial pre-clear. + cctx := &failAfterNChecks{Context: ctx, remaining: 1} + _, err = e.ReplaySourceCacheEntitlements(cctx, prev.PebbleEngine(), scope) + require.Error(t, err, "the partial pre-clear must surface its failure") + require.False(t, e.takeFreshEntitlementsEmpty(), + "a pre-clear that committed deletes mutated the keyspace; the empty-keyspace fast path must be disarmed even on failure") + }) + + t.Run("grants", func(t *testing.T) { + cur := newAdapter(t) + _, err := cur.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + e := cur.PebbleEngine() + for i := 0; i < 3; i++ { + plantScopedGrant(t, e, scope, fmt.Sprintf("u%d", i)) + } + cctx := &failAfterNChecks{Context: ctx, remaining: 1} + _, err = e.ReplaySourceCacheGrants(cctx, prev.PebbleEngine(), scope) + require.Error(t, err) + require.False(t, e.takeFreshGrantsEmpty(), + "grants pre-clear deletes must disarm the fast path even on failure") + }) + + t.Run("resources", func(t *testing.T) { + cur := newAdapter(t) + _, err := cur.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + e := cur.PebbleEngine() + for i := 0; i < 3; i++ { + plantScopedResource(t, e, scope, fmt.Sprintf("u%d", i)) + } + cctx := &failAfterNChecks{Context: ctx, remaining: 1} + _, err = e.ReplaySourceCacheResources(cctx, prev.PebbleEngine(), scope) + require.Error(t, err) + require.False(t, e.takeFreshResourcesEmpty(), + "resources pre-clear deletes must disarm the fast path even on failure") + }) + + // Control: a replay that fails BEFORE anything commits leaves the + // fast path armed — the disarm is keyed on landed mutations, not on + // the mere attempt. + t.Run("no mutation, no disarm", func(t *testing.T) { + cur := newAdapter(t) + _, err := cur.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + e := cur.PebbleEngine() + // Nothing planted: pre-clear iterates zero rows, copy loop + // copies zero rows (prev is empty), no commits with content. + _, err = e.ReplaySourceCacheEntitlements(ctx, prev.PebbleEngine(), scope) + require.NoError(t, err) + require.True(t, e.takeFreshEntitlementsEmpty(), + "an empty replay must not burn the first-put fast path") + }) +} diff --git a/pkg/dotc1z/engine/pebble/source_cache_replay_resume_test.go b/pkg/dotc1z/engine/pebble/source_cache_replay_resume_test.go new file mode 100644 index 000000000..76a99c867 --- /dev/null +++ b/pkg/dotc1z/engine/pebble/source_cache_replay_resume_test.go @@ -0,0 +1,167 @@ +package pebble + +// Resumed-replay regression tests: a crashed sync attempt can commit a +// replay page's OWN overlay rows (net-new identities stamped with the +// scope) after the replay copy lands and before the action finishes. The +// resumed action re-runs the same page, and the re-run's replay must +// converge — clear the earlier attempt's rows, re-copy the base, and pass +// its post-copy recount — instead of false-failing verifyReplayedScopeCount +// with ErrReplayIntegrity (which would burn the whole warm sync on a cold +// retry). + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + v3 "github.com/conductorone/baton-sdk/pb/c1/storage/v3" + "github.com/conductorone/baton-sdk/pkg/connectorstore" + "github.com/conductorone/baton-sdk/pkg/sourcecache" +) + +// TestSourceCacheReplayResumeAfterOverlayCommit pins the grants shape: +// replay copy + committed overlay row with a net-new identity, then a +// re-run of the same replay. +func TestSourceCacheReplayResumeAfterOverlayCommit(t *testing.T) { + ctx := context.Background() + + prev := newAdapter(t) + _, err := prev.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + base1 := scGrant("member", "alice", false) + base2 := scGrant("member", "bob", false) + require.NoError(t, prev.PutGrants(sourcecache.WithScope(ctx, scopeA), base1, base2)) + + cur := newAdapter(t) + _, err = cur.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + + // First attempt: the replay copies the base... + res1, err := cur.PebbleEngine().ReplaySourceCacheGrants(ctx, prev.PebbleEngine(), scopeA) + require.NoError(t, err) + require.Equal(t, int64(2), res1.Rows) + require.Zero(t, res1.ResumedRowsCleared) + + // ...then the page's own overlay row commits under the same scope — + // a NET-NEW identity the previous file does not contain... + overlay := scGrant("member", "carol", false) + require.NoError(t, cur.PutGrants(sourcecache.WithScope(ctx, scopeA), overlay)) + + // ...plus a row another scope legitimately owns, and a stale scopeA + // index entry pointing at it: the re-run's pre-clear must keep the + // row and drop only the orphaned entry. + otherScopeRow := scGrant("owner", "dave", false) + require.NoError(t, cur.PutGrants(sourcecache.WithScope(ctx, scopeB), otherScopeRow)) + otherRec, err := cur.PebbleEngine().GetGrantRecord(ctx, otherScopeRow.GetId()) + require.NoError(t, err) + staleIdx := replayTestGrantScopeIndexKey(t, scopeA, otherRec) + require.NoError(t, cur.PebbleEngine().DB().Set(staleIdx, nil, nil)) + + // The sync crashes before the action finishes; the resumed action + // re-runs the same page. The replay must converge, not false-fail + // its post-copy recount against the overlay row. + res2, err := cur.PebbleEngine().ReplaySourceCacheGrants(ctx, prev.PebbleEngine(), scopeA) + require.NoError(t, err, "resumed replay over committed overlay rows must not false-fail") + require.Equal(t, int64(2), res2.Rows) + require.Equal(t, int64(3), res2.ResumedRowsCleared, + "the crashed attempt's two base copies plus its overlay row") + + // The other scope's row survived the pre-clear. + _, err = cur.PebbleEngine().GetGrantRecord(ctx, otherScopeRow.GetId()) + require.NoError(t, err, "a row stamped with another scope must survive the pre-clear") + + // The resumed page re-delivers its rows after the replay. + require.NoError(t, cur.PutGrants(sourcecache.WithScope(ctx, scopeA), overlay)) + + // Converged state: base + overlay + the other scope's row, once each. + for _, g := range []*v2.Grant{base1, base2, overlay, otherScopeRow} { + _, err := cur.PebbleEngine().GetGrantRecord(ctx, g.GetId()) + require.NoError(t, err) + } + var seen int + require.NoError(t, cur.PebbleEngine().IterateGrants(ctx, func(*v3.GrantRecord) bool { + seen++ + return true + })) + require.Equal(t, 4, seen) + + // The scope index is exact again: a replay FROM this store (as the + // next sync's previous) finds exactly the scope's three rows. + next := newAdapter(t) + _, err = next.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + res3, err := next.PebbleEngine().ReplaySourceCacheGrants(ctx, cur.PebbleEngine(), scopeA) + require.NoError(t, err) + require.Equal(t, int64(3), res3.Rows) + require.Zero(t, res3.StaleSkipped) +} + +// TestSourceCacheReplayResumeAfterOverlayCommitResourcesAndEntitlements +// covers the same crash shape for the other two row kinds, including the +// entitlement bare-id lookup invalidation for pre-clear deletes. +func TestSourceCacheReplayResumeAfterOverlayCommitResourcesAndEntitlements(t *testing.T) { + ctx := context.Background() + + prev := newAdapter(t) + _, err := prev.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + + baseRes := v2.Resource_builder{ + Id: v2.ResourceId_builder{ResourceType: "group", Resource: "g1"}.Build(), + DisplayName: "Group One", + }.Build() + baseEntID := "group:g1:member" + baseEnt := v2.Entitlement_builder{Id: baseEntID, Resource: baseRes, DisplayName: "member"}.Build() + scoped := sourcecache.WithScope(ctx, scopeA) + require.NoError(t, prev.PutResources(scoped, baseRes)) + require.NoError(t, prev.PutEntitlements(scoped, baseEnt)) + + cur := newAdapter(t) + _, err = cur.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + + // First attempt: replay copies, then overlay rows commit. + _, err = cur.PebbleEngine().ReplaySourceCacheResources(ctx, prev.PebbleEngine(), scopeA) + require.NoError(t, err) + _, err = cur.PebbleEngine().ReplaySourceCacheEntitlements(ctx, prev.PebbleEngine(), scopeA) + require.NoError(t, err) + + overlayRes := v2.Resource_builder{ + Id: v2.ResourceId_builder{ResourceType: "group", Resource: "g2"}.Build(), + DisplayName: "Group Two", + }.Build() + overlayEntID := "group:g2:member" + overlayEnt := v2.Entitlement_builder{Id: overlayEntID, Resource: overlayRes, DisplayName: "member"}.Build() + curScoped := sourcecache.WithScope(ctx, scopeA) + require.NoError(t, cur.PutResources(curScoped, overlayRes)) + require.NoError(t, cur.PutEntitlements(curScoped, overlayEnt)) + + // Resumed re-run: both kinds converge instead of false-failing. + rres, err := cur.PebbleEngine().ReplaySourceCacheResources(ctx, prev.PebbleEngine(), scopeA) + require.NoError(t, err, "resumed resources replay must not false-fail") + require.Equal(t, int64(1), rres.Rows) + require.Equal(t, int64(2), rres.ResumedRowsCleared) + + eres, err := cur.PebbleEngine().ReplaySourceCacheEntitlements(ctx, prev.PebbleEngine(), scopeA) + require.NoError(t, err, "resumed entitlements replay must not false-fail") + require.Equal(t, int64(1), eres.Rows) + require.Equal(t, int64(2), eres.ResumedRowsCleared) + + // The pre-clear's deletes invalidated the bare-id lookup: the overlay + // entitlement is gone until the resumed page re-delivers it. + _, err = cur.PebbleEngine().GetEntitlementRecord(ctx, overlayEntID) + require.Error(t, err, "cleared overlay entitlement must not resolve through a stale bare-id map") + + // The resumed page re-delivers its rows; state converges. + require.NoError(t, cur.PutResources(curScoped, overlayRes)) + require.NoError(t, cur.PutEntitlements(curScoped, overlayEnt)) + + gotRes, err := cur.PebbleEngine().GetResourceRecord(ctx, "group", "g2") + require.NoError(t, err) + require.Equal(t, "Group Two", gotRes.GetDisplayName()) + gotEnt, err := cur.PebbleEngine().GetEntitlementRecord(ctx, overlayEntID) + require.NoError(t, err) + require.Equal(t, scopeA, gotEnt.GetSourceScopeKey()) +} diff --git a/pkg/dotc1z/engine/pebble/sync_stats_sidecar.go b/pkg/dotc1z/engine/pebble/sync_stats_sidecar.go index 040cfedda..41d2ab6f8 100644 --- a/pkg/dotc1z/engine/pebble/sync_stats_sidecar.go +++ b/pkg/dotc1z/engine/pebble/sync_stats_sidecar.go @@ -240,6 +240,16 @@ func (e *Engine) stashDeferredGrantStats(ds *deferredGrantStats) { e.deferredGrantStatsMu.Unlock() } +// invalidateDeferredGrantStats drops any stashed grant counts. Called by +// the dangling-reference grant drops (ingest_repair.go): rows deleted +// AFTER the deferred build ran would make the stash overcount, so the +// stats sidecar must fall back to its own post-drop scan. +func (e *Engine) invalidateDeferredGrantStats() { + e.deferredGrantStatsMu.Lock() + e.deferredGrantStats = nil + e.deferredGrantStatsMu.Unlock() +} + // takeDeferredGrantStats pops the stashed grant stats if they belong to // syncID, or returns nil. Consume-once: a later RecalculateStats falls back // to the full scan. diff --git a/pkg/dotc1z/entitlements.go b/pkg/dotc1z/entitlements.go index 58049ec2c..6ca4764c3 100644 --- a/pkg/dotc1z/entitlements.go +++ b/pkg/dotc1z/entitlements.go @@ -127,6 +127,6 @@ func (c *C1File) putEntitlementsInternal(ctx context.Context, f entitlementPutFu if err != nil { return err } - c.dbUpdated = true + c.dbUpdated.Store(true) return nil } diff --git a/pkg/dotc1z/grants.go b/pkg/dotc1z/grants.go index 44faaf418..abf3b39a2 100644 --- a/pkg/dotc1z/grants.go +++ b/pkg/dotc1z/grants.go @@ -514,7 +514,7 @@ func (c *C1File) upsertGrants(ctx context.Context, opts grantUpsertOptions, bulk return err } - c.dbUpdated = true + c.dbUpdated.Store(true) return nil } diff --git a/pkg/dotc1z/pebble_store.go b/pkg/dotc1z/pebble_store.go index 9714d4281..5d3afbc34 100644 --- a/pkg/dotc1z/pebble_store.go +++ b/pkg/dotc1z/pebble_store.go @@ -320,7 +320,7 @@ func (s *pebbleStore) CloseEngineOnly() error { s.closeMu.Unlock() return nil } - if !s.readOnly && s.dirty { + if !s.readOnly && (s.dirty || s.engine.Mutated()) { s.closeMu.Unlock() return errors.New("pebble CloseEngineOnly: refusing to discard dirty writable store") } @@ -675,7 +675,11 @@ func (s *pebbleStore) Close(ctx context.Context) (retErr error) { return nil } - if !s.readOnly && s.dirty { + // The engine's mutated latch is the backstop for the per-method + // wrapper dirty bit: any engine write path that bypassed a wrapper + // still forces the envelope save. Conservative in the safe + // direction — see Engine.Mutated. + if !s.readOnly && (s.dirty || s.engine.Mutated()) { if err := s.save(ctx); err != nil { // Tear NOTHING down: the unpacked DB under tmpDir is the only // copy of the synced data, and save failures are frequently diff --git a/pkg/dotc1z/pebble_store_mutated_test.go b/pkg/dotc1z/pebble_store_mutated_test.go new file mode 100644 index 000000000..f16049f2a --- /dev/null +++ b/pkg/dotc1z/pebble_store_mutated_test.go @@ -0,0 +1,64 @@ +package dotc1z + +// Engine-derived dirty backstop: the pebbleStore marks its envelope-save +// dirty bit per wrapper method BY CONVENTION, and every engine mutation +// path that misses a wrapper silently skips the save on Close — the +// clear-only replay pre-clear was one such path, and the class was only +// fixed instance-by-instance. Engine.Mutated latches when any write +// closure runs, and Close saves on EITHER signal, so the class is closed +// structurally: bypassing a wrapper can cost a spurious save, never a +// lost one. + +import ( + "context" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/conductorone/baton-sdk/pkg/connectorstore" + "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" + pebbleengine "github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble" + "github.com/conductorone/baton-sdk/pkg/sourcecache" +) + +func TestPebbleCloseSavesOnEngineMutationBypassingWrappers(t *testing.T) { + ctx := context.Background() + tmpDir := t.TempDir() + path := filepath.Join(tmpDir, "mutated.c1z") + + // A finished sync, saved and closed: the reopened handle starts with + // the wrapper dirty bit clear. + store, err := NewStore(ctx, path, WithEngine(c1zstore.EnginePebble), WithTmpDir(tmpDir)) + require.NoError(t, err) + syncID, err := store.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + require.NoError(t, store.EndSync(ctx)) + require.NoError(t, store.Close(ctx)) + + // Reopen writable and mutate through the ENGINE directly — the + // wrapper-bypass shape (any future engine method without a + // dirty-marking wrapper takes exactly this path). + store, err = NewStore(ctx, path, WithEngine(c1zstore.EnginePebble), WithTmpDir(tmpDir)) + require.NoError(t, err) + require.NoError(t, store.SetCurrentSync(ctx, syncID)) + holder, ok := any(store).(interface{ PebbleEngine() *pebbleengine.Engine }) + require.True(t, ok) + require.NoError(t, holder.PebbleEngine().PutSourceCacheEntry(ctx, "grants", "scope-x", "etag-x")) + require.NoError(t, store.Close(ctx)) + + // The mutation must have survived Close: without the engine-derived + // backstop the wrapper dirty bit was never set, the envelope save was + // skipped, and the write existed only in the discarded temp dir. + reopened, err := NewStore(ctx, path, WithEngine(c1zstore.EnginePebble), WithReadOnly(true), WithTmpDir(tmpDir)) + require.NoError(t, err) + defer func() { _ = reopened.Close(ctx) }() + require.NoError(t, reopened.SetCurrentSync(ctx, syncID)) + sc, ok := reopened.(SourceCacheStore) + require.True(t, ok) + entry, found, err := sc.LookupSourceCacheEntry(ctx, sourcecache.RowKindGrants, "scope-x") + require.NoError(t, err) + require.True(t, found, + "an engine mutation that bypassed the store wrappers must still force the envelope save on Close") + require.Equal(t, "etag-x", entry.CacheValidator) +} diff --git a/pkg/dotc1z/pool_test.go b/pkg/dotc1z/pool_test.go index b5be5239f..432588660 100644 --- a/pkg/dotc1z/pool_test.go +++ b/pkg/dotc1z/pool_test.go @@ -28,7 +28,11 @@ func TestEncoderPool(t *testing.T) { enc2, fromPool2 := getEncoder() require.NotNil(t, enc2) - require.True(t, fromPool2) + if !raceEnabled { + // sync.Pool randomly discards Puts under -race; reuse is + // only deterministic without it. + require.True(t, fromPool2) + } putEncoder(enc2) }) @@ -121,7 +125,11 @@ func TestDecoderPool(t *testing.T) { dec2, fromPool2 := getDecoder() require.NotNil(t, dec2) - require.True(t, fromPool2) + if !raceEnabled { + // See the encoder variant: sync.Pool discards randomly + // under -race. + require.True(t, fromPool2) + } putDecoder(dec2) }) @@ -333,7 +341,12 @@ func TestPoolGrowsFromDecoder(t *testing.T) { // Now the decoder pool should have a decoder. dec2, fromPool2 := getDecoder() - require.True(t, fromPool2, "NewDecoder.Close should have returned decoder to pool") + if !raceEnabled { + // See TestEncoderPool/TestDecoderPool: sync.Pool randomly + // discards Puts under -race; reuse is only deterministic + // without it. + require.True(t, fromPool2, "NewDecoder.Close should have returned decoder to pool") + } putDecoder(dec2) } diff --git a/pkg/dotc1z/race_disabled_test.go b/pkg/dotc1z/race_disabled_test.go new file mode 100644 index 000000000..75dfc5747 --- /dev/null +++ b/pkg/dotc1z/race_disabled_test.go @@ -0,0 +1,6 @@ +//go:build !race + +package dotc1z + +// See race_enabled_test.go. +const raceEnabled = false diff --git a/pkg/dotc1z/race_enabled_test.go b/pkg/dotc1z/race_enabled_test.go new file mode 100644 index 000000000..c4e1d39ac --- /dev/null +++ b/pkg/dotc1z/race_enabled_test.go @@ -0,0 +1,9 @@ +//go:build race + +package dotc1z + +// raceEnabled reports whether this test binary was built with -race. +// sync.Pool intentionally randomizes item retention under the race +// detector (Put may discard), so tests asserting pool REUSE are +// structurally flaky there and must skip that assertion. +const raceEnabled = true diff --git a/pkg/dotc1z/resouce_types.go b/pkg/dotc1z/resouce_types.go index 7af8a2e29..e1c1f70c3 100644 --- a/pkg/dotc1z/resouce_types.go +++ b/pkg/dotc1z/resouce_types.go @@ -115,6 +115,6 @@ func (c *C1File) putResourceTypesInternal(ctx context.Context, f resourceTypePut if err != nil { return err } - c.dbUpdated = true + c.dbUpdated.Store(true) return nil } diff --git a/pkg/dotc1z/resources.go b/pkg/dotc1z/resources.go index 89186c2e5..75a02e64c 100644 --- a/pkg/dotc1z/resources.go +++ b/pkg/dotc1z/resources.go @@ -131,6 +131,6 @@ func (c *C1File) putResourcesInternal(ctx context.Context, f resourcePutFunc, re if err != nil { return err } - c.dbUpdated = true + c.dbUpdated.Store(true) return nil } diff --git a/pkg/dotc1z/rollback_expansion.go b/pkg/dotc1z/rollback_expansion.go index 7fafd69b5..0fc80f6c8 100644 --- a/pkg/dotc1z/rollback_expansion.go +++ b/pkg/dotc1z/rollback_expansion.go @@ -231,7 +231,7 @@ func (c *C1File) RollbackExpansion(ctx context.Context, syncID string, dryRun bo return nil, err } - c.dbUpdated = true + c.dbUpdated.Store(true) // The cached view sync run carries the now-stale stats; drop it so the // next Stats()/GetSync recomputes against the rolled-back rows. c.invalidateCachedViewSyncRun() diff --git a/pkg/dotc1z/source_cache.go b/pkg/dotc1z/source_cache.go new file mode 100644 index 000000000..666c2411f --- /dev/null +++ b/pkg/dotc1z/source_cache.go @@ -0,0 +1,448 @@ +package dotc1z + +import ( + "context" + "errors" + "fmt" + + cdbpebble "github.com/cockroachdb/pebble/v2" + + v3 "github.com/conductorone/baton-sdk/pb/c1/storage/v3" + "github.com/conductorone/baton-sdk/pkg/bid" + "github.com/conductorone/baton-sdk/pkg/connectorstore" + "github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble" + "github.com/conductorone/baton-sdk/pkg/sourcecache" +) + +// SourceCacheReplayResult reports what one scope's replay copied. +type SourceCacheReplayResult = pebble.SourceCacheReplayResult + +// SourceCacheStore is the optional store capability backing source-cache +// replay (see proto/c1/connector/v2/annotation_source_cache.proto). It is +// implemented ONLY by the Pebble engine; the syncer type-asserts for it and +// treats a store without it as "source cache unsupported" (no-op lookup, +// no replay). It is deliberately NOT part of c1zstore.Store. +type SourceCacheStore interface { + // LookupSourceCacheEntry returns this store's manifest entry for + // (kind, scopeKey). Backs the connector-facing lookup when this + // store is the previous sync. + LookupSourceCacheEntry(ctx context.Context, kind sourcecache.RowKind, scopeKey string) (sourcecache.Entry, bool, error) + + // PutSourceCacheEntry writes the current sync's manifest entry for + // (kind, scopeKey). Zero-row scopes still get entries. + PutSourceCacheEntry(ctx context.Context, kind sourcecache.RowKind, scopeKey string, cacheValidator string) error + + // ReplaySourceCache copies every row stamped with scopeKey from prev + // (the previous sync's store, opened read-only) into this store. prev + // must be a Pebble store. Does NOT write the manifest entry — the + // caller writes it after the scope's overlay/deletes complete, so a + // failed replay can't leave a phantom hit for the next sync. + ReplaySourceCache(ctx context.Context, prev connectorstore.Reader, kind sourcecache.RowKind, scopeKey string) (SourceCacheReplayResult, error) + + // ApplyTombstones applies one page's delta tombstones for a scope + // after replay + overlay, returning the number of rows deleted. + // + // - deletedIDs are public canonical IDs (grant/entitlement IDs, or + // resource BIDs "bid:r:..."). Grants resolve WITHIN the scope's + // own rows by stored id, so connector-custom grant-id shapes work + // and the cost stays bounded by the scope's size; entitlements + // and resources resolve by bounded id probing (never an O(all + // rows) scan). + // - deletedPrincipalIDs are bare object ids: grants delete every + // row in the scope whose principal id matches (no principal + // type, no canonical-id reconstruction), resources by resource + // id (any type). Not supported for entitlements. + // + // Ids with no matching rows are no-ops. One scope-index scan per + // call; a page's tombstones are batched into one call. + ApplyTombstones(ctx context.Context, kind sourcecache.RowKind, scopeKey string, deletedIDs, deletedPrincipalIDs []string) (int64, error) + + // SourceCacheOrphanScopes returns, per row kind, scope keys present + // in the by_source_scope indexes with no manifest entry — an + // invariant violation at a sealed sync's quiesce point (ingestion + // invariant I6): the orphaned stamps would poison a future sync's + // replay. + SourceCacheOrphanScopes(ctx context.Context) (map[string][]string, error) + + // PutSourceCacheCompat records the current sync's replay- + // compatibility key (written when the write side enables, BEFORE any + // manifest entry). GetSourceCacheCompat returns the recorded key; + // found is false on pre-key artifacts and compaction folds — the + // syncer treats absent or unreadable keys as a mismatch (cold). + PutSourceCacheCompat(ctx context.Context, key sourcecache.CompatKey) error + GetSourceCacheCompat(ctx context.Context) (sourcecache.CompatKey, bool, error) +} + +var _ SourceCacheStore = (*pebbleStore)(nil) + +// SourceCacheInspector is the read-only inspection surface for tooling +// (the `baton source-cache` audit command) — typed access to the +// manifest, the per-scope index counts, and the I6 orphan check, without +// reaching for the raw engine. +type SourceCacheInspector interface { + // SourceCacheManifestSnapshot returns every manifest entry as + // "row_kind\x00scope_key" -> typed entry (validator + invalidation + // verdict as separate fields — validators are opaque connector + // strings, so invalidation must never be encoded in-band). + SourceCacheManifestSnapshot(ctx context.Context) (map[string]pebble.SourceCacheManifestEntry, error) + // SourceScopeIndexSnapshot returns, per row kind, scope_key -> the + // number of by_source_scope index entries. + SourceScopeIndexSnapshot(ctx context.Context) (map[string]map[string]int, error) + // SourceCacheOrphanScopes: see SourceCacheStore. + SourceCacheOrphanScopes(ctx context.Context) (map[string][]string, error) + // LookupSourceCacheEntry: see SourceCacheStore. + LookupSourceCacheEntry(ctx context.Context, kind sourcecache.RowKind, scopeKey string) (sourcecache.Entry, bool, error) + // GetSourceCacheCompat: see SourceCacheStore. Tooling uses key + // PRESENCE (the offline-decidable half of the compat gate — a key's + // exact match depends on the next run's connector declarations, which + // no offline audit can know; an ABSENT key can never match anything). + GetSourceCacheCompat(ctx context.Context) (sourcecache.CompatKey, bool, error) +} + +var _ SourceCacheInspector = (*pebbleStore)(nil) + +func (s *pebbleStore) SourceCacheManifestSnapshot(ctx context.Context) (map[string]pebble.SourceCacheManifestEntry, error) { + return s.engine.SourceCacheManifestSnapshot(ctx) +} + +func (s *pebbleStore) SourceScopeIndexSnapshot(ctx context.Context) (map[string]map[string]int, error) { + return s.engine.SourceScopeIndexSnapshot(ctx) +} + +// IngestInvariantStore is the optional store capability backing the +// syncer's ingestion invariants (see +// docs/tasks/source-cache-ingestion-invariants.md): dense row facts +// tracked as existence bits, the ordered-key scans the referential +// invariants (I3/I7/I8/I9) ride, and the drop/repair mechanics their +// default-mode verdicts apply. Pebble-only; the syncer degrades +// per-invariant when the store lacks it (SQLite artifacts are never +// scanned or mutated). +type IngestInvariantStore interface { + // HasExternalMatchGrants reports whether the open sync holds at + // least one grant carrying an ExternalResourceMatch* annotation — + // the store-derived truth behind HasExternalResourcesGrants. + HasExternalMatchGrants(ctx context.Context) (bool, error) + + // ForEachDistinctGrantEntitlementResource visits each distinct + // entitlement resource referenced by any grant: one seek per + // distinct resource, never O(grants). + ForEachDistinctGrantEntitlementResource(ctx context.Context, visit func(resourceTypeID, resourceID string) error) error + + // GrantsForEntResourceCarryInsertFact reports whether any grant + // under the entitlement resource carries InsertResourceGrants. + // Value-reading; reserved for dangling-reference probes. + GrantsForEntResourceCarryInsertFact(ctx context.Context, resourceTypeID, resourceID string) (bool, error) + + // GrantsForEntitlementAllCarryInsertFact reports whether every grant + // under the entitlement identity carries InsertResourceGrants — the + // fail-fast side of I8's per-grant exemption. Value-reading; + // reserved for dangling-reference probes. + GrantsForEntitlementAllCarryInsertFact(ctx context.Context, entitlementID, entResourceTypeID, entResourceID string) (bool, error) + + // HasResourceRecord reports whether a resource row exists. + HasResourceRecord(ctx context.Context, resourceTypeID, resourceID string) (bool, error) + + // ForEachDistinctEntitlementResource visits each distinct resource + // referenced by any entitlement row: one seek per distinct + // resource, never O(entitlements). + ForEachDistinctEntitlementResource(ctx context.Context, visit func(resourceTypeID, resourceID string) error) error + + // ForEachDanglingGrantEntitlement visits each distinct entitlement + // referenced by grants that has no entitlement row: one seek plus + // one point probe per distinct entitlement, never O(grants). + ForEachDanglingGrantEntitlement(ctx context.Context, visit func(entitlementID, resourceTypeID, resourceID string) error) error + + // EnsureGrantIndexes forces a pending deferred grant-index build to + // run now, making by_principal complete for the dangling-principal + // scan. Near-free: the same build would run at EndSync. + EnsureGrantIndexes(ctx context.Context) error + + // ForEachDanglingGrantPrincipal visits each distinct principal + // referenced by grants that has no resource row. matchAnnotatedOnly + // reports that every grant of the principal is an unprocessed + // ExternalResourceMatch* carrier (exempt shape); carrierGrants is + // the per-grant count of that population. + ForEachDanglingGrantPrincipal(ctx context.Context, visit func(principalRT, principalID string, matchAnnotatedOnly bool, carrierGrants int64) error) error + + // DeleteEntitlementsForResource drops every entitlement row under a + // resource; returns the count and up to maxIDs deleted ids + // (maxIDs <= 0 collects none — callers pass a shrinking budget). + DeleteEntitlementsForResource(ctx context.Context, resourceTypeID, resourceID string, maxIDs int) (int64, []string, error) + + // DeleteGrantsForEntitlement drops every grant row under one + // entitlement identity except InsertResourceGrants carriers (the + // machinery-owned shape); returns (deleted, skippedInsertFact). + DeleteGrantsForEntitlement(ctx context.Context, entitlementID, entResourceTypeID, entResourceID string) (int64, int64, error) + + // DeleteGrantsForPrincipal drops every grant of one principal except + // ExternalResourceMatch* carriers; returns (deleted, skipped). + DeleteGrantsForPrincipal(ctx context.Context, principalRT, principalID string) (int64, int64, error) +} + +var _ IngestInvariantStore = (*pebbleStore)(nil) + +func (s *pebbleStore) HasExternalMatchGrants(ctx context.Context) (bool, error) { + return s.engine.HasExternalMatchGrants(), nil +} + +func (s *pebbleStore) ForEachDistinctGrantEntitlementResource(ctx context.Context, visit func(resourceTypeID, resourceID string) error) error { + return s.engine.ForEachDistinctGrantEntitlementResource(ctx, visit) +} + +func (s *pebbleStore) GrantsForEntResourceCarryInsertFact(ctx context.Context, resourceTypeID, resourceID string) (bool, error) { + return s.engine.GrantsForEntResourceCarryInsertFact(ctx, resourceTypeID, resourceID) +} + +func (s *pebbleStore) GrantsForEntitlementAllCarryInsertFact(ctx context.Context, entitlementID, entResourceTypeID, entResourceID string) (bool, error) { + return s.engine.GrantsForEntitlementAllCarryInsertFact(ctx, entitlementID, entResourceTypeID, entResourceID) +} + +func (s *pebbleStore) HasResourceRecord(ctx context.Context, resourceTypeID, resourceID string) (bool, error) { + return s.engine.HasResourceRecord(ctx, resourceTypeID, resourceID) +} + +func (s *pebbleStore) ForEachDistinctEntitlementResource(ctx context.Context, visit func(resourceTypeID, resourceID string) error) error { + return s.engine.ForEachDistinctEntitlementResource(ctx, visit) +} + +func (s *pebbleStore) ForEachDanglingGrantEntitlement(ctx context.Context, visit func(entitlementID, resourceTypeID, resourceID string) error) error { + return s.engine.ForEachDanglingGrantEntitlement(ctx, visit) +} + +func (s *pebbleStore) EnsureGrantIndexes(ctx context.Context) error { + return s.engine.EnsureGrantIndexes(ctx) +} + +func (s *pebbleStore) ForEachDanglingGrantPrincipal(ctx context.Context, visit func(principalRT, principalID string, matchAnnotatedOnly bool, carrierGrants int64) error) error { + return s.engine.ForEachDanglingGrantPrincipal(ctx, visit) +} + +func (s *pebbleStore) DeleteEntitlementsForResource(ctx context.Context, resourceTypeID, resourceID string, maxIDs int) (int64, []string, error) { + n, ids, err := s.engine.DeleteEntitlementsForResource(ctx, resourceTypeID, resourceID, maxIDs) + if n > 0 { + s.MarkDirty() + } + return n, ids, err +} + +func (s *pebbleStore) DeleteGrantsForEntitlement(ctx context.Context, entitlementID, entResourceTypeID, entResourceID string) (int64, int64, error) { + n, skipped, err := s.engine.DeleteGrantsForEntitlement(ctx, entitlementID, entResourceTypeID, entResourceID) + if n > 0 { + s.MarkDirty() + } + return n, skipped, err +} + +func (s *pebbleStore) DeleteGrantsForPrincipal(ctx context.Context, principalRT, principalID string) (int64, int64, error) { + n, skipped, err := s.engine.DeleteGrantsForPrincipal(ctx, principalRT, principalID) + if n > 0 { + s.MarkDirty() + } + return n, skipped, err +} + +func (s *pebbleStore) SourceCacheOrphanScopes(ctx context.Context) (map[string][]string, error) { + return s.engine.SourceCacheOrphanScopes(ctx) +} + +// RepairRelatedResource copies one resource row from prev (the previous +// sync's store) into the current sync — ingestion invariant I3's repair +// arm for grant-inserted resources lost by a replay-path failure. +// Returns false when prev holds no such row (repair impossible). +func (s *pebbleStore) RepairRelatedResource(ctx context.Context, prev connectorstore.Reader, resourceTypeID, resourceID string) (bool, error) { + prevEngine, ok := sourceCacheEngine(prev) + if !ok { + return false, errors.New("related-resource repair: previous sync store is not a pebble store") + } + copied, err := s.engine.CopyResourceRowFrom(ctx, prevEngine, resourceTypeID, resourceID) + if err != nil { + return false, err + } + if copied { + s.MarkDirty() + } + return copied, nil +} + +// sourceCacheEngine recovers the Pebble engine from an arbitrary store, +// nil-safe. Mirrors pebble.AsEngine but accepts any value so the syncer +// can probe its previous-sync reader without caring about its static type. +func sourceCacheEngine(store any) (*pebble.Engine, bool) { + a, ok := store.(interface{ PebbleEngine() *pebble.Engine }) + if !ok { + return nil, false + } + e := a.PebbleEngine() + return e, e != nil +} + +func (s *pebbleStore) LookupSourceCacheEntry(ctx context.Context, kind sourcecache.RowKind, scopeKey string) (sourcecache.Entry, bool, error) { + if err := sourcecache.ValidateRowKind(kind); err != nil { + return sourcecache.Entry{}, false, err + } + rec, err := s.engine.GetSourceCacheEntry(ctx, string(kind), scopeKey) + if err != nil { + if errors.Is(err, cdbpebble.ErrNotFound) { + return sourcecache.Entry{}, false, nil + } + return sourcecache.Entry{}, false, err + } + if rec.GetInvalidated() { + // The dangling-reference drops removed rows from this scope after + // its manifest entry was written: the validator no longer + // describes the artifact's contents, so the scope must miss and + // re-fetch cold rather than 304-replay the sanitized subset. + return sourcecache.Entry{}, false, nil + } + return sourcecache.Entry{ + CacheValidator: rec.GetCacheValidator(), + DiscoveredAt: rec.GetDiscoveredAt().AsTime(), + }, true, nil +} + +func (s *pebbleStore) PutSourceCacheEntry(ctx context.Context, kind sourcecache.RowKind, scopeKey string, cacheValidator string) error { + if err := sourcecache.ValidateRowKind(kind); err != nil { + return err + } + return s.markDirty(s.engine.PutSourceCacheEntry(ctx, string(kind), scopeKey, cacheValidator)) +} + +func (s *pebbleStore) PutSourceCacheCompat(ctx context.Context, key sourcecache.CompatKey) error { + rec := &v3.SourceCacheCompatRecord{} + rec.SetConnectorCacheGeneration(key.ConnectorCacheGeneration) + rec.SetConnectorConfigFingerprint(key.ConnectorConfigFingerprint) + rec.SetSdkMaterializationGeneration(key.SDKMaterializationGeneration) + rec.SetSyncSelectionFingerprint(key.SyncSelectionFingerprint) + return s.markDirty(s.engine.PutSourceCacheCompat(ctx, rec)) +} + +func (s *pebbleStore) GetSourceCacheCompat(ctx context.Context) (sourcecache.CompatKey, bool, error) { + rec, err := s.engine.GetSourceCacheCompat(ctx) + if err != nil { + if errors.Is(err, cdbpebble.ErrNotFound) { + return sourcecache.CompatKey{}, false, nil + } + return sourcecache.CompatKey{}, false, err + } + return sourcecache.CompatKey{ + ConnectorCacheGeneration: rec.GetConnectorCacheGeneration(), + ConnectorConfigFingerprint: rec.GetConnectorConfigFingerprint(), + SDKMaterializationGeneration: rec.GetSdkMaterializationGeneration(), + SyncSelectionFingerprint: rec.GetSyncSelectionFingerprint(), + }, true, nil +} + +func (s *pebbleStore) ReplaySourceCache(ctx context.Context, prev connectorstore.Reader, kind sourcecache.RowKind, scopeKey string) (SourceCacheReplayResult, error) { + prevEngine, ok := sourceCacheEngine(prev) + if !ok { + return SourceCacheReplayResult{}, errors.New("source cache replay: previous sync store is not a pebble store") + } + var res SourceCacheReplayResult + var err error + switch kind { + case sourcecache.RowKindResources: + res, err = s.engine.ReplaySourceCacheResources(ctx, prevEngine, scopeKey) + case sourcecache.RowKindEntitlements: + res, err = s.engine.ReplaySourceCacheEntitlements(ctx, prevEngine, scopeKey) + case sourcecache.RowKindGrants: + res, err = s.engine.ReplaySourceCacheGrants(ctx, prevEngine, scopeKey) + default: + return SourceCacheReplayResult{}, fmt.Errorf("source cache replay: invalid row kind %q", kind) + } + if err != nil { + return SourceCacheReplayResult{}, err + } + if res.Rows > 0 || res.ResumedRowsCleared > 0 { + // ResumedRowsCleared counts destination rows the resume pre-clear + // deleted before an (possibly empty) copy — a mutation that must + // survive Close even when nothing was copied back. + s.MarkDirty() + } + return res, nil +} + +// ApplyTombstones applies one page's delta tombstones; see the interface +// doc for id semantics. Every delete path returns its row count so the +// syncer's tombstone accounting is symmetric across kinds. +// +// NOTE on the bare-id lookup safety contract (engine/pebble/lookup.go): +// sync paths normally must not resolve rows by string. Canonical-id +// tombstones are a deliberate, narrow exception: the ids are strings the +// connector itself emitted for these rows, volumes are delta-sized (not +// O(rows)), and resolution keeps the exactly-one rule — an ambiguous id +// fails the sync loudly rather than guessing a delete. Grant tombstones +// avoid string resolution entirely by matching stored ids within the +// scope's own rows. +func (s *pebbleStore) ApplyTombstones(ctx context.Context, kind sourcecache.RowKind, scopeKey string, deletedIDs, deletedPrincipalIDs []string) (int64, error) { + var total int64 + + if len(deletedIDs) > 0 { + switch kind { + case sourcecache.RowKindGrants: + idSet := make(map[string]struct{}, len(deletedIDs)) + for _, id := range deletedIDs { + idSet[id] = struct{}{} + } + deleted, err := s.engine.DeleteGrantsByExternalIDsInScope(ctx, scopeKey, idSet) + if err != nil { + return total, fmt.Errorf("source cache grant-id delete for scope %q: %w", scopeKey, err) + } + total += deleted + case sourcecache.RowKindEntitlements: + for _, id := range deletedIDs { + deleted, err := s.engine.DeleteEntitlementRecord(ctx, id) + if err != nil { + return total, fmt.Errorf("source cache delete entitlement %q: %w", id, err) + } + if deleted { + total++ + } + } + case sourcecache.RowKindResources: + for _, id := range deletedIDs { + r, err := bid.ParseResourceBid(id) + if err != nil { + return total, fmt.Errorf("source cache delete resource: invalid resource bid %q: %w", id, err) + } + rid := r.GetId() + deleted, err := s.engine.DeleteResourceRecord(ctx, rid.GetResourceType(), rid.GetResource()) + if err != nil { + return total, fmt.Errorf("source cache delete resource %q: %w", id, err) + } + if deleted { + total++ + } + } + default: + return total, fmt.Errorf("source cache delete: invalid row kind %q", kind) + } + } + + if len(deletedPrincipalIDs) > 0 { + idSet := make(map[string]struct{}, len(deletedPrincipalIDs)) + for _, id := range deletedPrincipalIDs { + idSet[id] = struct{}{} + } + var deleted int64 + var err error + switch kind { + case sourcecache.RowKindGrants: + deleted, err = s.engine.DeleteGrantsByPrincipalsInScope(ctx, scopeKey, idSet) + case sourcecache.RowKindResources: + deleted, err = s.engine.DeleteResourcesByIDsInScope(ctx, scopeKey, idSet) + case sourcecache.RowKindEntitlements: + return total, fmt.Errorf("source cache scoped delete: not supported for entitlements") + default: + return total, fmt.Errorf("source cache scoped delete: invalid row kind %q", kind) + } + if err != nil { + return total, fmt.Errorf("source cache scoped delete for scope %q: %w", scopeKey, err) + } + total += deleted + } + + if total > 0 { + s.MarkDirty() + } + return total, nil +} diff --git a/pkg/dotc1z/sync_runs.go b/pkg/dotc1z/sync_runs.go index 9171ac446..ee53c2f39 100644 --- a/pkg/dotc1z/sync_runs.go +++ b/pkg/dotc1z/sync_runs.go @@ -518,7 +518,7 @@ func (c *C1File) CheckpointSync(ctx context.Context, syncToken string) error { return err } - c.dbUpdated = true + c.dbUpdated.Store(true) return nil } @@ -707,7 +707,7 @@ func (c *C1File) insertSyncRunWithLink(ctx context.Context, syncID string, syncT if err != nil { return err } - c.dbUpdated = true + c.dbUpdated.Store(true) return nil } @@ -762,7 +762,7 @@ func (c *C1File) endSyncRun(ctx context.Context, syncID string) error { if err != nil { return err } - c.dbUpdated = true + c.dbUpdated.Store(true) // Run stats to generate and save the cached stats. _, _, statsErr := c.stats(ctx, connectorstore.SyncTypeAny, syncID, true) @@ -803,7 +803,7 @@ func (c *C1File) SetSupportsDiff(ctx context.Context, syncID string) error { if err != nil { return err } - c.dbUpdated = true + c.dbUpdated.Store(true) return nil } @@ -840,7 +840,7 @@ func (c *C1File) SetSyncLink(ctx context.Context, syncID string, linkedSyncID st if err != nil { return err } - c.dbUpdated = true + c.dbUpdated.Store(true) return nil } @@ -930,7 +930,7 @@ func (c *C1File) Cleanup(ctx context.Context) error { l.Debug("vacuum complete") } - c.dbUpdated = true + c.dbUpdated.Store(true) // If DB is open in WAL mode, truncate the WAL. var journalMode string @@ -1025,7 +1025,7 @@ func (c *C1File) DeleteSyncRun(ctx context.Context, syncID string) error { deleted += rowsDeleted l.Debug("deleted sync run", zap.String("sync_id", syncID), zap.Int64("rows", deleted)) - c.dbUpdated = true + c.dbUpdated.Store(true) return nil } @@ -1063,7 +1063,7 @@ func (c *C1File) Vacuum(ctx context.Context) error { } } - c.dbUpdated = true + c.dbUpdated.Store(true) return nil } diff --git a/pkg/field/defaults.go b/pkg/field/defaults.go index e45726643..03a76342b 100644 --- a/pkg/field/defaults.go +++ b/pkg/field/defaults.go @@ -264,23 +264,37 @@ var ( WithDescription("The path to the c1z file to sync external baton resources with"), WithPersistent(true), WithExportTarget(ExportTargetNone)) + // PreviousSyncC1ZField is hidden by default: source-cache replay is + // author-opt-in functionality (connectorrunner.WithKeepPreviousSyncC1Z), + // and non-replaying connectors shouldn't advertise it to operators. + // DefineConfiguration unhides it for connectors that declare the + // capability; the flag still parses everywhere (hidden ≠ disabled) so + // expert/debug use keeps working. + PreviousSyncC1ZField = StringField("previous-sync-c1z", + WithDescription("The path to the previous sync c1z file to use as a source-cache replay input"), + WithPersistent(true), + WithHidden(true), + WithExportTarget(ExportTargetNone)) externalResourceEntitlementIdFilter = StringField("external-resource-entitlement-id-filter", WithDescription("The entitlement that external users, groups must have access to sync external baton resources"), WithPersistent(true), WithExportTarget(ExportTargetNone)) // KeepPreviousSyncC1ZField is the CUSTOMER's runtime half of the - // service-mode ETag-replay opt-in: keep the last successfully + // service-mode source-cache-replay opt-in: keep the last successfully // uploaded c1z on disk as a spare and feed it to the next full sync // as the previous-sync replay source. It only takes effect on - // connectors whose author also declared ETag-replay support at build + // connectors whose author also declared source-cache-replay support at build // time (connectorrunner.WithKeepPreviousSyncC1Z) — both are // required. Costs one c1z of local disk. + // Hidden by default for the same reason as PreviousSyncC1ZField; unhidden + // when the connector author declares the replay capability. KeepPreviousSyncC1ZField = BoolField("keep-previous-sync-c1z", - WithDescription("Keep the previously synced c1z on disk to enable ETag replay across service-mode syncs "+ - "(requires a connector that supports ETag replay; costs one c1z of local disk)"), + WithDescription("Keep the previously synced c1z on disk to enable source-cache replay across service-mode syncs "+ + "(requires a connector that supports source-cache replay; costs one c1z of local disk)"), WithDefaultValue(false), WithPersistent(true), + WithHidden(true), WithExportTarget(ExportTargetNone)) LambdaServerClientIDField = StringField("lambda-client-id", WithRequired(true), WithDescription("The oauth client id to use with the configuration endpoint"), @@ -431,6 +445,7 @@ var DefaultFields = append([]SchemaField{ skipEntitlementsAndGrants, skipGrants, externalResourceC1ZField, + PreviousSyncC1ZField, externalResourceEntitlementIdFilter, KeepPreviousSyncC1ZField, diffSyncsField, diff --git a/pkg/field/fields.go b/pkg/field/fields.go index 1107861c1..738d1ca63 100644 --- a/pkg/field/fields.go +++ b/pkg/field/fields.go @@ -126,6 +126,34 @@ func (s SchemaField) ExportAs(et ExportTarget) SchemaField { return c } +// WithConnectorDefault returns a copy of a shared/default SDK field carrying +// a connector-specific default value. Include the copy in the connector's +// field.Configuration Fields — DefineConfiguration replaces the SDK's copy +// with it (same re-export mechanism as ExportAs), so --help, flag parsing, +// and exported config schemas all reflect the connector's default. +// +// Value-resolution precedence is unchanged and sentinel-free: an explicit +// flag beats the environment beats the config file beats this default (the +// default lives on the flag itself, so a user-supplied zero value remains +// distinguishable from "unset"). +// +// Example — a connector whose sync fans out well declares its own worker +// default without hiding the shared flag's semantics: +// +// field.NewConfiguration([]field.SchemaField{ +// field.WithConnectorDefault(field.WorkerCountField, 16), +// ... +// }) +// +// The value's type must match the field's variant (int for IntField, etc.); +// a mismatch fails loudly at startup when the flag is registered. +func WithConnectorDefault[T SchemaTypes](s SchemaField, defaultValue T) SchemaField { + c := s + c.DefaultValue = defaultValue + c.WasReExported = true + return c +} + // Go doesn't allow generic methods on a non-generic struct. func ValidateField[T SchemaTypes](s *SchemaField, value T) (bool, error) { return s.validate(value) diff --git a/pkg/lambda/grpc/continuation_wire_test.go b/pkg/lambda/grpc/continuation_wire_test.go new file mode 100644 index 000000000..a6029283d --- /dev/null +++ b/pkg/lambda/grpc/continuation_wire_test.go @@ -0,0 +1,144 @@ +package grpc + +// Wire round-trip pins for the source-cache lookup continuation +// annotations (SourceCacheLookupOffer / Ask / Answers). The protocol rides +// request and response annotations through this transport, so both +// encodings must preserve them for peers that know the types: +// +// - v2 wire frame: lossless binary proto (covered here and by the +// composed lambda-stack e2e in pkg/sync); +// - legacy protojson: known types survive; the recursive annotation +// filter only prunes types missing from the registry — which is the +// designed degradation for old peers (offer stripped → connector +// never defers → cold sync), not a data-integrity risk. + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/require" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/anypb" + + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + pbtransport "github.com/conductorone/baton-sdk/pb/c1/transport/v1" + "github.com/conductorone/baton-sdk/pkg/annotations" +) + +func continuationRequestPayload(t *testing.T) []byte { + t.Helper() + annos := annotations.New(&v2.SourceCacheLookupOffer{}) + annos.Update(v2.SourceCacheLookupAnswers_builder{ + Answers: []*v2.SourceCacheLookupAnswers_Answer{ + v2.SourceCacheLookupAnswers_Answer_builder{ + RowKind: "grants", + ScopeKey: "groups/g1/members", + Found: true, + CacheValidator: `W/"abc123"`, + }.Build(), + v2.SourceCacheLookupAnswers_Answer_builder{ + RowKind: "grants", + ScopeKey: "groups/g2/members", + Found: false, + }.Build(), + }, + }.Build()) + inner := v2.GrantsServiceListGrantsRequest_builder{ + Resource: v2.Resource_builder{ + Id: v2.ResourceId_builder{ResourceType: "group", Resource: "g1"}.Build(), + }.Build(), + Annotations: annos, + }.Build() + + req, err := NewRequest("/c1.connector.v2.GrantsService/ListGrants", inner, metadata.MD{}) + require.NoError(t, err) + payload, err := json.Marshal(req) + require.NoError(t, err) + return payload +} + +func assertContinuationRequest(t *testing.T, req *Request) { + t.Helper() + inner := &v2.GrantsServiceListGrantsRequest{} + require.NoError(t, req.UnmarshalRequest(inner)) + annos := annotations.Annotations(inner.GetAnnotations()) + require.True(t, annos.Contains(&v2.SourceCacheLookupOffer{}), "offer must survive the wire") + answers := &v2.SourceCacheLookupAnswers{} + ok, err := annos.Pick(answers) + require.NoError(t, err) + require.True(t, ok, "answers must survive the wire") + require.Len(t, answers.GetAnswers(), 2) + require.True(t, answers.GetAnswers()[0].GetFound()) + require.Equal(t, `W/"abc123"`, answers.GetAnswers()[0].GetCacheValidator()) + require.False(t, answers.GetAnswers()[1].GetFound(), "explicit not-found must survive (distinct from absent)") +} + +func TestContinuationAnnotations_RequestRoundTrip_V2Frame(t *testing.T) { + payload := continuationRequestPayload(t) + + decoded := &Request{} + require.NoError(t, json.Unmarshal(payload, decoded)) + require.True(t, decoded.wireV2, "dual-encoded payload must decode via the v2 frame") + assertContinuationRequest(t, decoded) +} + +func TestContinuationAnnotations_RequestRoundTrip_LegacyJSON(t *testing.T) { + payload := continuationRequestPayload(t) + + // Strip the v2 frame fields, forcing the legacy protojson path (an + // old-SDK worker's encoding, or the frame-dropped oversize fallback). + var obj map[string]json.RawMessage + require.NoError(t, json.Unmarshal(payload, &obj)) + delete(obj, "v") + delete(obj, "frame") + legacy, err := json.Marshal(obj) + require.NoError(t, err) + + decoded := &Request{} + require.NoError(t, json.Unmarshal(legacy, decoded)) + require.False(t, decoded.wireV2) + // Types are registered in this process: the annotation filter keeps them. + assertContinuationRequest(t, decoded) +} + +func TestContinuationAnnotations_AskResponseRoundTrip(t *testing.T) { + inner := v2.GrantsServiceListGrantsResponse_builder{ + Annotations: annotations.New(v2.SourceCacheLookupAsk_builder{ + Queries: []*v2.SourceCacheLookupAsk_Query{ + v2.SourceCacheLookupAsk_Query_builder{RowKind: "grants", ScopeKey: "groups/g1/members"}.Build(), + }, + }.Build()), + }.Build() + + // Response path: frame encoding is selected when the request carried a + // frame (wireV2), legacy protojson otherwise. Pin both. + for _, wireV2 := range []bool{true, false} { + anyResp, err := anypb.New(inner) + require.NoError(t, err) + anyStatus, err := anypb.New(status.New(codes.OK, "OK").Proto()) + require.NoError(t, err) + resp := &Response{ + msg: pbtransport.Response_builder{ + Resp: anyResp, + Status: anyStatus, + }.Build(), + wireV2: wireV2, + } + payload, err := json.Marshal(resp) + require.NoError(t, err) + + decoded := &Response{} + require.NoError(t, json.Unmarshal(payload, decoded)) + out := &v2.GrantsServiceListGrantsResponse{} + require.NoError(t, decoded.UnmarshalResponse(out)) + ask := &v2.SourceCacheLookupAsk{} + annos := annotations.Annotations(out.GetAnnotations()) + ok, err := annos.Pick(ask) + require.NoError(t, err) + require.True(t, ok, "ask must survive the wire (wireV2=%v)", wireV2) + require.Len(t, ask.GetQueries(), 1) + require.Equal(t, "groups/g1/members", ask.GetQueries()[0].GetScopeKey()) + } +} diff --git a/pkg/lambda/grpc/wire.go b/pkg/lambda/grpc/wire.go index 8be74902f..78160eaa0 100644 --- a/pkg/lambda/grpc/wire.go +++ b/pkg/lambda/grpc/wire.go @@ -81,8 +81,9 @@ func spliceWireFrame(legacy []byte, msg proto.Message) ([]byte, error) { } var buf bytes.Buffer buf.Grow(len(legacy) + len(suffix)) - buf.Write(legacy[:len(legacy)-1]) - buf.WriteByte(',') - buf.Write(suffix[1:]) + // bytes.Buffer writes are documented to never return an error. + _, _ = buf.Write(legacy[:len(legacy)-1]) + _ = buf.WriteByte(',') + _, _ = buf.Write(suffix[1:]) return buf.Bytes(), nil } diff --git a/pkg/sourcecache/compat.go b/pkg/sourcecache/compat.go new file mode 100644 index 000000000..0619381cb --- /dev/null +++ b/pkg/sourcecache/compat.go @@ -0,0 +1,92 @@ +package sourcecache + +import ( + "crypto/sha256" + "encoding/hex" + "sort" + "strconv" + "strings" +) + +// Replay-compatibility key. +// +// Replay is OPTIONAL — a pure optimization — and is permitted only when +// the prior artifact and the current run match BYTE-EXACTLY on every +// component below. Any absent, unreadable, or mismatched component +// degrades the sync to cold (no-op lookup): a wrong cold sync does not +// exist, a wrong warm sync is silent bad data. Compatibility is never +// inferred from versions (semver says nothing about replay semantics); +// each component is an explicit declaration by the party that owns it. + +// MaterializationPolicyGeneration is the SDK's replay-compatibility +// component: bump it when the SDK changes how response rows or their +// side effects MATERIALIZE into the store in a replay-visible way — row +// translation, scope stamping, index derivation semantics, replay-carried +// side effects (child scheduling), or the post-collection invariant +// policy. Bumping it costs every deployment exactly one cold sync after +// upgrade; NOT bumping it when materialization changed replays rows the +// new code would have written differently. +const MaterializationPolicyGeneration = "1" + +// CompatKey is the replay-compatibility key recorded into every +// source-cache-enabled artifact and required to match exactly before +// that artifact may serve replay. +type CompatKey struct { + // ConnectorCacheGeneration is the connector's declared cache + // generation (SourceCacheCapability.cache_generation): scope + // computation, validator semantics, row/id construction. + ConnectorCacheGeneration string + // ConnectorConfigFingerprint is the connector's declared digest of + // config/permission inputs that change what upstream data it can see + // (SourceCacheCapability.config_fingerprint). + ConnectorConfigFingerprint string + // SDKMaterializationGeneration is MaterializationPolicyGeneration at + // record time. + SDKMaterializationGeneration string + // SyncSelectionFingerprint digests the SDK-side sync-shaping inputs + // (enabled resource types, skip flags): a selection change means the + // prior artifact's scopes cover a different row universe. + SyncSelectionFingerprint string +} + +// MismatchReason compares the current run's key against the previous +// artifact's recorded key and returns "" on an exact match, or a +// human-readable reason naming the FIRST mismatched component. Values +// are deliberately not echoed (fingerprints may derive from secrets); +// the component name is enough to act on. +func (k CompatKey) MismatchReason(prev CompatKey) string { + switch { + case k.ConnectorCacheGeneration != prev.ConnectorCacheGeneration: + return "connector cache-generation changed since the previous artifact was recorded" + case k.ConnectorConfigFingerprint != prev.ConnectorConfigFingerprint: + return "connector configuration/permission fingerprint changed since the previous artifact was recorded" + case k.SDKMaterializationGeneration != prev.SDKMaterializationGeneration: + return "SDK materialization-policy generation changed since the previous artifact was recorded" + case k.SyncSelectionFingerprint != prev.SyncSelectionFingerprint: + return "sync-selection fingerprint (enabled types, filters) changed since the previous artifact was recorded" + default: + return "" + } +} + +// SelectionFingerprint digests the sync-shaping inputs into the +// SyncSelectionFingerprint component. Canonical: resource type ids are +// sorted and length-prefixed (no delimiter injection), flags appended +// explicitly. An empty selection (all types, nothing skipped) has a +// well-defined fingerprint too — "no selection" must still match only +// "no selection". +func SelectionFingerprint(resourceTypeIDs []string, skipEntitlementsAndGrants, skipGrants bool) string { + sorted := append([]string{}, resourceTypeIDs...) + sort.Strings(sorted) + parts := make([]string, 0, len(sorted)+3) + parts = append(parts, "v1\x00types:") + for _, id := range sorted { + parts = append(parts, strconv.Itoa(len(id))+":"+id+"\x00") + } + parts = append(parts, + "skipEG:"+strconv.FormatBool(skipEntitlementsAndGrants), + "\x00skipG:"+strconv.FormatBool(skipGrants), + ) + sum := sha256.Sum256([]byte(strings.Join(parts, ""))) + return hex.EncodeToString(sum[:]) +} diff --git a/pkg/sourcecache/context.go b/pkg/sourcecache/context.go new file mode 100644 index 000000000..da56f938b --- /dev/null +++ b/pkg/sourcecache/context.go @@ -0,0 +1,20 @@ +package sourcecache + +import "context" + +type scopeContextKey struct{} + +// WithScope returns a context carrying the source-cache scope key for +// rows written under it. The syncer wraps a page's store writes in this +// context when the page carried a SourceCacheRecord annotation; the Pebble +// write path stamps the record's source_scope_key from it. +func WithScope(ctx context.Context, scopeKey string) context.Context { + return context.WithValue(ctx, scopeContextKey{}, scopeKey) +} + +// ScopeFromContext returns the scope key set by WithScope, or "" when +// the context carries none (the common, unstamped case). +func ScopeFromContext(ctx context.Context) string { + s, _ := ctx.Value(scopeContextKey{}).(string) + return s +} diff --git a/pkg/sourcecache/continuation.go b/pkg/sourcecache/continuation.go new file mode 100644 index 000000000..21f24e5c5 --- /dev/null +++ b/pkg/sourcecache/continuation.go @@ -0,0 +1,285 @@ +package sourcecache + +// Lookup continuation (ask/answer): the lookup transport for connector +// runtimes that cannot call back to the syncer mid-request (single-shot +// request/response tunnels, e.g. gRPC-over-Lambda). The connector's first +// execution of a page ("phase 1") records the scopes it needs and fails +// with ErrLookupDeferred; the SDK converts that into a +// SourceCacheLookupAsk response; the syncer resolves the queries against +// its local previous-sync store and re-invokes the same request with +// SourceCacheLookupAnswers attached ("phase 2"), where the same connector +// code gets real answers. See docs/tasks/source-cache-lambda-lookup.md. + +import ( + "context" + "errors" + "fmt" + "sync" + + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" +) + +// ErrLookupDeferred is returned by a deferring Lookup (phase 1 of the +// ask/answer continuation) when the answer is not yet available. The SDK +// intercepts it and answers the RPC with a SourceCacheLookupAsk. +// +// Propagation contract for connectors: wrap with %w if you must, NEVER +// swallow. The SDK matches with errors.Is, so idiomatic wrapping +// (fmt.Errorf("listing members: %w", err)) is fine. A connector that +// logs-and-continues past this error re-asks for scopes it already +// "handled" and turns every warm sync into a hard failure at the bounce +// cap. +var ErrLookupDeferred = errors.New("source cache lookup deferred: answer arrives on re-invoke") + +// Query identifies one scope to resolve against the previous sync. +type Query struct { + RowKind RowKind + ScopeKey string +} + +// Answer resolves one Query. Found=false means the previous sync has no +// entry for the scope: fetch fresh. (Distinct from a query that got no +// Answer at all, which means unresolved: ask again.) +type Answer struct { + Query + Found bool + CacheValidator string +} + +// BatchLookup is optionally implemented by Lookup implementations that can +// resolve many scopes in one round trip. Connectors should not type-assert +// for it directly; call LookupMany, which falls back to per-query lookups. +type BatchLookup interface { + LookupMany(ctx context.Context, queries []Query) ([]Answer, error) +} + +// LookupMany resolves a batch of queries through lookup, using one round +// trip when the implementation supports it (BatchLookup) and a loop of +// single lookups otherwise. This is the topology-uniform batch API: +// in-process and subprocess lookups loop over local/loopback point-reads; +// a deferring lookup collects the whole batch into ONE ask. +// +// The returned answers are exact and complete for the queried set — one +// Answer per Query, order preserved, explicit Found per entry. (A +// deferring lookup returns ErrLookupDeferred instead of answers; phase 2 +// then answers the same calls. A found answer whose validator does not fit the +// transport size budget arrives as not-found — the scope goes cold, which +// is correct, just slower — never as a silent omission.) +func LookupMany(ctx context.Context, lookup Lookup, queries []Query) ([]Answer, error) { + if bl, ok := lookup.(BatchLookup); ok { + return bl.LookupMany(ctx, queries) + } + answers := make([]Answer, 0, len(queries)) + for _, q := range queries { + entry, found, err := lookup.Lookup(ctx, q.RowKind, q.ScopeKey) + if err != nil { + return nil, err + } + a := Answer{Query: q, Found: found} + if found { + a.CacheValidator = entry.CacheValidator + } + answers = append(answers, a) + } + return answers, nil +} + +// ContinuationLookup is the per-request Lookup installed for the +// ask/answer continuation. It serves lookups from the answers delivered on +// the request (phase 2) and defers everything else by recording the query +// and returning ErrLookupDeferred (phase 1, or a phase 2 that queries +// scopes it did not ask for in phase 1 — allowed but wasteful). +// +// It is constructed per RPC by the SDK; connectors only ever see it as +// SyncOpAttrs.SourceCache. +type ContinuationLookup struct { + mu sync.Mutex + answers map[Query]Answer + asked []Query + askedSet map[Query]struct{} +} + +var ( + _ Lookup = (*ContinuationLookup)(nil) + _ BatchLookup = (*ContinuationLookup)(nil) +) + +// NewContinuationLookup builds a ContinuationLookup pre-loaded with the +// request's answers (nil/empty on phase 1). +func NewContinuationLookup(answers []Answer) *ContinuationLookup { + m := make(map[Query]Answer, len(answers)) + for _, a := range answers { + m[a.Query] = a + } + return &ContinuationLookup{ + answers: m, + askedSet: map[Query]struct{}{}, + } +} + +func (c *ContinuationLookup) Lookup(_ context.Context, rowKind RowKind, scopeKey string) (Entry, bool, error) { + q := Query{RowKind: rowKind, ScopeKey: scopeKey} + c.mu.Lock() + defer c.mu.Unlock() + if a, ok := c.answers[q]; ok { + if !a.Found { + return Entry{}, false, nil + } + return Entry{CacheValidator: a.CacheValidator}, true, nil + } + c.recordLocked(q) + return Entry{}, false, fmt.Errorf("lookup %s/%s: %w", rowKind, scopeKey, ErrLookupDeferred) +} + +func (c *ContinuationLookup) LookupMany(_ context.Context, queries []Query) ([]Answer, error) { + c.mu.Lock() + defer c.mu.Unlock() + answers := make([]Answer, 0, len(queries)) + missing := 0 + for _, q := range queries { + a, ok := c.answers[q] + if !ok { + c.recordLocked(q) + missing++ + continue + } + answers = append(answers, a) + } + if missing > 0 { + // Defer the whole batch: phase 2 re-runs the same call with the + // full answer set (already-answered queries remain answered on the + // re-invoked request). + return nil, fmt.Errorf("batch lookup: %d of %d queries unresolved: %w", missing, len(queries), ErrLookupDeferred) + } + return answers, nil +} + +func (c *ContinuationLookup) recordLocked(q Query) { + if _, seen := c.askedSet[q]; seen { + return + } + c.askedSet[q] = struct{}{} + c.asked = append(c.asked, q) +} + +// Asked returns the queries recorded by deferred lookups, deduplicated, in +// first-ask order. Empty means no lookup deferred and the handler's result +// stands. +func (c *ContinuationLookup) Asked() []Query { + c.mu.Lock() + defer c.mu.Unlock() + out := make([]Query, len(c.asked)) + copy(out, c.asked) + return out +} + +// --- proto conversions (shared by connectorbuilder and the syncer) ------ + +// AskProto builds the SourceCacheLookupAsk annotation for a set of +// deferred queries. +func AskProto(queries []Query) *v2.SourceCacheLookupAsk { + qs := make([]*v2.SourceCacheLookupAsk_Query, 0, len(queries)) + for _, q := range queries { + qs = append(qs, v2.SourceCacheLookupAsk_Query_builder{ + RowKind: string(q.RowKind), + ScopeKey: q.ScopeKey, + }.Build()) + } + return v2.SourceCacheLookupAsk_builder{Queries: qs}.Build() +} + +// maxQueriesPerAsk mirrors SourceCacheLookupAsk.queries.max_items. Asks +// arrive in connector response annotations; annotations.Pick unmarshals the +// Any but does not invoke generated protobuf validation, so this boundary +// enforces the resource cap explicitly. +const maxQueriesPerAsk = 4096 + +// MaxLookupBouncesPerRequest bounds consecutive asks for one request +// (same page token; only the answers annotation differs between +// re-invokes). Owned here so the answer-set cap below stays coherent +// with the syncer's bounce loop, which enforces it: answers accumulate +// across bounces, so the most answers one request can legally carry is +// this many asks' worth of queries. +const MaxLookupBouncesPerRequest = 4 + +// QueriesFromProto extracts and validates the queries of an ask. +func QueriesFromProto(ask *v2.SourceCacheLookupAsk) ([]Query, error) { + if len(ask.GetQueries()) == 0 { + // Without this an empty ask would die on the syncer's no-progress + // guard with a misleading "re-asked already-answered scopes" + // message; name the actual bug. + return nil, fmt.Errorf("source cache lookup ask: no queries (a connector must not ask without at least one unresolved scope)") + } + if len(ask.GetQueries()) > maxQueriesPerAsk { + return nil, fmt.Errorf("source cache lookup ask: %d queries (max %d)", len(ask.GetQueries()), maxQueriesPerAsk) + } + out := make([]Query, 0, len(ask.GetQueries())) + for _, q := range ask.GetQueries() { + kind := RowKind(q.GetRowKind()) + if err := ValidateRowKind(kind); err != nil { + return nil, err + } + if err := ValidateScopeKey(q.GetScopeKey()); err != nil { + return nil, err + } + out = append(out, Query{RowKind: kind, ScopeKey: q.GetScopeKey()}) + } + return out, nil +} + +// AnswersProto builds the SourceCacheLookupAnswers annotation. +func AnswersProto(answers []Answer) *v2.SourceCacheLookupAnswers { + as := make([]*v2.SourceCacheLookupAnswers_Answer, 0, len(answers)) + for _, a := range answers { + as = append(as, v2.SourceCacheLookupAnswers_Answer_builder{ + RowKind: string(a.RowKind), + ScopeKey: a.ScopeKey, + Found: a.Found, + CacheValidator: a.CacheValidator, + }.Build()) + } + return v2.SourceCacheLookupAnswers_builder{Answers: as}.Build() +} + +// Wire caps for answer sets. Answers come from the SDK (a trusted +// parent), but the same validation discipline as QueriesFromProto keeps a +// corrupted or hostile transport from smuggling malformed entries into +// lookup results. +// +// maxAnswersPerMessage is NOT one ask's max_items: answers accumulate +// across bounces (every re-invoke carries the union of all resolved +// queries), so the coherent bound is the most a compliant exchange can +// legally produce — MaxLookupBouncesPerRequest maximal asks. Capping at +// one ask's size would hard-fail a legal 4096-scope first ask followed by +// a single late scope on a later bounce. +const ( + maxAnswersPerMessage = maxQueriesPerAsk * MaxLookupBouncesPerRequest + maxAnswerValidatorLen = 65536 +) + +// AnswersFromProto extracts and validates the answers delivered on a +// request. +func AnswersFromProto(msg *v2.SourceCacheLookupAnswers) ([]Answer, error) { + if len(msg.GetAnswers()) > maxAnswersPerMessage { + return nil, fmt.Errorf("source cache lookup answers: %d entries (max %d)", len(msg.GetAnswers()), maxAnswersPerMessage) + } + out := make([]Answer, 0, len(msg.GetAnswers())) + for _, a := range msg.GetAnswers() { + kind := RowKind(a.GetRowKind()) + if err := ValidateRowKind(kind); err != nil { + return nil, fmt.Errorf("source cache lookup answers: %w", err) + } + if err := ValidateScopeKey(a.GetScopeKey()); err != nil { + return nil, fmt.Errorf("source cache lookup answers: %w", err) + } + if len(a.GetCacheValidator()) > maxAnswerValidatorLen { + return nil, fmt.Errorf("source cache lookup answers: cache validator for scope %q is %d bytes (max %d)", a.GetScopeKey(), len(a.GetCacheValidator()), maxAnswerValidatorLen) + } + out = append(out, Answer{ + Query: Query{RowKind: kind, ScopeKey: a.GetScopeKey()}, + Found: a.GetFound(), + CacheValidator: a.GetCacheValidator(), + }) + } + return out, nil +} diff --git a/pkg/sourcecache/grpc_lookup.go b/pkg/sourcecache/grpc_lookup.go new file mode 100644 index 000000000..b69206adf --- /dev/null +++ b/pkg/sourcecache/grpc_lookup.go @@ -0,0 +1,91 @@ +package sourcecache + +import ( + "context" + "sync/atomic" + "time" + + "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" + "go.uber.org/zap" + + v1 "github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1" +) + +// GRPCLookup is the connector-side Lookup implementation that talks to +// BatonSourceCacheService on the parent SDK. +// +// This is deliberately not routed through the session store: session data +// passes through the connector's local MemorySessionCache (otter), which +// would apply generic TTL/eviction policies to sync-scoped validator state. +// The dedicated service keeps the path uncached and the message shape +// explicit. +// +// The parent has exactly one active Lookup registered at a time (set per +// sync via SetSourceCache on the server), so the wire format carries no +// sync_id; routing is implicit. +// +// Per the Lookup contract, an RPC failure degrades to a miss rather than +// failing the connector's list call: a miss is always safe — the +// connector fetches fresh — while the error would ride back through the +// connector's handler wrappings, where its status code rarely survives to +// the syncer's retryer, hard-failing the sync on a transient loopback +// blip. Failures are logged at most once per minute with a running count +// (this object lives for the whole subprocess run, so a once-ever log +// would make a persistently broken parent path invisible after the first +// scope), and not at all when the caller's context is already done — a +// teardown-time Canceled is noise, and logging it would eat the rate +// window a real failure needs. Validation failures stay loud: a malformed +// row kind or scope key is a connector bug, and degrading it to a miss +// would let the connector silently cold-fetch forever with no signal. +type GRPCLookup struct { + client v1.BatonSourceCacheServiceClient + // failures counts degraded lookups for the log line; lastLogNano is + // the unix-nano timestamp of the last emitted warning. + failures atomic.Int64 + lastLogNano atomic.Int64 +} + +// lookupFailureLogInterval rate-limits the degraded-to-miss warning. +const lookupFailureLogInterval = time.Minute + +// NewGRPCLookup returns a Lookup backed by the given client. A nil client +// yields NoopLookup so callers can configure the client optionally without +// nil checks at every call site. +func NewGRPCLookup(client v1.BatonSourceCacheServiceClient) Lookup { + if client == nil { + return NoopLookup{} + } + return &GRPCLookup{client: client} +} + +func (g *GRPCLookup) Lookup(ctx context.Context, rowKind RowKind, scopeKey string) (Entry, bool, error) { + if err := ValidateRowKind(rowKind); err != nil { + return Entry{}, false, err + } + if err := ValidateScopeKey(scopeKey); err != nil { + return Entry{}, false, err + } + resp, err := g.client.Lookup(ctx, v1.LookupRequest_builder{ + RowKind: string(rowKind), + ScopeKey: scopeKey, + }.Build()) + if err != nil { + count := g.failures.Add(1) + if ctx.Err() == nil { + now := time.Now().UnixNano() + last := g.lastLogNano.Load() + if now-last >= int64(lookupFailureLogInterval) && g.lastLogNano.CompareAndSwap(last, now) { + ctxzap.Extract(ctx).Warn("source cache rpc lookup failed; treating as miss", + zap.Error(err), + zap.Int64("failures_since_start", count)) + } + } + // Intentional nil error: a failed lookup degrades to a miss + // (connector fetches fresh) rather than failing the connector call. + return Entry{}, false, nil + } + if !resp.GetFound() { + return Entry{}, false, nil + } + return Entry{CacheValidator: resp.GetCacheValidator()}, true, nil +} diff --git a/pkg/sourcecache/grpc_lookup_test.go b/pkg/sourcecache/grpc_lookup_test.go new file mode 100644 index 000000000..ba9d6112e --- /dev/null +++ b/pkg/sourcecache/grpc_lookup_test.go @@ -0,0 +1,145 @@ +package sourcecache + +import ( + "context" + "errors" + "sync" + "testing" + + "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" + "github.com/stretchr/testify/require" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + "google.golang.org/grpc" + + v1 "github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1" +) + +type fakeLookupClient struct { + resp *v1.LookupResponse + err error +} + +func (f *fakeLookupClient) Lookup(context.Context, *v1.LookupRequest, ...grpc.CallOption) (*v1.LookupResponse, error) { + return f.resp, f.err +} + +// TestGRPCLookupDegradesRPCErrorToMiss pins the Lookup contract for the +// subprocess topology: a transport failure to the parent degrades to a +// miss (the connector fetches fresh — always safe) instead of failing the +// connector's list call, whose error would rarely keep a retryable status +// code by the time it reaches the syncer's retryer. +func TestGRPCLookupDegradesRPCErrorToMiss(t *testing.T) { + ctx := context.Background() + lookup := NewGRPCLookup(&fakeLookupClient{err: errors.New("connection refused")}) + + scope := HashScope("https://example.test/teams/1/members?page=1") + _, found, err := lookup.Lookup(ctx, RowKindGrants, scope) + require.NoError(t, err, "an RPC failure must degrade to a miss, not fail the connector call") + require.False(t, found) +} + +// capturedLog is one recorded zap entry (message + flattened fields). +// zaptest/observer is not vendored, so recordingCore stands in for it +// (same pattern as pkg/c1zsanitize). +type capturedLog struct { + msg string + fields map[string]any +} + +type recordingCore struct { + mu *sync.Mutex + logs *[]capturedLog +} + +func (c recordingCore) Enabled(zapcore.Level) bool { return true } +func (c recordingCore) With([]zapcore.Field) zapcore.Core { return c } +func (c recordingCore) Check(e zapcore.Entry, ce *zapcore.CheckedEntry) *zapcore.CheckedEntry { + return ce.AddCore(e, c) +} +func (c recordingCore) Sync() error { return nil } +func (c recordingCore) Write(e zapcore.Entry, fs []zapcore.Field) error { + enc := zapcore.NewMapObjectEncoder() + for _, f := range fs { + f.AddTo(enc) + } + c.mu.Lock() + *c.logs = append(*c.logs, capturedLog{msg: e.Message, fields: enc.Fields}) + c.mu.Unlock() + return nil +} + +func newRecordingContext() (context.Context, *[]capturedLog) { + logs := &[]capturedLog{} + logger := zap.New(recordingCore{mu: &sync.Mutex{}, logs: logs}) + return ctxzap.ToContext(context.Background(), logger), logs +} + +// TestGRPCLookupFailureLogRateLimited pins the failure-log policy: the +// degraded-to-miss warning is rate-limited (not once-ever — this object +// lives for the whole subprocess run), carries a running failure count, +// and is skipped entirely when the caller's context is already done so +// teardown-time cancellation can't consume the rate window. +func TestGRPCLookupFailureLogRateLimited(t *testing.T) { + ctx, logs := newRecordingContext() + lookup := NewGRPCLookup(&fakeLookupClient{err: errors.New("connection refused")}) + scope := HashScope("https://example.test/teams/1/members?page=1") + + // A failure on an already-canceled context degrades silently: the + // connector call is dying anyway, and logging would burn the rate + // window a real failure needs. + canceledCtx, cancel := context.WithCancel(ctx) + cancel() + _, found, err := lookup.Lookup(canceledCtx, RowKindGrants, scope) + require.NoError(t, err) + require.False(t, found) + require.Empty(t, *logs, "a failure on a done context must not log") + + // The first live-context failure logs immediately, with the running + // count including the silent canceled-context failure above. + _, found, err = lookup.Lookup(ctx, RowKindGrants, scope) + require.NoError(t, err) + require.False(t, found) + require.Len(t, *logs, 1) + require.Equal(t, "source cache rpc lookup failed; treating as miss", (*logs)[0].msg) + require.EqualValues(t, 2, (*logs)[0].fields["failures_since_start"]) + + // Immediate subsequent failures still degrade to misses but stay + // within the rate window and do not log again. + for range 10 { + _, found, err = lookup.Lookup(ctx, RowKindGrants, scope) + require.NoError(t, err) + require.False(t, found) + } + require.Len(t, *logs, 1, "failures inside the rate window must not log") +} + +// TestGRPCLookupValidationStaysLoud pins the other half: malformed +// arguments are connector bugs and must fail, never degrade — a miss here +// would let a broken connector silently cold-fetch forever. +func TestGRPCLookupValidationStaysLoud(t *testing.T) { + ctx := context.Background() + lookup := NewGRPCLookup(&fakeLookupClient{resp: v1.LookupResponse_builder{Found: true, CacheValidator: "e"}.Build()}) + + _, _, err := lookup.Lookup(ctx, RowKind("bogus"), HashScope("s")) + require.Error(t, err, "invalid row kind must fail loudly") + + _, _, err = lookup.Lookup(ctx, RowKindGrants, "") + require.Error(t, err, "empty scope key must fail loudly") +} + +func TestGRPCLookupHitAndMiss(t *testing.T) { + ctx := context.Background() + scope := HashScope("https://example.test/teams/1/members?page=1") + + hit := NewGRPCLookup(&fakeLookupClient{resp: v1.LookupResponse_builder{Found: true, CacheValidator: `W/"etag-1"`}.Build()}) + entry, found, err := hit.Lookup(ctx, RowKindGrants, scope) + require.NoError(t, err) + require.True(t, found) + require.Equal(t, `W/"etag-1"`, entry.CacheValidator) + + miss := NewGRPCLookup(&fakeLookupClient{resp: v1.LookupResponse_builder{Found: false}.Build()}) + _, found, err = miss.Lookup(ctx, RowKindGrants, scope) + require.NoError(t, err) + require.False(t, found) +} diff --git a/pkg/sourcecache/grpc_server.go b/pkg/sourcecache/grpc_server.go new file mode 100644 index 000000000..c47dee384 --- /dev/null +++ b/pkg/sourcecache/grpc_server.go @@ -0,0 +1,116 @@ +package sourcecache + +import ( + "context" + "fmt" + "sync" + "sync/atomic" + + "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" + + v1 "github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1" +) + +// GRPCServer is the parent-side BatonSourceCacheService implementation. +// +// The parent SDK holds a single GRPCServer for the lifetime of the connector +// subprocess and swaps the active Lookup via SetSourceCache as syncs come +// and go. The syncer installs a real lookup once it has resolved a usable +// previous sync, and clears it when the sync ends so a late RPC can't serve +// from a store the syncer no longer owns. +// +// Until the first SetSourceCache call the server answers every lookup with +// found=false, which the connector treats as "no previous sync" and falls +// back to an unconditional fetch. +type GRPCServer struct { + v1.UnimplementedBatonSourceCacheServiceServer + lookup atomic.Pointer[Lookup] + + // installMu guards the install accounting below; the lookup pointer + // stays atomic so the RPC hot path never takes the mutex. + installMu sync.Mutex + // installs counts live installs (non-nil SetSourceCache calls not yet + // cleared). The wire carries no sync id, so an RPC cannot be + // attributed to one of several concurrent syncs; the slot is only + // servable while exactly one owner holds it. + installs int + // contended latches when a second concurrent install lands. From that + // point every lookup misses (found=false — always safe: the connector + // fetches cold) until the slot fully drains, because the server keeps + // no per-owner state and cannot know which lookup a surviving sync + // owns after the others clear. + contended bool +} + +var _ v1.BatonSourceCacheServiceServer = (*GRPCServer)(nil) +var _ SourceCacheSetter = (*GRPCServer)(nil) + +// NewGRPCServer returns a GRPCServer with no active Lookup registered. +func NewGRPCServer() *GRPCServer { + return &GRPCServer{} +} + +// SetSourceCache installs (non-nil) or clears (nil) a sync's lookup. Safe +// to call concurrently with in-flight RPCs: existing RPCs continue against +// the value they read at entry; new RPCs see the swapped value. +// +// The slot is SINGLE — the wire carries no sync id — so with two live +// syncs the server cannot attribute a Lookup RPC to either one. Serving +// the last-installed lookup would cross-wire them: one sync's connector +// would resolve validators against the other's previous store, and its +// syncer would then replay rows from its OWN store under a validator the +// other artifact vouched for — stale rows sealed as current. A miss, by +// contrast, is always safe (the connector fetches cold). So on contention +// the server blinds every caller until all concurrent owners have +// cleared, and warns loudly so the degraded overlap is visible. +func (s *GRPCServer) SetSourceCache(ctx context.Context, lookup Lookup) { + s.installMu.Lock() + defer s.installMu.Unlock() + if lookup == nil { + if s.installs > 0 { + s.installs-- + } + if s.installs == 0 { + // Slot fully drained: the next sync starts clean. + s.contended = false + } + s.lookup.Store(nil) + return + } + s.installs++ + if s.installs > 1 || s.contended { + s.contended = true + s.lookup.Store(nil) + ctxzap.Extract(ctx).Warn("source cache: concurrent syncs share this connector's single lookup slot; " + + "serving misses to every sync (cold fetches) until the overlap drains, to avoid cross-wiring validators between previous-sync artifacts") + return + } + s.lookup.Store(&lookup) +} + +func (s *GRPCServer) Lookup(ctx context.Context, req *v1.LookupRequest) (*v1.LookupResponse, error) { + rowKind := RowKind(req.GetRowKind()) + if err := ValidateRowKind(rowKind); err != nil { + return nil, err + } + scopeKey := req.GetScopeKey() + if err := ValidateScopeKey(scopeKey); err != nil { + return nil, err + } + + lookupPtr := s.lookup.Load() + if lookupPtr == nil { + return v1.LookupResponse_builder{Found: false}.Build(), nil + } + entry, found, err := (*lookupPtr).Lookup(ctx, rowKind, scopeKey) + if err != nil { + return nil, fmt.Errorf("source cache lookup: %w", err) + } + if !found { + return v1.LookupResponse_builder{Found: false}.Build(), nil + } + return v1.LookupResponse_builder{ + Found: true, + CacheValidator: entry.CacheValidator, + }.Build(), nil +} diff --git a/pkg/sourcecache/grpc_server_test.go b/pkg/sourcecache/grpc_server_test.go new file mode 100644 index 000000000..6859ab80a --- /dev/null +++ b/pkg/sourcecache/grpc_server_test.go @@ -0,0 +1,106 @@ +package sourcecache + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + + v1 "github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1" +) + +// staticLookup answers every scope with a fixed validator, tagging which +// sync's store it stands in for. +type staticLookup struct{ validator string } + +func (l staticLookup) Lookup(context.Context, RowKind, string) (Entry, bool, error) { + return Entry{CacheValidator: l.validator}, true, nil +} + +func lookupReq(t *testing.T) *v1.LookupRequest { + t.Helper() + return v1.LookupRequest_builder{ + RowKind: string(RowKindGrants), + ScopeKey: HashScope("scope"), + }.Build() +} + +func requireServes(t *testing.T, s *GRPCServer, wantValidator string) { + t.Helper() + resp, err := s.Lookup(context.Background(), lookupReq(t)) + require.NoError(t, err) + require.True(t, resp.GetFound()) + require.Equal(t, wantValidator, resp.GetCacheValidator()) +} + +func requireMisses(t *testing.T, s *GRPCServer) { + t.Helper() + resp, err := s.Lookup(context.Background(), lookupReq(t)) + require.NoError(t, err) + require.False(t, resp.GetFound()) +} + +// TestGRPCServerSingleOwnerLifecycle pins the uncontended path: install +// serves, clear blinds, a fresh install serves again. +func TestGRPCServerSingleOwnerLifecycle(t *testing.T) { + ctx := context.Background() + s := NewGRPCServer() + + requireMisses(t, s) + s.SetSourceCache(ctx, staticLookup{validator: "etag-a"}) + requireServes(t, s, "etag-a") + s.SetSourceCache(ctx, nil) + requireMisses(t, s) + s.SetSourceCache(ctx, staticLookup{validator: "etag-b"}) + requireServes(t, s, "etag-b") +} + +// TestGRPCServerContentionBlindsAllOwners pins the cross-wiring defense: +// the wire carries no sync id, so with two live syncs the server cannot +// attribute an RPC to either — serving the last-installed lookup would +// hand sync A validators from sync B's previous artifact while A replays +// rows from its own. On contention every lookup must MISS (cold fetch, +// always safe) until the slot fully drains. +func TestGRPCServerContentionBlindsAllOwners(t *testing.T) { + ctx := context.Background() + s := NewGRPCServer() + + // Sync A installs; sync B overlaps. + s.SetSourceCache(ctx, staticLookup{validator: "etag-a"}) + requireServes(t, s, "etag-a") + s.SetSourceCache(ctx, staticLookup{validator: "etag-b"}) + requireMisses(t, s) + + // A finishes and clears. B is the sole survivor, but the server kept + // no per-owner state — it must stay blind rather than guess. + s.SetSourceCache(ctx, nil) + requireMisses(t, s) + + // A third sync overlapping the still-live B must not be served either + // (this is the exact validator-from-S1/rows-from-S0 shape). + s.SetSourceCache(ctx, staticLookup{validator: "etag-c"}) + requireMisses(t, s) + s.SetSourceCache(ctx, nil) + requireMisses(t, s) + + // Only when the overlap fully drains does the slot reset. + s.SetSourceCache(ctx, nil) + requireMisses(t, s) + s.SetSourceCache(ctx, staticLookup{validator: "etag-d"}) + requireServes(t, s, "etag-d") +} + +// TestGRPCServerUnbalancedClearIsClamped pins the defensive clamp: a +// spurious extra clear (a sync that never installed) must not wedge the +// counter below zero and break the next legitimate install. +func TestGRPCServerUnbalancedClearIsClamped(t *testing.T) { + ctx := context.Background() + s := NewGRPCServer() + + s.SetSourceCache(ctx, nil) + s.SetSourceCache(ctx, nil) + s.SetSourceCache(ctx, staticLookup{validator: "etag-a"}) + requireServes(t, s, "etag-a") + s.SetSourceCache(ctx, nil) + requireMisses(t, s) +} diff --git a/pkg/sourcecache/sourcecache.go b/pkg/sourcecache/sourcecache.go new file mode 100644 index 000000000..fcd0003a9 --- /dev/null +++ b/pkg/sourcecache/sourcecache.go @@ -0,0 +1,150 @@ +// Package sourcecache defines the connector-facing surface of source-cache +// replay (see proto/c1/connector/v2/annotation_source_cache.proto). +// +// A connector that can cheaply revalidate upstream data — HTTP conditional +// requests (GitHub), delta queries (Microsoft Graph) — opts in by attaching +// SourceCacheCapability MODE_READ_WRITE to its Validate response. During a +// sync it looks up the previous validator for a scope via the Lookup the SDK +// provides on SyncOpAttrs, revalidates upstream, and either emits fresh rows +// tagged with SourceCacheRecord or asks the SDK to replay the previous rows +// with SourceCacheReplay. +// +// The connector owns scope computation; the SDK only keys storage by the +// connector-supplied scope key. The validator (HTTP ETag, delta token, ...) is opaque +// to the SDK. +// +// Invariant that keeps replay safe: a connector must only emit +// SourceCacheReplay for a scope whose validator it received from THIS sync's +// Lookup. The lookup need not happen in the same call that emits the +// replay: a planning call may batch-resolve many scopes and pass the +// verdicts to sibling cursors through EnqueuePageTokens page tokens — that +// satisfies the invariant, because the validator still originates from the +// consuming sync. What's forbidden is a validator that outlives a sync +// (connector-side caches, config, upstream echoes). When source cache is +// disabled or degraded (no capability, no usable previous sync, unsupported +// storage engine) the SDK installs NoopLookup, every lookup misses, and a +// well-behaved connector naturally falls back to full fetch. +// +// Replay equivalence: a cached sync must reproduce what a full resync +// would produce. Replayed rows are verbatim copies of the previous sync's +// rows with one deliberate exception — expander-written Sources on direct +// grants (classified by a self-source entry, mirroring RollbackExpansion) +// are stripped at copy time so the current sync's expansion recomputes +// them from true state; re-expansion only adds contributions, so carrying +// them verbatim would immortalize contributions removed upstream. +// Connector-set Sources (no self-source) are public connector data and +// survive replay byte-for-byte. +package sourcecache + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "strings" + "time" +) + +// RowKind partitions source-cache scopes by the row type they produce. +// It doubles as the row_kind value stored in SourceCacheEntryRecord. +type RowKind string + +const ( + RowKindResources RowKind = "resources" + RowKindEntitlements RowKind = "entitlements" + RowKindGrants RowKind = "grants" +) + +// Valid reports whether k is one of the defined row kinds. +func (k RowKind) Valid() bool { + switch k { + case RowKindResources, RowKindEntitlements, RowKindGrants: + return true + } + return false +} + +// ValidateRowKind returns an error if rowKind is not one of the known +// RowKind* constants. +func ValidateRowKind(rowKind RowKind) error { + if !rowKind.Valid() { + return fmt.Errorf("invalid source cache row kind: %q", rowKind) + } + return nil +} + +// maxScopeKeyLen bounds scope identifiers on the wire and in storage +// keys. Deliberately generous: the shape is a connector convention +// (HashScope produces 64 hex chars) and is not enforced beyond +// non-emptiness and this cap while the model is being proven out against +// real providers. +const maxScopeKeyLen = 256 + +// ValidateScopeKey returns an error when scopeKey is empty, unreasonably +// long, or contains a NUL byte. Connectors conventionally use HashScope, +// but any stable identifier is accepted. NUL is reserved: composite +// "kind\x00scope" keys appear in inspection surfaces (engine snapshots), +// and a scope key containing NUL could alias another entry there. +func ValidateScopeKey(scopeKey string) error { + if scopeKey == "" { + return fmt.Errorf("source cache scope key is required") + } + if len(scopeKey) > maxScopeKeyLen { + return fmt.Errorf("source cache scope key too long: %d bytes (max %d)", len(scopeKey), maxScopeKeyLen) + } + if strings.ContainsRune(scopeKey, 0) { + return fmt.Errorf("source cache scope key must not contain NUL bytes") + } + return nil +} + +// Entry is a previous sync's persisted validator for one scope. +type Entry struct { + // CacheValidator is the opaque upstream validator: a literal HTTP + // ETag, a delta token, etc. Never interpreted by the SDK. + CacheValidator string + + // DiscoveredAt is when the entry was written. + DiscoveredAt time.Time +} + +// Lookup resolves a scope's previous-sync validator. The SDK provides an +// implementation on SyncOpAttrs; connectors call it before revalidating +// upstream. +type Lookup interface { + // Lookup returns the previous sync's entry for + // (rowKind, scopeKey). found=false means no entry: fetch fresh. + // Implementations must treat internal read errors that leave fresh + // fetch available as misses rather than failing the connector call. + Lookup(ctx context.Context, rowKind RowKind, scopeKey string) (entry Entry, found bool, err error) +} + +// NoopLookup is the Lookup installed when source cache is disabled or +// degraded. Every lookup misses. +type NoopLookup struct{} + +var _ Lookup = NoopLookup{} + +func (NoopLookup) Lookup(context.Context, RowKind, string) (Entry, bool, error) { + return Entry{}, false, nil +} + +// SourceCacheSetter is implemented by connector clients/servers that can receive a +// source-cache lookup implementation from the sync runner. The SDK calls +// SetSourceCache(lookup) at the start of each sync and SetSourceCache(nil) +// when the sync ends so a late RPC can't read stale state. Calls must be +// balanced per sync — exactly one nil clear for each non-nil install, and +// no clear from a sync that never installed — because the shared +// subprocess slot counts installs to detect concurrent-sync contention +// (see GRPCServer.SetSourceCache). +type SourceCacheSetter interface { + SetSourceCache(ctx context.Context, lookup Lookup) +} + +// HashScope returns the lowercase-hex sha256 of a canonical scope string. +// Convenience for connectors; any stable identifier is acceptable as a +// scope key (only non-emptiness and a length cap are enforced). +func HashScope(canonicalScope string) string { + sum := sha256.Sum256([]byte(canonicalScope)) + return hex.EncodeToString(sum[:]) +} diff --git a/pkg/sync/enqueue_page_tokens_test.go b/pkg/sync/enqueue_page_tokens_test.go new file mode 100644 index 000000000..655d5325b --- /dev/null +++ b/pkg/sync/enqueue_page_tokens_test.go @@ -0,0 +1,580 @@ +package sync //nolint:revive,nolintlint // we can't change the package name for backwards compatibility + +// Per-resource EnqueuePageTokens: parallel warm revalidation for page-numbered +// APIs (the GitHub shape). On page one of a resource's grant collection the +// connector already knows every other page's URL and stored validator, so +// it answers page one and spawns pages 2..N as sibling actions; each +// spawned page independently hits or misses its source-cache lookup. +// +// Under test: +// - EnqueuePageTokens on a per-resource grants response fans out sibling +// actions carrying the resource identity (no warning, no drops); +// - spawned pages are ordinary pages: replay on hit, cold fetch on miss +// (page boundary shifted), and a cold spawned page may CHAIN to a +// fresh page via NextPageToken (probe past a full tail); +// - reader-surface equivalence against a cold control sync, at +// workers=0 (sequential) and workers=4 (parallel). + +import ( + "context" + "fmt" + "math/rand" + "path/filepath" + "slices" + "sync" + "testing" + + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + "github.com/conductorone/baton-sdk/pkg/annotations" + "github.com/conductorone/baton-sdk/pkg/dotc1z" + "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" + "github.com/conductorone/baton-sdk/pkg/logging" + "github.com/conductorone/baton-sdk/pkg/sourcecache" + gt "github.com/conductorone/baton-sdk/pkg/types/grant" + rs "github.com/conductorone/baton-sdk/pkg/types/resource" +) + +const spawnPageSize = 4 + +// pagedSpawnMockConnector serves each group's member grants through +// numbered pages of size spawnPageSize. Page 1 carries its own rows plus +// EnqueuePageTokens for every other page the previous sync recorded (warm) or +// that exist upstream (cold). Every page is its own source-cache scope +// (page-URL style). Membership mutations shift page contents, so mutated +// pages miss their lookup and refetch cold — including discovering fresh +// tail pages a warm spawn never knew about. +type pagedSpawnMockConnector struct { + *mockConnector + + mu sync.Mutex + lookup sourcecache.Lookup + + groupIDs []string + members map[string][]string // groupID -> userIDs, page k = members[k*4:(k+1)*4] + // prevMembers models the connector's knowledge of the STORED page + // chain (what the previous sync recorded): the spawn horizon on warm + // syncs, and the row counts behind the full-tail probe decision. The + // test snapshots it after each sync, mirroring what a real connector + // derives from its stored validators. + prevMembers map[string][]string + resByID map[string]*v2.Resource + + replayPages int + coldPages int + probePages int + + // failPageOnce injects a one-shot error when serving "gid/page" — + // simulating a crash while spawned siblings are pending in the + // checkpointed action stack. + failPageOnce map[string]bool + // servedPages counts servePage attempts per "gid/page" (including the + // failed attempt), so resume tests can see which pages re-executed. + servedPages map[string]int +} + +func newPagedSpawnMockConnector(t *testing.T, groupCount, membersPerGroup int) *pagedSpawnMockConnector { + t.Helper() + mc := &pagedSpawnMockConnector{ + mockConnector: newMockConnector(), + members: map[string][]string{}, + prevMembers: map[string][]string{}, + resByID: map[string]*v2.Resource{}, + failPageOnce: map[string]bool{}, + servedPages: map[string]int{}, + } + mc.rtDB = append(mc.rtDB, groupResourceType, userResourceType) + + for i := 0; i < 30; i++ { + uid := fmt.Sprintf("user-%02d", i) + u, err := rs.NewUserResource(uid, userResourceType, uid, nil, rs.WithAnnotation(&v2.SkipEntitlementsAndGrants{})) + require.NoError(t, err) + mc.AddResource(context.Background(), u) + mc.resByID[uid] = u + } + for i := 0; i < groupCount; i++ { + gid := fmt.Sprintf("group-%02d", i) + g, err := rs.NewGroupResource(gid, groupResourceType, gid, nil) + require.NoError(t, err) + mc.AddResource(context.Background(), g) + mc.resByID[gid] = g + mc.groupIDs = append(mc.groupIDs, gid) + mc.entDB[gid] = []*v2.Entitlement{mkMemberEnt(g)} + for m := 0; m < membersPerGroup; m++ { + mc.members[gid] = append(mc.members[gid], fmt.Sprintf("user-%02d", (i*3+m)%30)) + } + } + return mc +} + +func (mc *pagedSpawnMockConnector) SetSourceCache(_ context.Context, lookup sourcecache.Lookup) { + mc.mu.Lock() + defer mc.mu.Unlock() + if lookup == nil { + lookup = sourcecache.NoopLookup{} + } + mc.lookup = lookup +} + +func (mc *pagedSpawnMockConnector) Validate(context.Context, *v2.ConnectorServiceValidateRequest, ...grpc.CallOption) (*v2.ConnectorServiceValidateResponse, error) { + return v2.ConnectorServiceValidateResponse_builder{ + Annotations: annotations.New(v2.SourceCacheCapability_builder{ + Mode: v2.SourceCacheCapability_MODE_READ_WRITE, + }.Build()), + }.Build(), nil +} + +func pageScopeFor(gid string, page int) string { + return fmt.Sprintf("groups/%s/members?page=%d", gid, page) +} + +func (mc *pagedSpawnMockConnector) pageRows(gid string, page int) []string { + lo := page * spawnPageSize + hi := lo + spawnPageSize + all := mc.members[gid] + if lo >= len(all) { + return nil + } + if hi > len(all) { + hi = len(all) + } + return all[lo:hi] +} + +func (mc *pagedSpawnMockConnector) grantsFor(gid string, uids []string) []*v2.Grant { + out := make([]*v2.Grant, 0, len(uids)) + for _, uid := range uids { + out = append(out, gt.NewGrant(mc.resByID[gid], "member", mc.resByID[uid].GetId())) + } + return out +} + +// pageEtag is a content-hash-style validator: stable across syncs when the +// page's rows are unchanged, different otherwise. Growth always changes +// the stored tail page's content (rows shift into it), so the last stored +// page misses its lookup and chains — which is how fresh tail pages get +// discovered on warm syncs. +func pageEtag(gid string, page int, rows []string) string { + return fmt.Sprintf("etag:%s:%d:%v", gid, page, rows) +} + +// prevPageCount is the connector's stored-chain knowledge: how many pages +// the previous sync recorded for gid. The test snapshots members → +// prevMembers after each sync, mirroring what a real connector derives +// from its stored validators. +func (mc *pagedSpawnMockConnector) prevPageCount(gid string) int { + return (len(mc.prevMembers[gid]) + spawnPageSize - 1) / spawnPageSize +} + +func (mc *pagedSpawnMockConnector) snapshotPrev() { + mc.mu.Lock() + defer mc.mu.Unlock() + mc.prevMembers = map[string][]string{} + for gid, uids := range mc.members { + mc.prevMembers[gid] = append([]string(nil), uids...) + } +} + +// servePage answers one numbered page: lookup hit with matching validator → +// replay (no rows); miss/changed → cold rows + scope. A cold page chains to +// page+1 only when page+1 is NOT covered by a spawned sibling (i.e. beyond +// the stored chain) — that's the probe-past-tail path discovering pages the +// previous sync never had. +func (mc *pagedSpawnMockConnector) servePage(ctx context.Context, gid string, page int) (*v2.GrantsServiceListGrantsResponse, error) { + pageKey := fmt.Sprintf("%s/%d", gid, page) + mc.mu.Lock() + mc.servedPages[pageKey]++ + if mc.failPageOnce[pageKey] { + delete(mc.failPageOnce, pageKey) + mc.mu.Unlock() + return nil, fmt.Errorf("injected crash serving %s", pageKey) + } + lookup := mc.lookup + rows := mc.pageRows(gid, page) + etag := pageEtag(gid, page, rows) + prevPages := mc.prevPageCount(gid) + hasNextUpstream := len(mc.pageRows(gid, page+1)) > 0 + mc.mu.Unlock() + if lookup == nil { + lookup = sourcecache.NoopLookup{} + } + scope := pageScopeFor(gid, page) + + entry, found, err := lookup.Lookup(ctx, sourcecache.RowKindGrants, scope) + if err != nil { + return nil, err + } + if found && entry.CacheValidator == etag { + mc.mu.Lock() + mc.replayPages++ + mc.mu.Unlock() + return v2.GrantsServiceListGrantsResponse_builder{ + Annotations: annotations.New(v2.SourceCacheReplay_builder{ + ScopeKey: scope, + CacheValidator: etag, + }.Build()), + }.Build(), nil + } + + next := "" + if hasNextUpstream && page+1 >= prevPages { + // page+1 exists upstream but was never stored, so no sibling was + // spawned for it — this cold page's chain is its only route in. + next = fmt.Sprintf("page=%d", page+1) + } + mc.mu.Lock() + mc.coldPages++ + if page >= prevPages && prevPages > 0 { + mc.probePages++ + } + mc.mu.Unlock() + return v2.GrantsServiceListGrantsResponse_builder{ + List: mc.grantsFor(gid, rows), + Annotations: annotations.New(v2.SourceCacheRecord_builder{ + ScopeKey: scope, + CacheValidator: etag, + }.Build()), + NextPageToken: next, + }.Build(), nil +} + +func (mc *pagedSpawnMockConnector) ListGrants(ctx context.Context, in *v2.GrantsServiceListGrantsRequest, _ ...grpc.CallOption) (*v2.GrantsServiceListGrantsResponse, error) { + gid := in.GetResource().GetId().GetResource() + if in.GetResource().GetId().GetResourceType() != groupResourceType.GetId() { + return v2.GrantsServiceListGrantsResponse_builder{}.Build(), nil + } + + tok := in.GetPageToken() + if tok != "" { + var page int + if _, err := fmt.Sscanf(tok, "page=%d", &page); err != nil { + return nil, fmt.Errorf("bad page token %q", tok) + } + return mc.servePage(ctx, gid, page) + } + + // Page one: serve it, then spawn a sibling for every OTHER page of the + // STORED chain (the pages whose URLs and validators the previous sync + // recorded). Cold first syncs have no stored chain and just paginate + // serially; pages beyond the stored tail are discovered by the last + // stored page missing its lookup and chaining (see servePage). + resp, err := mc.servePage(ctx, gid, 0) + if err != nil { + return nil, err + } + mc.mu.Lock() + prevPages := mc.prevPageCount(gid) + mc.mu.Unlock() + + var tokens []string + for p := 1; p < prevPages; p++ { + tokens = append(tokens, fmt.Sprintf("page=%d", p)) + } + if len(tokens) > 0 { + annos := annotations.Annotations(resp.GetAnnotations()) + annos.Update(v2.EnqueuePageTokens_builder{PageTokens: tokens}.Build()) + resp.SetAnnotations(annos) + // Page one's own chain ends here; siblings carry the rest. + resp.SetNextPageToken("") + } + return resp, nil +} + +func mkMemberEnt(group *v2.Resource) *v2.Entitlement { + ent := v2.Entitlement_builder{ + Id: fmt.Sprintf("%s:%s:member", group.GetId().GetResourceType(), group.GetId().GetResource()), + Resource: group, + DisplayName: "member", + Slug: "member", + GrantableTo: []*v2.ResourceType{userResourceType}, + }.Build() + return ent +} + +// runSpawnSync runs one sync and returns the grant-coverage counter for +// the group type — the number the "more grant resources than resources" +// anomaly warning compares against the type's resource total. With spawn +// fan-out it must equal the group count exactly (one per resource), not +// one per spawned page. +func runSpawnSync(ctx context.Context, t *testing.T, mc *pagedSpawnMockConnector, path, prevPath, tmpDir string, workers int) int { + t.Helper() + store, err := dotc1z.NewStore(ctx, path, + dotc1z.WithEngine(c1zstore.EnginePebble), + dotc1z.WithTmpDir(tmpDir), + ) + require.NoError(t, err) + opts := []SyncOpt{WithConnectorStore(store), WithTmpDir(tmpDir)} + if prevPath != "" { + opts = append(opts, WithPreviousSyncC1ZPath(prevPath)) + } + if workers > 0 { + opts = append(opts, WithWorkerCount(workers)) + } + sc, err := NewSyncer(ctx, mc, opts...) + require.NoError(t, err) + require.NoError(t, sc.Sync(ctx)) + impl, ok := sc.(*syncer) + require.True(t, ok) + covered := impl.counts.GrantsProgress(groupResourceType.GetId()) + require.NoError(t, sc.Close(ctx)) + return covered +} + +func TestPerResourceEnqueuePageTokensWarmRevalidation(t *testing.T) { + for _, workers := range []int{0, 4} { + t.Run(fmt.Sprintf("workers=%d", workers), func(t *testing.T) { + ctx, err := logging.Init(context.Background()) + require.NoError(t, err) + tmpDir := t.TempDir() + + // 6 groups × 11 members = 3 pages each (4/4/3), 66 grants. + mc := newPagedSpawnMockConnector(t, 6, 11) + + // --- Sync 1: cold — no stored chain, serial pagination. ------- + sync1 := filepath.Join(tmpDir, "sync1.c1z") + covered := runSpawnSync(ctx, t, mc, sync1, "", tmpDir, workers) + require.Equal(t, 6, covered, "cold: each group counts once toward grant coverage") + grants1 := listGrantsInFile(ctx, t, sync1) + require.Len(t, grants1, 66) + require.Zero(t, mc.replayPages) + require.Equal(t, 18, mc.coldPages, "6 groups × 3 pages fetch cold") + mc.snapshotPrev() + + // --- Sync 2: warm, unchanged — page 1 spawns pages 2..3 per + // group and EVERY page replays. -------------------------------- + sync2 := filepath.Join(tmpDir, "sync2.c1z") + covered = runSpawnSync(ctx, t, mc, sync2, sync1, tmpDir, workers) + require.Equal(t, 6, covered, + "warm fan-out: a group with 3 spawned/replayed pages still counts ONCE — spawned siblings must not inflate coverage past the resource total") + require.Equal(t, 18, mc.replayPages, "every page of every group replays") + require.Equal(t, 18, mc.coldPages, "no cold fetches on the unchanged warm sync") + requireSameGrantIDs(t, grants1, listGrantsInFile(ctx, t, sync2), "warm vs cold") + mc.snapshotPrev() + + // --- Sync 3: warm with churn: + // group-00 gains 2 members → its stored tail page (page 2) + // changes content, misses its lookup, fetches cold, and CHAINS + // into a brand-new page 3 no sibling was spawned for; + // group-01 loses 3 members → its stored page 2 now returns + // zero rows (spawned probe of a vanished tail). -------------- + mc.mu.Lock() + mc.members["group-00"] = append(mc.members["group-00"], "user-20", "user-21") + mc.members["group-01"] = mc.members["group-01"][:8] // exactly 2 pages now + mc.mu.Unlock() + + sync3 := filepath.Join(tmpDir, "sync3.c1z") + runSpawnSync(ctx, t, mc, sync3, sync2, tmpDir, workers) + grants3 := listGrantsInFile(ctx, t, sync3) + require.Positive(t, mc.probePages, "the fresh tail page must be reached by a cold page chaining past the stored horizon") + + // Cold control against the same tenant state. + control := filepath.Join(tmpDir, "control.c1z") + runSpawnSync(ctx, t, mc, control, "", tmpDir, workers) + requireSameGrantIDs(t, listGrantsInFile(ctx, t, control), grants3, "churned warm vs cold control") + }) + } +} + +// TestPerResourceEnqueuePageTokensResumeWithPendingSiblings pins suspend/resume +// across a spawn fan-out. A warm sync crashes while serving one of a +// group's spawned sibling pages and is then resumed on the same c1z. +// Checkpoints are rate-limited (minCheckpointInterval), so resume +// granularity is at-least-once for everything since the last persisted +// checkpoint — possibly the whole grants phase in a fast sync. The +// invariants are therefore: +// +// - no DROPS: the crashed sibling and the sibling spawned alongside it +// both execute by the time the resumed sync finishes; +// - no DUPES on double-execution: pages that committed before the crash +// re-run after resume and must be idempotent — replay re-copies, cold +// re-upserts, and the final file is grant-for-grant identical to a +// cold control sync. +// +// (The complementary serialization half — a checkpoint can never hold +// "page finished but siblings missing", and the Spawned marker survives +// the round trip — is pinned by TestSpawnedActionsSurviveCheckpoint.) +func TestPerResourceEnqueuePageTokensResumeWithPendingSiblings(t *testing.T) { + for _, workers := range []int{0, 4} { + t.Run(fmt.Sprintf("workers=%d", workers), func(t *testing.T) { + ctx, err := logging.Init(context.Background()) + require.NoError(t, err) + tmpDir := t.TempDir() + + // 6 groups × 11 members = 3 pages each. + mc := newPagedSpawnMockConnector(t, 6, 11) + + sync1 := filepath.Join(tmpDir, "sync1.c1z") + runSpawnSync(ctx, t, mc, sync1, "", tmpDir, workers) + mc.snapshotPrev() + + // Churn group-01 so the warm sync mixes replayed and cold + // spawned pages across the crash boundary. + mc.mu.Lock() + mc.servedPages = map[string]int{} // count only the warm attempts + mc.members["group-01"] = mc.members["group-01"][:9] + // Crash while serving group-00's spawned page 1: page 0 (the + // spawner) has already finished, so pages 1 and 2 are pending + // Spawned actions in the checkpoint when the sync dies. + mc.failPageOnce["group-00/1"] = true + mc.mu.Unlock() + + opts := []SyncOpt{WithTmpDir(tmpDir), WithPreviousSyncC1ZPath(sync1)} + if workers > 0 { + opts = append(opts, WithWorkerCount(workers)) + } + + sync2 := filepath.Join(tmpDir, "sync2.c1z") + crashed, err := NewSyncer(ctx, mc, append(opts, WithC1ZPath(sync2))...) + require.NoError(t, err) + err = crashed.Sync(ctx) + require.ErrorContains(t, err, "injected crash", "the warm sync must die mid-fan-out") + // Close on a cancelled context so the in-flight sync stays + // resumable instead of being finalized (the established + // interrupt pattern from the child-resources resume test). + cancelledCtx, cancel := context.WithCancel(ctx) + cancel() + require.NoError(t, crashed.Close(cancelledCtx)) + + resumed, err := NewSyncer(ctx, mc, append(opts, WithC1ZPath(sync2))...) + require.NoError(t, err) + require.NoError(t, resumed.Sync(ctx)) + require.NoError(t, resumed.Close(ctx)) + + mc.mu.Lock() + require.GreaterOrEqual(t, mc.servedPages["group-00/1"], 2, + "the crashed sibling must be re-executed on resume") + require.GreaterOrEqual(t, mc.servedPages["group-00/2"], 1, + "the sibling spawned alongside the crashed one must not be dropped") + mc.mu.Unlock() + + // No drops, no duplicates: cold control equivalence. Note the + // resumed file also re-executed pages that committed before the + // crash — this passing IS the double-execution idempotency + // assertion. + control := filepath.Join(tmpDir, "control.c1z") + runSpawnSync(ctx, t, mc, control, "", tmpDir, workers) + requireSameGrantIDs(t, listGrantsInFile(ctx, t, control), listGrantsInFile(ctx, t, sync2), + "resumed warm sync vs cold control") + }) + } +} + +// TestSpawnedActionsSurviveCheckpoint pins the state-level half of +// spawn-fan-out resume: the exact sequence syncGrantsForResource performs +// (push siblings, then finish the spawner — one state mutation window, so +// no checkpoint can see "finished but siblings missing") marshals into a +// sync token from which a fresh state pops every sibling with its Spawned +// marker and resource identity intact. If the marker were dropped in the +// round trip, a resumed sync would double-count grant coverage for every +// pending sibling. +func TestSpawnedActionsSurviveCheckpoint(t *testing.T) { + ctx := context.Background() + st := newState() + require.NoError(t, st.Unmarshal("")) + + // Drain the implicit init action and stand up a grants action for one + // resource, as the syncer would. + st.FinishAction(ctx, st.Current()) + st.PushAction(ctx, Action{Op: SyncGrantsOp, ResourceTypeID: "group", ResourceID: "group-00"}) + spawner := st.Current() + + // nextPageOrFinishAction order: push spawned siblings, then finish the + // spawning action. + for _, tok := range []string{"page=1", "page=2"} { + st.PushAction(ctx, Action{ + Op: SyncGrantsOp, + ResourceTypeID: "group", + ResourceID: "group-00", + PageToken: tok, + Spawned: true, + }) + } + st.FinishAction(ctx, spawner) + + token, err := st.Marshal() + require.NoError(t, err) + + resumed := newState() + require.NoError(t, resumed.Unmarshal(token)) + + var siblings []Action + for resumed.Current() != nil { + a := *resumed.Current() + siblings = append(siblings, a) + resumed.FinishAction(ctx, &a) + } + require.Len(t, siblings, 2, "both pending siblings must survive the checkpoint round trip") + tokens := map[string]bool{} + for _, a := range siblings { + require.True(t, a.Spawned, "the Spawned marker must survive serialization (progress accounting depends on it)") + require.Equal(t, "group-00", a.ResourceID, "the resource identity rides the action across resume") + require.Equal(t, SyncGrantsOp, a.Op) + tokens[a.PageToken] = true + } + require.Equal(t, map[string]bool{"page=1": true, "page=2": true}, tokens) +} + +// TestPerResourceEnqueuePageTokensChurnSoak is the randomized soak (the shape +// that caught real bugs in both the Entra and GitHub POCs): seeded rounds +// of membership churn against the paged-spawn connector, each round +// running a warm chained sync at workers=4 and asserting reader-surface +// equivalence against a cold control sync of the same tenant state. Churn +// shifts page boundaries arbitrarily, so every round mixes replayed pages, +// cold refetches of shifted pages, vanished tails (zero-row probes), and +// fresh tails discovered by chaining — concurrently. +func TestPerResourceEnqueuePageTokensChurnSoak(t *testing.T) { + if testing.Short() { + t.Skip("soak test") + } + for _, seed := range []int64{1, 7} { + t.Run(fmt.Sprintf("seed=%d", seed), func(t *testing.T) { + ctx, err := logging.Init(context.Background()) + require.NoError(t, err) + tmpDir := t.TempDir() + rng := rand.New(rand.NewSource(seed)) //nolint:gosec // deterministic test seed + + mc := newPagedSpawnMockConnector(t, 8, 9) + + prev := filepath.Join(tmpDir, "sync0.c1z") + runSpawnSync(ctx, t, mc, prev, "", tmpDir, 4) + mc.snapshotPrev() + + for round := 1; round <= 6; round++ { + // Churn: random adds/removes across random groups. Removes + // from the middle shift every following page of the group; + // adds can grow a fresh tail page. + mc.mu.Lock() + for i := 0; i < 1+rng.Intn(4); i++ { + gid := mc.groupIDs[rng.Intn(len(mc.groupIDs))] + cur := mc.members[gid] + if rng.Intn(2) == 0 && len(cur) > 1 { + k := rng.Intn(len(cur)) + mc.members[gid] = append(append([]string(nil), cur[:k]...), cur[k+1:]...) + } else { + uid := fmt.Sprintf("user-%02d", rng.Intn(30)) + if !slices.Contains(cur, uid) { + mc.members[gid] = append(cur, uid) + } + } + } + mc.mu.Unlock() + + warm := filepath.Join(tmpDir, fmt.Sprintf("warm-%d.c1z", round)) + runSpawnSync(ctx, t, mc, warm, prev, tmpDir, 4) + warmGrants := listGrantsInFile(ctx, t, warm) + + control := filepath.Join(tmpDir, fmt.Sprintf("control-%d.c1z", round)) + runSpawnSync(ctx, t, mc, control, "", tmpDir, 4) + requireSameGrantIDs(t, listGrantsInFile(ctx, t, control), warmGrants, + fmt.Sprintf("round %d: warm chain vs cold control", round)) + + mc.snapshotPrev() + prev = warm + } + require.Positive(t, mc.replayPages, "soak must actually exercise replay") + require.Positive(t, mc.coldPages, "soak must actually exercise cold refetch") + }) + } +} diff --git a/pkg/sync/expand/drop_stats.go b/pkg/sync/expand/drop_stats.go new file mode 100644 index 000000000..6261ebdca --- /dev/null +++ b/pkg/sync/expand/drop_stats.go @@ -0,0 +1,102 @@ +package expand + +import ( + "context" + "sync" + + "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" + "go.uber.org/zap" +) + +// DroppedEdgeStats aggregates expansion edges dropped because a +// referenced entitlement has no row — connector magic-id bugs and +// disabled-by-default resource types. Production measured ~2.9M +// per-edge warnings a week from these sites, dominated by a few +// thousand distinct ids, so per-edge logging is demoted to Debug and +// ONE aggregated warning per sync (emitted by the syncer when +// expansion completes) carries the totals and a capped sample of +// distinct ids — the same reporting shape as the ingestion invariants' +// dangling-reference warnings. +// +// One instance lives on the syncer for the whole sync (the Expander is +// reconstructed every step); guarded for parallel use. In-memory only: +// a resume after a crash undercounts, which is acceptable for +// observability. +type DroppedEdgeStats struct { + mu sync.Mutex + sourceMissing int64 + destinationMissing int64 + seen map[string]struct{} + examples []string +} + +const maxDroppedEdgeExamples = 25 + +func (s *DroppedEdgeStats) record(entitlementID string, dest bool, edges int64) { + if s == nil || edges <= 0 { + return + } + s.mu.Lock() + defer s.mu.Unlock() + if dest { + s.destinationMissing += edges + } else { + s.sourceMissing += edges + } + if s.seen == nil { + s.seen = map[string]struct{}{} + } + if _, ok := s.seen[entitlementID]; ok { + return + } + s.seen[entitlementID] = struct{}{} + if len(s.examples) < maxDroppedEdgeExamples { + s.examples = append(s.examples, entitlementID) + } +} + +// RecordSourceMissing counts ONE edge dropped because its SOURCE +// entitlement has no row. +func (s *DroppedEdgeStats) RecordSourceMissing(entitlementID string) { + s.record(entitlementID, false, 1) +} + +// RecordSourceMissingEdges counts a BATCH of edges sharing one missing +// source entitlement (the legacy expander drops a whole action's edges +// when the shared source is gone — the total must count every edge, not +// the batch). +func (s *DroppedEdgeStats) RecordSourceMissingEdges(entitlementID string, edges int) { + s.record(entitlementID, false, int64(edges)) +} + +// RecordDestinationMissing counts ONE edge dropped because its +// DESTINATION entitlement has no row. +func (s *DroppedEdgeStats) RecordDestinationMissing(entitlementID string) { + s.record(entitlementID, true, 1) +} + +// RecordDestinationMissingEdges counts every incoming edge skipped when +// one destination entitlement has no row (the topological path skips the +// destination's whole reduction — all of its incoming edges — at once). +func (s *DroppedEdgeStats) RecordDestinationMissingEdges(entitlementID string, edges int) { + s.record(entitlementID, true, int64(edges)) +} + +// LogSummary emits the one aggregated warning for the sync, if any +// edges were dropped. Safe on nil. +func (s *DroppedEdgeStats) LogSummary(ctx context.Context) { + if s == nil { + return + } + s.mu.Lock() + defer s.mu.Unlock() + if s.sourceMissing == 0 && s.destinationMissing == 0 { + return + } + ctxzap.Extract(ctx).Warn("grant expansion: DROPPED edges referencing entitlements with no entitlement row (connector bug or resource type not enabled — check magic-id construction)", + zap.Int64("edges_missing_source_entitlement", s.sourceMissing), + zap.Int64("edges_missing_destination_entitlement", s.destinationMissing), + zap.Int("distinct_missing_entitlements", len(s.seen)), + zap.Strings("entitlement_id_examples", s.examples), + ) +} diff --git a/pkg/sync/expand/drop_stats_test.go b/pkg/sync/expand/drop_stats_test.go new file mode 100644 index 000000000..9cee3fa8c --- /dev/null +++ b/pkg/sync/expand/drop_stats_test.go @@ -0,0 +1,81 @@ +package expand + +// DroppedEdgeStats counting semantics: totals count EDGES, not drop +// events. The legacy expander drops a whole action's edge batch when the +// shared SOURCE entitlement is missing, and the topological path skips a +// destination's entire reduction (every incoming edge) when the +// DESTINATION is missing — both previously incremented the aggregate +// once per event, undercounting the very numbers the per-sync warning +// exists to report. + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" +) + +func TestDroppedEdgeStatsCountsEdgesNotEvents(t *testing.T) { + s := &DroppedEdgeStats{} + + // A missing source shared by a 5-destination batch drops 5 edges. + s.RecordSourceMissingEdges("ent-src", 5) + // A missing destination with 3 incoming edges skips 3 edges. + s.RecordDestinationMissingEdges("ent-dst", 3) + // Single-edge sites still count one. + s.RecordSourceMissing("ent-src-2") + s.RecordDestinationMissing("ent-dst-2") + // Zero/negative counts are no-ops (defensive: empty batches). + s.RecordSourceMissingEdges("ent-none", 0) + s.RecordDestinationMissingEdges("ent-none", -1) + + s.mu.Lock() + defer s.mu.Unlock() + require.Equal(t, int64(6), s.sourceMissing, "source total must count every edge in a dropped batch") + require.Equal(t, int64(4), s.destinationMissing, "destination total must count every skipped incoming edge") + require.Len(t, s.seen, 4, "distinct-id set counts entitlements, not edges; no-op records add nothing") + require.NotContains(t, s.seen, "ent-none") +} + +func TestDroppedEdgeStatsExamplesDedupeAcrossRepeats(t *testing.T) { + s := &DroppedEdgeStats{} + for i := 0; i < 3; i++ { + s.RecordSourceMissingEdges("ent-repeat", 2) + } + s.mu.Lock() + defer s.mu.Unlock() + require.Equal(t, int64(6), s.sourceMissing, "repeat drops of one id still count their edges") + require.Equal(t, []string{"ent-repeat"}, s.examples, "examples list one entry per distinct id") +} + +// TestTopologicalSourceMissingRecordedOnDropStats pins the topological +// reduce's SOURCE-missing path onto the aggregate: a graph edge whose +// source entitlement has no row must complete (drop-don't-fail), be +// counted on DroppedEdgeStats, and not fail the run — the migration +// that moved the destination-side to the aggregate had left this path +// as a per-edge Warn invisible to LogSummary. +func TestTopologicalSourceMissingRecordedOnDropStats(t *testing.T) { + ctx := context.Background() + store := NewMockExpanderStore() + + group := makeResource("group", "org") + source := makeEntitlement("ent:source", group) + dest := makeEntitlement("ent:dest", group) + // The DESTINATION row exists; the SOURCE row deliberately does not. + store.AddEntitlement(dest) + + graph := NewEntitlementGraph(ctx) + graph.AddEntitlementID(source.GetId()) + graph.AddEntitlementID(dest.GetId()) + require.NoError(t, graph.AddEdge(ctx, source.GetId(), dest.GetId(), false, nil)) + + e := NewExpander(store, graph) + stats := &DroppedEdgeStats{} + e.SetDropStats(stats) + require.NoError(t, e.RunTopologicalMergeStreaming(ctx), "a missing source must complete, not error") + + stats.mu.Lock() + defer stats.mu.Unlock() + require.Equal(t, int64(1), stats.sourceMissing, "the dropped source edge must land on the aggregate") + require.Contains(t, stats.seen, source.GetId()) +} diff --git a/pkg/sync/expand/expander.go b/pkg/sync/expand/expander.go index c54e25b41..034a1d142 100644 --- a/pkg/sync/expand/expander.go +++ b/pkg/sync/expand/expander.go @@ -137,6 +137,15 @@ type entitlementGrantPrincipalKeyLister interface { type Expander struct { store ExpanderStore graph *EntitlementGraph + // dropStats, when set, aggregates dropped-edge reporting for the + // whole sync (the syncer owns it across step-wise Expander + // reconstruction). Nil is safe: recording no-ops. + dropStats *DroppedEdgeStats +} + +// SetDropStats installs the sync-scoped dropped-edge aggregator. +func (e *Expander) SetDropStats(s *DroppedEdgeStats) { + e.dropStats = s } // NewExpander creates a new Expander with the given store and graph. @@ -419,7 +428,12 @@ func (e *Expander) runAction(ctx context.Context, action *EntitlementGraphAction // drop them all and complete the action. Mirrors the inline handling // of missing destinations below. (MarkEdgeExpanded no-ops on deleted // edges, so returning "" here completes the action cleanly.) - l.Warn("runAction: source entitlement not found, dropping batch edges", + // Per-edge logging is Debug: production measured millions of + // these a week; the aggregate warning (DroppedEdgeStats) is + // the per-sync report. The batch shares one source, but every + // destination edge drops — count them all. + e.dropStats.RecordSourceMissingEdges(action.SourceEntitlementID, len(dests)) + l.Debug("runAction: source entitlement not found, dropping batch edges", zap.String("source_entitlement_id", action.SourceEntitlementID)) for _, d := range dests { _ = e.graph.DeleteEdge(ctx, action.SourceEntitlementID, d.EntitlementID) @@ -453,8 +467,9 @@ func (e *Expander) runAction(ctx context.Context, action *EntitlementGraphAction if err != nil { if status.Code(err) == codes.NotFound { // A single missing descendant drops only its own edge; the rest - // of the batch still expands. - l.Warn("runAction: descendant entitlement not found, dropping edge", + // of the batch still expands. Debug + aggregate, as above. + e.dropStats.RecordDestinationMissing(d.EntitlementID) + l.Debug("runAction: descendant entitlement not found, dropping edge", zap.String("descendant_entitlement_id", d.EntitlementID)) _ = e.graph.DeleteEdge(ctx, action.SourceEntitlementID, d.EntitlementID) continue diff --git a/pkg/sync/expand/topological_merge.go b/pkg/sync/expand/topological_merge.go index 594df7c2f..c3b358078 100644 --- a/pkg/sync/expand/topological_merge.go +++ b/pkg/sync/expand/topological_merge.go @@ -399,9 +399,14 @@ func (e *Expander) driveTopologicalLayer( destEntitlement := entitlements[destID] if destEntitlement == nil { // Same drop-don't-fail policy as the legacy expander's - // missing-descendant handling, but say so: silently zeroing - // a subtree of expansion is undiagnosable in production. - ctxzap.Extract(ctx).Warn("topological expansion: destination entitlement not in store; skipping its reduction", + // missing-descendant handling. Recorded on the sync-wide + // aggregate (one warning per sync with distinct-id + // examples); per-edge logging stays at Debug so a large + // dangling family can't flood the logs. Skipping this + // destination skips its ENTIRE reduction — every incoming + // edge — so the total counts each one. + e.dropStats.RecordDestinationMissingEdges(destID, len(incoming)) + ctxzap.Extract(ctx).Debug("topological expansion: destination entitlement not in store; skipping its reduction", zap.String("entitlement_id", destID)) continue } diff --git a/pkg/sync/expand/topological_merge_projection.go b/pkg/sync/expand/topological_merge_projection.go index d2081d935..a8cf4b05a 100644 --- a/pkg/sync/expand/topological_merge_projection.go +++ b/pkg/sync/expand/topological_merge_projection.go @@ -162,9 +162,13 @@ func (e *Expander) mergeDestinationStreams( for _, sourceID := range sortedCopy(sourceNode.EntitlementIDs) { sourceEntitlement := entitlements[sourceID] if sourceEntitlement == nil { - // Drop-don't-fail, but loudly (see the destination-side - // warning in driveTopologicalLayer). - ctxzap.Extract(ctx).Warn("topological expansion: source entitlement not in store; its contributions are skipped", + // Drop-don't-fail, recorded on the sync-wide aggregate + // (one warning per sync — see DroppedEdgeStats and the + // destination-side twin in driveTopologicalLayer); + // per-edge logging stays at Debug so a large dangling + // family can't flood the logs. + e.dropStats.RecordSourceMissing(sourceID) + ctxzap.Extract(ctx).Debug("topological expansion: source entitlement not in store; its contributions are skipped", zap.String("entitlement_id", sourceID)) continue } diff --git a/pkg/sync/gen_skew_inputs_test.go b/pkg/sync/gen_skew_inputs_test.go new file mode 100644 index 000000000..85fb0f52f --- /dev/null +++ b/pkg/sync/gen_skew_inputs_test.go @@ -0,0 +1,53 @@ +package sync //nolint:revive,nolintlint // we can't change the package name for backwards compatibility + +// Generator for the version-skew fixture's INPUTS (see +// testdata/README-old-fold.md): two new-format, source-cache-stamped +// sync files that an old-binary compactor is then run over. Env-gated +// and skipped in normal runs; retained so the fixture is regenerable: +// +// BATON_GEN_SKEW_INPUTS=/tmp/skew-gen go test -run TestGenerateSkewInputs ./pkg/sync/ + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + "github.com/conductorone/baton-sdk/pkg/logging" + gt "github.com/conductorone/baton-sdk/pkg/types/grant" +) + +func TestGenerateSkewInputs(t *testing.T) { + outDir := os.Getenv("BATON_GEN_SKEW_INPUTS") + if outDir == "" { + t.Skip("set BATON_GEN_SKEW_INPUTS to an output dir") + } + ctx, err := logging.Init(t.Context()) + require.NoError(t, err) + require.NoError(t, os.MkdirAll(outDir, 0o755)) //nolint:gosec // operator-supplied output dir in an env-gated generator + tmpDir := t.TempDir() + + group, ent, alice, bob := sourceCacheTestFixtures(t) + mc := newSourceCacheMockConnector() + mc.AddResource(ctx, group) + mc.AddResource(ctx, alice) + mc.AddResource(ctx, bob) + mc.entDB[group.GetId().GetResource()] = []*v2.Entitlement{ent} + mc.grantDB[group.GetId().GetResource()] = []*v2.Grant{gt.NewGrant(group, "member", alice)} + mc.etagByResource[group.GetId().GetResource()] = `W/"skew-v1"` + + base := filepath.Join(outDir, "base.c1z") + runSourceCacheSync(ctx, t, mc, base, "", tmpDir) + + // Second sync with a changed grant set (new-format, manifest-stamped). + mc.grantDB[group.GetId().GetResource()] = []*v2.Grant{ + gt.NewGrant(group, "member", alice), + gt.NewGrant(group, "member", bob), + } + mc.etagByResource[group.GetId().GetResource()] = `W/"skew-v2"` + newer := filepath.Join(outDir, "newer.c1z") + runSourceCacheSync(ctx, t, mc, newer, "", tmpDir) + t.Logf("wrote %s and %s", base, newer) +} diff --git a/pkg/sync/ingest_invariants.go b/pkg/sync/ingest_invariants.go new file mode 100644 index 000000000..fa428ac96 --- /dev/null +++ b/pkg/sync/ingest_invariants.go @@ -0,0 +1,1029 @@ +package sync //nolint:revive,nolintlint // we can't change the package name for backwards compatibility + +// Ingestion invariants: store-derived definitions of the syncer's +// side effects, evaluated at consuming seams. Design: +// docs/tasks/source-cache-ingestion-invariants.md. +// +// Background. The syncer historically attached side effects (flags, +// scheduled actions, derived rows, validation) to the connector-response +// ingest loop. That was correct while response pages were the only way +// rows entered the store; source-cache replay added an engine-side +// ingestion path and turned every stream-coupled side effect into a +// latent bug. Rather than mirroring each side effect once per ingestion +// path (unmarked coupling, rediscovered by review), each side effect is +// defined as a function of the store and evaluated here: +// +// - I1 expansion arming: PendingExpansion probe at SyncGrantExpansionOp +// (lives in parallel_syncer.go — the pattern's prototype). +// - I2 external-match arming: engine existence-bit probe at +// SyncExternalResourcesOp start; REPAIRS the flag. Pebble maintains +// the bit for fresh and replayed rows alike; the response-loop arm +// remains as the fast path and as SQLite's only mechanism. +// - I3 grant→resource referential integrity: post-collection prefix +// skip-scan over the grant keyspace. Dangling ref + an +// InsertResourceGrants-annotated grant = hard violation (the +// related-resource machinery lost a row); dangling without the +// annotation = warning (tolerated connector behavior). Pebble-only +// (the scan rides the pebble key order); full syncs only. +// - I4 child-scheduling completeness: every stored resource carrying +// ChildResourceType must have had its (childType, parent) action +// scheduled. Check-only; skipped when the resources phase did not +// run in this process (a zero-row child listing leaves no store +// evidence that its action ran). Full syncs only. FAIL-FAST ONLY: +// the check costs a full post-collection resource scan and its +// default outcome was only ever a warning, so production (default) +// skips it entirely — tests and the equivalence harness enforce it. +// - I5 exclusion-group validation: the stored entitlement keyspace is +// validated post-collection (one default per group, one resource +// type per group, size cap). This REPLACES the former streaming +// validation entirely — it is engine-agnostic and covers replayed, +// fresh, and generated (static) entitlements uniformly. +// - I6 source-cache scope consistency: every scope present in a +// by_source_scope index has a manifest entry at seal. Orphaned +// stamps mean a lost manifest write or a stamp leak — either would +// poison a FUTURE sync's replay while this sync reads clean. +// Pebble-only (the only engine with source-cache state); the +// replay-time counterpart (copied-row count vs the scope index) +// lives in the engine's ReplaySourceCache* and always hard-fails. +// - I7 entitlement→resource referential integrity: post-collection +// prefix skip-scan over the entitlement keyspace (one seek per +// distinct resource). Per-resource entitlement scopes cannot dangle +// by construction (the scope is only visited when the resource was +// synced); the exposure is TYPE-scoped listings and magic-id +// construction bugs in connectors. +// - I8 grant→entitlement referential integrity: post-collection +// skip-scan over the grant keyspace at entitlement-identity +// granularity plus one point probe per distinct entitlement. +// Exposures: independent scope freshness under replay (grants scope +// 304s while the entitlement listing refreshed away a row) and, +// dominantly, connector magic-id bugs — annotations/grants naming +// entitlement ids the entitlement enumeration never produced +// (production expansion drops millions of edges per week to this +// class). EXEMPT: grants under InsertResourceGrants pages (the +// machinery inserts the resource; no listing returns an entitlement +// for it — an established shape). +// - I9 grant→principal referential integrity: dangling-principal scan +// over the by_principal index (one seek + probe per distinct +// principal; a pending deferred index build is forced first, which +// just moves EndSync's own pass earlier). EXEMPT: grants carrying +// ExternalResourceMatch* annotations — unprocessed match carriers +// are evidence that no external resource file was configured, not +// bad data (the match op deletes carriers when it runs). +// +// I7/I8/I9 verdicts split on whether the referent's resource TYPE was +// synced (docs/tasks/dangling-reference-drops.md): +// +// - DISABLED-TYPE danglings (type never synced — a config gap) are +// DROPPED with one aggregated warning per invariant (total dropped, +// distinct dangling refs, capped id examples — never per-row logs; +// the expansion path's per-edge warnings produce millions of lines +// a week in production). The platform's ingestion provably discards +// such rows anyway. The DROP arm is deliberately NOT +// ErrReplayIntegrity: a dishonest validator makes the warm run fail +// but the cold run CLEAN, so the cold-retry ladder would never +// re-attribute to the connector and every sync would pay +// warm-fail-plus-retry forever. +// - ENABLED-TYPE danglings (type IS synced) FAIL the sync — never +// sealed away. A WARM sync's failure wraps ErrReplayIntegrity so +// the runners discard the output and retry cold; a COLD sync with +// the same evidence fails plainly, attributed to the connector +// (see unexpectedDanglingError). The compaction expand pass +// (onlyExpandGrants) is exempt from the FAIL arm: keep-newer merges +// manufacture dangling refs by design, and its drops are attributed +// separately in the warning. +// +// FAIL-FAST mode (tests, the harness) hard-fails every dangling so +// engineered violations are named. +// +// Evidence is monotone (existence bits, set-union scheduling records), +// so parallel workers and mixed stream/replay arrival orders yield +// identical verdicts; checks run at phase-quiesced points, so they are +// deterministic and idempotent under crash/resume. + +import ( + "context" + "errors" + "fmt" + "sort" + stdsync "sync" + + "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" + "go.uber.org/zap" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + reader_v2 "github.com/conductorone/baton-sdk/pb/c1/reader/v2" + "github.com/conductorone/baton-sdk/pkg/annotations" + "github.com/conductorone/baton-sdk/pkg/connectorstore" + "github.com/conductorone/baton-sdk/pkg/dotc1z" +) + +// sideEffectAnnotationCoverage is the enumerated coupling between +// connector annotations that imply syncer side effects and the mechanism +// that guarantees them independent of ingestion path. This map IS the +// audit that review rounds used to re-derive from memory; the +// completeness meta-test (ingest_invariants_test.go) fails when a listed +// annotation loses its mapping, and NEW side-effect-implying annotations +// must be added here with an invariant (or a documented exclusion) +// before they ship. +var sideEffectAnnotationCoverage = map[string]string{ + "c1.connector.v2.GrantExpandable": "I1: needs_expansion index probe at SyncGrantExpansionOp (parallel_syncer.go)", + "c1.connector.v2.ExternalResourceMatch": "I2: engine existence-bit repair at SyncExternalResourcesOp", + "c1.connector.v2.ExternalResourceMatchAll": "I2: engine existence-bit repair at SyncExternalResourcesOp", + "c1.connector.v2.ExternalResourceMatchID": "I2: engine existence-bit repair at SyncExternalResourcesOp", + "c1.connector.v2.InsertResourceGrants": "I3: grant→resource referential check post-collection; replay carries the row copy", + "c1.connector.v2.ChildResourceType": "I4: scheduled-set completeness check post-collection; replay carries the scheduling", + "c1.connector.v2.EntitlementExclusionGroup": "I5: stored-keyspace validation post-collection", + "c1.connector.v2.TypeScopedEntitlements": "I7: entitlement→resource referential check post-collection (type-granularity scopes can carry rows for vanished resources)", + "c1.connector.v2.TypeScopedGrants": "I8: grant→entitlement referential check post-collection (independently-fresh scopes can strand grant references)", +} + +// childScheduleSet is the monotone record of scheduled child-resource +// actions, keyed (childTypeID, parentTypeID, parentID). Guarded for +// parallel workers; in-memory only (see the I4 resume caveat). +type childScheduleSet struct { + mu stdsync.Mutex + m map[string]struct{} +} + +func childScheduleKey(childTypeID, parentTypeID, parentID string) string { + return childTypeID + "\x00" + parentTypeID + "\x00" + parentID +} + +// recordIfNew records the pair and reports whether it was NEW this sync. +// The set doubles as the scheduling dedupe: a parent can be discovered by +// several ingestion paths in one sync (replay copies its stored row AND a +// delta overlay re-emits it changed), and its children only need one +// SyncResourcesOp per pair — the second discovery would double the +// child-listing work. Atomic check-and-set so parallel workers can't both +// see "new". +func (c *childScheduleSet) recordIfNew(childTypeID, parentTypeID, parentID string) bool { + c.mu.Lock() + defer c.mu.Unlock() + if c.m == nil { + c.m = make(map[string]struct{}) + } + key := childScheduleKey(childTypeID, parentTypeID, parentID) + if _, ok := c.m[key]; ok { + return false + } + c.m[key] = struct{}{} + return true +} + +func (c *childScheduleSet) has(childTypeID, parentTypeID, parentID string) bool { + c.mu.Lock() + defer c.mu.Unlock() + _, ok := c.m[childScheduleKey(childTypeID, parentTypeID, parentID)] + return ok +} + +// repairExternalMatchFlag is invariant I2's repair: at the consuming seam +// (SyncExternalResourcesOp start), reconcile the in-memory flag with the +// engine's existence bit. Sound because the gated computation is a pure +// function of the store: running match processing when the store holds +// match-annotated grants is required; the flag is only an optimization +// that lets other syncs skip the scan. +// +// A fact-read error FAILS the sync: proceeding with an unset flag would +// silently skip match processing for replay-armed grants (the exact bug +// class I2 exists to prevent), and an engine-meta read error at this +// point means the store is unhealthy anyway. +func (s *syncer) repairExternalMatchFlag(ctx context.Context) error { + if s.state.HasExternalResourcesGrants() { + return nil + } + facts, ok := s.store.(dotc1z.IngestInvariantStore) + if !ok { + return nil // SQLite: the response-loop arm is the mechanism. + } + has, err := facts.HasExternalMatchGrants(ctx) + if err != nil { + // Deliberately NOT ErrReplayIntegrity: this reads engine-meta on + // the CURRENT store — a failure here is store health, not replay + // state, and a cold retry against a sick store just fails + // elsewhere. Fail plainly so the real problem is investigated. + return fmt.Errorf("ingest invariant I2: reading external-match fact: %w", err) + } + if has { + ctxzap.Extract(ctx).Debug("ingest invariant I2: arming external-match processing from store-derived fact") + s.state.SetHasExternalResourcesGrants() + } + return nil +} + +// ErrReplayIntegrity classifies sync failures where source-cache REPLAY +// state is implicated: a replay copy that lost or dropped rows, or +// store-derived evidence that a replayed artifact is inconsistent. Since +// replay is a pure optimization, every such failure has a safe +// remediation the runner can apply automatically — discard the previous +// sync source and re-run the sync cold (see the runners' cold-retry). +// +// The retry doubles as the attribution experiment: a clean cold run +// means replay was at fault (and the sync self-healed); a cold run that +// STILL violates the invariant is re-attributed to connector data — the +// cold-path failure deliberately does NOT carry this sentinel, so a +// recurring connector bug fails plainly (and cheaply) every sync instead +// of looping through wasted warm-attempt-plus-retry cycles under an SDK +// label. Also deliberately NOT applied to connector-contract violations +// (e.g. a replay requested for a scope that was never looked up): a cold +// retry would mask those without fixing the connector. +var ErrReplayIntegrity = errors.New("source-cache replay integrity failure") + +// relatedResourceRepairer is the optional store capability behind I3's +// repair arm (pebble implements it). +type relatedResourceRepairer interface { + RepairRelatedResource(ctx context.Context, prev connectorstore.Reader, resourceTypeID, resourceID string) (bool, error) +} + +// checkGrantResourceReferences is invariant I3: every distinct +// entitlement resource referenced by a grant must exist as a resource +// row. One seek per distinct resource (the grant keyspace leads with the +// entitlement resource), value reads only for dangling refs. +// +// The annotated arm (dangling ref + InsertResourceGrants) is +// REPAIR-flavored in default mode: those rows are the syncer's own +// establishment guarantee (no ListResources call ever returns them), and +// the previous sync's row — still open on s.sourceCache.prevReader — is +// a full-fidelity copy source, exactly the copy replay itself performs. +// Fail-fast mode (tests, the harness) skips repair and hard-fails so the +// underlying machinery bug is named rather than papered over. An +// unrepairable annotated ref fails the sync under ErrReplayIntegrity. +func (s *syncer) checkGrantResourceReferences(ctx context.Context) error { + if s.syncType != connectorstore.SyncTypeFull { + return nil + } + facts, ok := s.store.(dotc1z.IngestInvariantStore) + if !ok { + return nil // engine without the inspection surface + } + l := ctxzap.Extract(ctx) + return facts.ForEachDistinctGrantEntitlementResource(ctx, func(rt, rid string) error { + exists, err := facts.HasResourceRecord(ctx, rt, rid) + if err != nil { + return fmt.Errorf("ingest invariant I3: probing resource %s/%s: %w", rt, rid, err) + } + if exists { + return nil + } + annotated, err := facts.GrantsForEntResourceCarryInsertFact(ctx, rt, rid) + if err != nil { + return fmt.Errorf("ingest invariant I3: probing grant annotations for %s/%s: %w", rt, rid, err) + } + if annotated { + if s.sourceCache.prev == nil { + // COLD sync: replay cannot be the culprit — the cold + // path materializes these rows unconditionally in the + // same page handling, so this state means the connector + // destroyed or never supplied the row (e.g. a tombstone + // naming a grant-inserted resource). Deliberately NOT + // ErrReplayIntegrity: a cold retry can't fix connector + // data, and mislabeling would send operators hunting an + // SDK replay bug. This is also the terminal verdict of + // the runners' cold retry — a warm failure retried cold + // that still fails lands here, correctly re-attributed. + return fmt.Errorf( + "ingest invariant I3 violated on a COLD sync: grants reference resource %s/%s via InsertResourceGrants "+ + "but no resource row exists — the connector's own data is inconsistent "+ + "(check for tombstones deleting grant-inserted resources)", + rt, rid) + } + if !s.failFastInvariants { + if repaired := s.repairRelatedResource(ctx, rt, rid); repaired { + l.Warn("ingest invariant I3: REPAIRED a lost grant-inserted resource by copying the previous sync's row; "+ + "the replay path failed to carry it — investigate before trusting warm syncs", + zap.String("resource_type_id", rt), + zap.String("resource_id", rid), + ) + return nil + } + } + // WARM sync: replay is implicated (possibly alongside a + // connector bug — the runners' cold retry disambiguates: + // a clean cold run means replay was at fault and the sync + // self-healed; a cold failure lands in the branch above. + return fmt.Errorf( + "ingest invariant I3 violated: grants reference resource %s/%s via InsertResourceGrants but no resource row exists "+ + "(related-resource insertion was lost and could not be repaired from the previous sync): %w", + rt, rid, ErrReplayIntegrity) + } + // Tolerated today: connectors emitting grants for unlisted + // resources without InsertResourceGrants. Visible, not fatal — + // except under fail-fast, whose contract is that every tolerated + // warn becomes a named failure (the harness must catch connector + // shapes production only logs). + if s.failFastInvariants { + return fmt.Errorf( + "ingest invariant I3 violated: grants reference resource %s/%s which was never synced "+ + "(no InsertResourceGrants annotation — tolerated with a warning in production, promoted under fail-fast)", + rt, rid) + } + l.Warn("ingest invariant I3: grants reference a resource that was never synced", + zap.String("resource_type_id", rt), + zap.String("resource_id", rid), + ) + return nil + }) +} + +// repairRelatedResource attempts I3's repair: copy the missing +// grant-inserted resource row from the previous sync's store. Best +// effort — a false return falls through to the hard failure. +func (s *syncer) repairRelatedResource(ctx context.Context, rt, rid string) bool { + repairer, ok := s.store.(relatedResourceRepairer) + if !ok || s.sourceCache.prevReader == nil { + return false + } + repaired, err := repairer.RepairRelatedResource(ctx, s.sourceCache.prevReader, rt, rid) + if err != nil { + ctxzap.Extract(ctx).Warn("ingest invariant I3: related-resource repair failed", + zap.String("resource_type_id", rt), + zap.String("resource_id", rid), + zap.Error(err), + ) + return false + } + return repaired +} + +// checkChildScheduling is invariant I4: every stored resource carrying a +// ChildResourceType annotation (and passing the resource-type filter) +// must have had its child action scheduled. Check-only — scheduling +// cannot be derived after the fact (an executed child action that +// returned zero rows leaves no store evidence). +func (s *syncer) checkChildScheduling(ctx context.Context) error { + if !s.failFastInvariants { + // Fail-fast mode only: the check costs a full post-collection + // resource scan (value decode + annotation walk per row) and in + // default mode a violation was only ever a warning — a bad + // trade at whale scale. Tests and the harness run fail-fast and + // enforce it there. + return nil + } + if s.syncType != connectorstore.SyncTypeFull || !s.resourcesPhaseRanHere { + // Resumed past the resources phase: the scheduled set from the + // prior process is gone; the predicate is unverifiable. + return nil + } + syncResourceTypeMap := make(map[string]bool, len(s.syncResourceTypes)) + for _, rt := range s.syncResourceTypes { + syncResourceTypeMap[rt] = true + } + + var violations []string + pageToken := "" + for { + resp, err := s.store.ListResources(ctx, v2.ResourcesServiceListResourcesRequest_builder{ + PageToken: pageToken, + ActiveSyncId: s.getActiveSyncID(), + }.Build()) + if err != nil { + return fmt.Errorf("ingest invariant I4: listing resources: %w", err) + } + for _, r := range resp.GetList() { + for _, a := range r.GetAnnotations() { + if !a.MessageIs((*v2.ChildResourceType)(nil)) { + continue + } + crt := &v2.ChildResourceType{} + if err := a.UnmarshalTo(crt); err != nil { + return fmt.Errorf("ingest invariant I4: parsing ChildResourceType on %s/%s: %w", + r.GetId().GetResourceType(), r.GetId().GetResource(), err) + } + childType := crt.GetResourceTypeId() + if childType == "" { + continue + } + if len(s.syncResourceTypes) > 0 && !syncResourceTypeMap[childType] { + continue + } + if !s.childSchedule.has(childType, r.GetId().GetResourceType(), r.GetId().GetResource()) { + violations = append(violations, fmt.Sprintf("%s under %s/%s", + childType, r.GetId().GetResourceType(), r.GetId().GetResource())) + } + } + } + if pageToken = resp.GetNextPageToken(); pageToken == "" { + break + } + } + if len(violations) > 0 { + // Sorted so the error text is byte-stable regardless of store + // iteration order (the verdict already is). + sort.Strings(violations) + return fmt.Errorf( + "ingest invariant I4 violated: %d child resource sync(s) were never scheduled: %v", + len(violations), violations) + } + return nil +} + +// exclusionGroupTracker is the pure validation core behind invariant I5: +// one default per group, one resource type per group, at most +// maxEntitlementsPerExclusionGroup members. Error text intentionally +// matches the retired streaming validator. Recording the same +// entitlement twice is idempotent for the default check but counts twice +// toward the cap — callers feed each stored entitlement exactly once +// (the store keyspace holds one row per entitlement id). +type exclusionGroupTracker struct { + groups map[string]*exclusionGroupState +} + +type exclusionGroupState struct { + resourceTypeID string + defaultEntID string + count uint32 +} + +func (t *exclusionGroupTracker) record(ent *v2.Entitlement) error { + eg := &v2.EntitlementExclusionGroup{} + annos := annotations.Annotations(ent.GetAnnotations()) + ok, err := annos.Pick(eg) + if err != nil { + return fmt.Errorf("parsing exclusion group on %q: %w", ent.GetId(), err) + } + if !ok || eg.GetExclusionGroupId() == "" { + return nil + } + groupID := eg.GetExclusionGroupId() + if t.groups == nil { + t.groups = map[string]*exclusionGroupState{} + } + st := t.groups[groupID] + if st == nil { + st = &exclusionGroupState{} + t.groups[groupID] = st + } + rt := ent.GetResource().GetId().GetResourceType() + if st.resourceTypeID == "" { + st.resourceTypeID = rt + } else if st.resourceTypeID != rt { + return fmt.Errorf("exclusion group %q is used on multiple resource types (%q and %q); "+ + "exclusion groups may span resources but must be scoped to a single resource type", + groupID, st.resourceTypeID, rt) + } + if eg.GetIsDefault() { + if st.defaultEntID != "" && st.defaultEntID != ent.GetId() { + return fmt.Errorf("exclusion group %q has multiple default entitlements (%q and %q); "+ + "at most one entitlement per exclusion group may set is_default=true", + groupID, st.defaultEntID, ent.GetId()) + } + st.defaultEntID = ent.GetId() + } + st.count++ + if st.count > maxEntitlementsPerExclusionGroup { + return fmt.Errorf("exclusion group %q has too many entitlements (%d); "+ + "at most %d entitlements are allowed per exclusion group", + groupID, st.count, maxEntitlementsPerExclusionGroup) + } + return nil +} + +// validateStoredExclusionGroups is invariant I5: exclusion-group +// invariants validated over the STORED entitlement keyspace. This is the +// sole exclusion-group validation: it covers fresh, replayed, and +// generated (static) entitlements uniformly, on every engine, regardless +// of how rows arrived. +// +// Verdicts ride the warm/cold ladder like the other FAIL-class +// invariants (I3/I6/enabled I7–I9): replayed stale entitlement rows can +// manufacture warm-only conflicts (a stale group member beside a fresh +// one), so a warm failure wraps ErrReplayIntegrity for discard+cold +// while a cold failure is attributed plainly to the connector. Store IO +// errors stay plain — they are not replay-implicated. +func (s *syncer) validateStoredExclusionGroups(ctx context.Context) error { + if s.syncType != connectorstore.SyncTypeFull { + // Store-derived verdicts are only evaluable over a COMPLETE + // keyspace; a partial sync writes a deliberate subset (the + // referential invariants I3/I7-I9 gate identically). Partial + // syncs keep the page-level streaming validation + // (validateEntitlementExclusionGroups). + return nil + } + tracker := &exclusionGroupTracker{} + pageToken := "" + for { + resp, err := s.store.ListEntitlements(ctx, v2.EntitlementsServiceListEntitlementsRequest_builder{ + PageToken: pageToken, + ActiveSyncId: s.getActiveSyncID(), + }.Build()) + if err != nil { + return fmt.Errorf("ingest invariant I5: listing entitlements: %w", err) + } + for _, ent := range resp.GetList() { + if err := tracker.record(ent); err != nil { + return s.wrapWarmReplayIntegrity(fmt.Errorf("ingest invariant I5 violated: %w", err)) + } + } + if pageToken = resp.GetNextPageToken(); pageToken == "" { + break + } + } + return nil +} + +// maxDanglingIDExamples caps the id samples carried on the aggregated +// dangling-reference warnings. Production data shows a handful of +// distinct ids explain millions of dangling rows (magic-id construction +// bugs repeat), so a small sample is enough to pivot on. +const maxDanglingIDExamples = 25 + +// danglingTypeProbe memoizes "does a resource-type row exist" — the +// attribution split for dangling references. A reference into a type +// with NO type row means the type was never synced (typically a type +// added to the connector after the instance was configured and left +// disabled by default — a CONFIG gap); a missing row under a type that +// EXISTS means the id was constructed for a row the enumeration never +// produced (a magic-id BUG). +type danglingTypeProbe struct { + s *syncer + known map[string]bool +} + +func (p *danglingTypeProbe) typeExists(ctx context.Context, resourceTypeID string) (bool, error) { + if v, ok := p.known[resourceTypeID]; ok { + return v, nil + } + _, err := p.s.store.GetResourceType(ctx, reader_v2.ResourceTypesReaderServiceGetResourceTypeRequest_builder{ + ResourceTypeId: resourceTypeID, + }.Build()) + if err != nil && status.Code(err) != codes.NotFound { + // A read failure is NOT "type never synced": that verdict picks + // the DROP arm, so an IO error or canceled context here would + // seal a sanitized artifact where policy demands a loud failure. + // Propagate, and memoize nothing — only definitive answers may + // be cached (a memoized transient error would poison every later + // reference to the type). + return false, fmt.Errorf("resource-type probe for %q: %w", resourceTypeID, err) + } + if p.known == nil { + p.known = map[string]bool{} + } + exists := err == nil + p.known[resourceTypeID] = exists + return exists, nil +} + +// unexpectedDanglingError renders the ENABLED-TYPE dangling verdict of +// the replay policy: a missing referent whose resource type IS synced is +// never an expected configuration gap — it is connector data/behavior +// (magic-id construction, deleted-referent emission) or replay-carried +// staleness, and the artifact must not seal with the rows silently +// dropped. A WARM sync is replay-implicated: the error carries +// ErrReplayIntegrity so the runners discard the output and retry cold. +// A COLD sync with the same evidence is terminal — the cold retry +// already happened (or could never have helped), so the failure is +// attributed plainly to the connector. +// +// Disabled-type gaps (referent's type never synced) keep the drop arm: +// they are expected configuration shapes, sealed referentially +// consistent with one aggregated warning, and their scopes' manifest +// entries are invalidated so a later type-enable re-fetches them cold. +func (s *syncer) unexpectedDanglingError(format string, args ...any) error { + return s.wrapWarmReplayIntegrity(fmt.Errorf(format, args...)) +} + +// wrapWarmReplayIntegrity classifies a FAIL-class invariant verdict for +// the cold-retry ladder: on a WARM sync the evidence may be +// replay-carried staleness, so the error wraps ErrReplayIntegrity and +// the runners discard the output and retry cold; the same evidence on a +// COLD sync is terminal and attributed plainly. Only VERDICTS ride the +// ladder — store IO errors are not replay-implicated and stay plain. +func (s *syncer) wrapWarmReplayIntegrity(err error) error { + if s.sourceCache.prev != nil { + return fmt.Errorf("%w (warm sync: discard the output and retry cold): %w", err, ErrReplayIntegrity) + } + return err +} + +// danglingAttribution renders the attribution split for the aggregated +// drop warnings. The drop arm only ever sees two shapes: references +// into never-synced types (config gaps, any mode) and enabled-type +// references reached under the compaction expand pass (keep-newer +// merges manufacture dangling refs no input contained — by design, not +// a connector bug; enabled-type danglings in a NORMAL sync take the +// FAIL arm and never reach a warning). +func danglingAttribution(unsyncedTypeCount, mergeManufacturedCount int) string { + switch { + case mergeManufacturedCount == 0: + return "referenced resource type(s) never synced — likely disabled on the connector (config gap), not a code bug" + case unsyncedTypeCount == 0: + return "references target SYNCED types, dropped during the compaction expand pass — keep-newer merges manufacture these by design; not a connector bug" + default: + return "mixed: some references target never-synced types (config gap), some are compaction-merge-manufactured (expand pass)" + } +} + +// checkEntitlementResourceReferences is invariant I7: every distinct +// resource referenced by an entitlement row must exist as a resource +// row. One seek per distinct resource among entitlements — bounded by +// the entitlement count. Default mode DROPS disabled-type danglings +// (uplift skips them anyway — see the header) and warns once with +// aggregates; enabled-type danglings FAIL (warm → ErrReplayIntegrity; +// see unexpectedDanglingError); fail-fast mode hard-fails everything. +func (s *syncer) checkEntitlementResourceReferences(ctx context.Context) error { + if s.syncType != connectorstore.SyncTypeFull { + return nil + } + facts, ok := s.store.(dotc1z.IngestInvariantStore) + if !ok { + return nil // engine without the inspection surface + } + probe := &danglingTypeProbe{s: s} + danglingResources, unsyncedType, mergeManufactured := 0, 0, 0 + var droppedRows int64 + var resourceExamples, entIDExamples []string + err := facts.ForEachDistinctEntitlementResource(ctx, func(rt, rid string) error { + exists, err := facts.HasResourceRecord(ctx, rt, rid) + if err != nil { + return fmt.Errorf("ingest invariant I7: probing resource %s/%s: %w", rt, rid, err) + } + if exists { + return nil + } + typeSynced, probeErr := probe.typeExists(ctx, rt) + if probeErr != nil { + return fmt.Errorf("ingest invariant I7: %w", probeErr) + } + if s.failFastInvariants { + return fmt.Errorf( + "ingest invariant I7 violated: entitlements reference resource %s/%s but no resource row exists "+ + "(resource type synced: %t — false means a config gap, true a magic-id construction bug)", + rt, rid, typeSynced) + } + if !s.onlyExpandGrants && typeSynced { + // Enabled-type dangling: the referent's type IS synced, so the + // missing row is unexpected — never sealed away (replay + // policy; see unexpectedDanglingError). The compaction + // expand pass is exempt: keep-newer merges legitimately + // manufacture dangling refs no input contained, and the drop + // pass is what converges the artifact. + return s.unexpectedDanglingError( + "ingest invariant I7 violated: entitlements reference resource %s/%s but no resource row exists under a SYNCED type", + rt, rid) + } + n, ids, err := facts.DeleteEntitlementsForResource(ctx, rt, rid, maxDanglingIDExamples-len(entIDExamples)) + if err != nil { + return fmt.Errorf("ingest invariant I7: dropping entitlements for %s/%s: %w", rt, rid, err) + } + danglingResources++ + droppedRows += n + if typeSynced { + // Reachable only under onlyExpandGrants (a normal sync FAILs + // enabled-type danglings above): a keep-newer merge + // manufactured this gap, not a connector bug. + mergeManufactured++ + } else { + unsyncedType++ + } + entIDExamples = append(entIDExamples, ids...) + if len(resourceExamples) < maxDanglingIDExamples { + resourceExamples = append(resourceExamples, rt+"/"+rid) + } + return nil + }) + if err != nil { + return err + } + if droppedRows > 0 { + ctxzap.Extract(ctx).Warn("ingest invariant I7: DROPPED entitlements referencing resources with no resource row", + zap.Int64("dropped_entitlements", droppedRows), + zap.Int("dangling_resources", danglingResources), + zap.Int("refs_into_unsynced_types", unsyncedType), + zap.Int("refs_manufactured_by_compaction_merge", mergeManufactured), + zap.String("attribution", danglingAttribution(unsyncedType, mergeManufactured)), + zap.Strings("resource_examples", resourceExamples), + zap.Strings("entitlement_id_examples", entIDExamples), + ) + } + return nil +} + +// checkGrantEntitlementReferences is invariant I8: every distinct +// entitlement referenced by a grant must exist as an entitlement row. +// One seek plus one point probe per distinct entitlement — O(distinct +// entitlements), never O(grants). +// +// InsertResourceGrants grants are EXEMPT in every mode, per GRANT: that +// pattern has always produced grants whose entitlements have no row (the +// machinery inserts the grant's embedded resource; no listing ever +// returns an entitlement for it), and downstream consumers synthesize +// the entitlement from the grant. The exemption follows the ownership +// rule (docs/tasks/dangling-reference-drops.md) at the grant's own +// granularity: a connector-owned grant dangling under the same missing +// entitlement as machinery-owned IRG grants is still a violation — a +// resource- or entitlement-level exemption would let one IRG grant +// launder every dangling reference that shares its resource. +// +// Default mode DROPS disabled-type danglings (uplift reads grants BY +// entitlement, so rows under a missing entitlement are never even read +// platform-side) and warns once with aggregates; enabled-type danglings +// FAIL (warm → ErrReplayIntegrity; see unexpectedDanglingError and the +// header); fail-fast mode hard-fails everything. +func (s *syncer) checkGrantEntitlementReferences(ctx context.Context) error { + if s.syncType != connectorstore.SyncTypeFull { + return nil + } + facts, ok := s.store.(dotc1z.IngestInvariantStore) + if !ok { + return nil // engine without the inspection surface + } + probe := &danglingTypeProbe{s: s} + danglingEnts, unsyncedType, mergeManufactured := 0, 0, 0 + var droppedRows, insertFactKept int64 + var entIDExamples []string + err := facts.ForEachDanglingGrantEntitlement(ctx, func(entitlementID, rt, rid string) error { + typeSynced, probeErr := probe.typeExists(ctx, rt) + if probeErr != nil { + return fmt.Errorf("ingest invariant I8: %w", probeErr) + } + if s.failFastInvariants { + allCarry, err := facts.GrantsForEntitlementAllCarryInsertFact(ctx, entitlementID, rt, rid) + if err != nil { + return fmt.Errorf("ingest invariant I8: probing grant annotations for entitlement %q: %w", entitlementID, err) + } + if allCarry { + return nil // the established InsertResourceGrants shape + } + return fmt.Errorf( + "ingest invariant I8 violated: grants without InsertResourceGrants reference entitlement %q (resource %s/%s) but no entitlement row exists "+ + "(resource type synced: %t — false means a config gap, true a magic-id construction bug "+ + "or a grants scope replayed against a refreshed entitlement set)", + entitlementID, rt, rid, typeSynced) + } + if !s.onlyExpandGrants && typeSynced { + // Enabled-type dangling (replay policy; see I7's twin arm). + // The per-grant IRG exemption still applies: an entitlement + // whose grants ALL carry InsertResourceGrants is the + // machinery-owned shape in every mode. + allCarry, err := facts.GrantsForEntitlementAllCarryInsertFact(ctx, entitlementID, rt, rid) + if err != nil { + return fmt.Errorf("ingest invariant I8: probing grant annotations for entitlement %q: %w", entitlementID, err) + } + if allCarry { + return nil + } + return s.unexpectedDanglingError( + "ingest invariant I8 violated: grants without InsertResourceGrants reference entitlement %q (resource %s/%s) but no entitlement row exists under a SYNCED type", + entitlementID, rt, rid) + } + n, skipped, err := facts.DeleteGrantsForEntitlement(ctx, entitlementID, rt, rid) + if err != nil { + return fmt.Errorf("ingest invariant I8: dropping grants for entitlement %q: %w", entitlementID, err) + } + insertFactKept += skipped + if n == 0 { + // Pure machinery shape: every grant carried the insert fact. + return nil + } + danglingEnts++ + droppedRows += n + if typeSynced { + // Only reachable under onlyExpandGrants — see I7's twin arm. + mergeManufactured++ + } else { + unsyncedType++ + } + if len(entIDExamples) < maxDanglingIDExamples { + entIDExamples = append(entIDExamples, entitlementID) + } + return nil + }) + if err != nil { + return err + } + if droppedRows > 0 { + ctxzap.Extract(ctx).Warn("ingest invariant I8: DROPPED grants referencing entitlements with no entitlement row", + zap.Int64("dropped_grants", droppedRows), + zap.Int("dangling_entitlements", danglingEnts), + zap.Int64("insert_resource_grants_kept", insertFactKept), + zap.Int("refs_into_unsynced_types", unsyncedType), + zap.Int("refs_manufactured_by_compaction_merge", mergeManufactured), + zap.String("attribution", danglingAttribution(unsyncedType, mergeManufactured)), + zap.Strings("entitlement_id_examples", entIDExamples), + ) + } + return nil +} + +// checkGrantPrincipalReferences is invariant I9: every distinct +// principal referenced by a grant must exist as a resource row. Scans +// the by_principal index (one seek + probe per distinct principal); a +// pending deferred index build is forced first so synth-written grants +// are covered — that build is EndSync's own pass moved earlier, so the +// net cost is ~zero. +// +// Grants carrying ExternalResourceMatch* annotations are EXEMPT in +// every mode: unprocessed match carriers mean no external resource file +// was configured (the match op deletes carriers when it runs) — they +// are evidence of a config gap, not bad data, and dropping them would +// make a misconfigured deployment look clean. +// +// Default mode DROPS disabled-type danglings (uplift skips grants whose +// principal resolves no platform object) and warns once with +// aggregates; enabled-type danglings FAIL (warm → ErrReplayIntegrity; +// see unexpectedDanglingError); fail-fast mode hard-fails everything. +func (s *syncer) checkGrantPrincipalReferences(ctx context.Context) error { + if s.syncType != connectorstore.SyncTypeFull { + return nil + } + facts, ok := s.store.(dotc1z.IngestInvariantStore) + if !ok { + return nil // engine without the inspection surface + } + if err := facts.EnsureGrantIndexes(ctx); err != nil { + return fmt.Errorf("ingest invariant I9: ensuring grant indexes: %w", err) + } + if s.testSourceCacheHaltHook != nil { + // Seam between the deferred index build (marker durably cleared) + // and I9's scan/drops: a crash here must resume without EndSync + // repeating the O(grants) rebuild and with I9 re-scanning cleanly. + if err := s.testSourceCacheHaltHook("invariants-I9-indexes-ensured", ""); err != nil { + return err + } + } + probe := &danglingTypeProbe{s: s} + danglingPrincipals, unsyncedType, mergeManufactured := 0, 0, 0 + var droppedRows, matchCarriersKept int64 + var principalExamples []string + // matchCarriersKept counts GRANTS uniformly: fully-annotated + // principals contribute their carrier-grant count, mixed principals + // contribute the per-grant skip count from the drop. + err := facts.ForEachDanglingGrantPrincipal(ctx, func(rt, rid string, matchAnnotatedOnly bool, carrierGrants int64) error { + if matchAnnotatedOnly { + matchCarriersKept += carrierGrants + return nil + } + typeSynced, probeErr := probe.typeExists(ctx, rt) + if probeErr != nil { + return fmt.Errorf("ingest invariant I9: %w", probeErr) + } + if s.failFastInvariants { + return fmt.Errorf( + "ingest invariant I9 violated: grants reference principal %s/%s but no resource row exists "+ + "(resource type synced: %t — false means a config gap, true means the connector "+ + "emitted grants for a principal it never synced)", + rt, rid, typeSynced) + } + if !s.onlyExpandGrants && typeSynced { + // Enabled-type dangling (replay policy; see I7's twin arm). + // Mixed populations land here too: exempt carriers were + // already excluded above (matchAnnotatedOnly), so a plain + // grant on a synced-type principal with no row is never + // silently dropped. + return s.unexpectedDanglingError( + "ingest invariant I9 violated: grants reference principal %s/%s but no resource row exists under a SYNCED type", + rt, rid) + } + deleted, skipped, err := facts.DeleteGrantsForPrincipal(ctx, rt, rid) + if err != nil { + return fmt.Errorf("ingest invariant I9: dropping grants for principal %s/%s: %w", rt, rid, err) + } + danglingPrincipals++ + droppedRows += deleted + matchCarriersKept += skipped + if typeSynced { + // Only reachable under onlyExpandGrants — see I7's twin arm. + mergeManufactured++ + } else { + unsyncedType++ + } + if len(principalExamples) < maxDanglingIDExamples { + principalExamples = append(principalExamples, rt+"/"+rid) + } + return nil + }) + if err != nil { + return err + } + l := ctxzap.Extract(ctx) + if droppedRows > 0 { + l.Warn("ingest invariant I9: DROPPED grants referencing principals with no resource row", + zap.Int64("dropped_grants", droppedRows), + zap.Int("dangling_principals", danglingPrincipals), + zap.Int("refs_into_unsynced_types", unsyncedType), + zap.Int("refs_manufactured_by_compaction_merge", mergeManufactured), + zap.String("attribution", danglingAttribution(unsyncedType, mergeManufactured)), + zap.Strings("principal_examples", principalExamples), + ) + } + if matchCarriersKept > 0 { + l.Warn("ingest invariant I9: kept unprocessed external-match carrier grants with dangling principals — was the external resource file configured?", + zap.Int64("match_carriers_kept", matchCarriersKept), + ) + } + return nil +} + +// checkSourceCacheScopeConsistency is invariant I6: at the sealed sync's +// quiesce point, every scope present in a by_source_scope index must have +// a manifest entry. Post-processing only re-stamps rows with scopes that +// already completed (external-match transformed grants and expansion +// writes inherit the SOURCE scope), so an orphan at this seam is always a +// lost manifest write or a stamp leak — and the damage is deferred: THIS +// sync reads clean, the NEXT sync replays from the damaged state. +func (s *syncer) checkSourceCacheScopeConsistency(ctx context.Context) error { + if s.syncType != connectorstore.SyncTypeFull { + // Partial syncs never replay and never enable the source-cache + // write side (configureSourceCache degrades on non-full syncs), + // so scope/manifest evidence in a partial store is inherited + // state this sync neither produced nor will serve replay from — + // same gate as the referential invariants. + return nil + } + if s.onlyExpandGrants { + // Expansion-only runs ingest no pages, so scope/manifest state + // was settled (and I6-checked) by whoever produced the store. + // The load-bearing case is compaction's expand-grants pass: a + // fold output deliberately violates I6's premise — the manifest + // was cleared (no input's validator describes the merged rows) + // while the base copy's scope stamps remain — so every stamped + // scope would be reported as an orphan on a healthy artifact. + return nil + } + sc, ok := s.store.(dotc1z.SourceCacheStore) + if !ok { + return nil // engine without source-cache state + } + orphans, err := sc.SourceCacheOrphanScopes(ctx) + if err != nil { + return fmt.Errorf("ingest invariant I6: scanning scope indexes: %w", err) + } + if len(orphans) == 0 { + return nil + } + // A stale replay index in the sealed output ALWAYS fails the sync + // (replay policy): I6 has no repair arm, and warn-and-seal would + // publish an artifact whose next replay silently drops or resurrects + // rows. On a WARM sync the error carries ErrReplayIntegrity — the + // runners discard the output and re-run cold (replay-path fault, + // self-healed). A cold re-run that reproduces the orphan fails + // terminally here WITHOUT the sentinel (a put-path bug, correctly + // fatal and correctly attributed). + err = fmt.Errorf( + "ingest invariant I6 violated: scope index entries exist with no manifest entry (lost manifest write or stamp leak): %v", + orphans) + if s.sourceCache.prev != nil { + return fmt.Errorf("%w: %w", err, ErrReplayIntegrity) + } + return err +} + +// runIngestionInvariants evaluates the post-collection (check-flavored) +// invariants: I5, I6, I4, I7, I3, I8, I9, in that order — cheapest and +// most behavior-critical first, and drop CASCADES respected: I7 drops +// dangling entitlements, which orphans their grants, which I8 then +// catches and drops; I9 runs last over whatever grants survive. Runs +// after the action loop drains and before EndSync, so every ingestion +// path — stream, replay, resume re-runs, expansion, external-match +// processing — has finished writing. Idempotent: a resumed sync that +// re-reaches this point re-evaluates with the same verdict (drops +// already applied leave nothing dangling on re-run). +func (s *syncer) runIngestionInvariants(ctx context.Context) error { + // halt fires the test-only seam hook between invariants, so the + // crash sweep (TestSourceCache_DirtyHaltSweepInsideInvariantPass) + // can pin resume convergence at every mid-pass stop point — e.g. + // "I7's drops committed, I8 never ran" must converge on re-run via + // the drop cascade. Nil-guarded no-op in production. + halt := func(stage string) error { + if s.testSourceCacheHaltHook == nil { + return nil + } + return s.testSourceCacheHaltHook(stage, "") + } + if err := s.validateStoredExclusionGroups(ctx); err != nil { + return err + } + if err := halt("invariants-I5-complete"); err != nil { + return err + } + if err := s.checkSourceCacheScopeConsistency(ctx); err != nil { + return err + } + if err := halt("invariants-I6-complete"); err != nil { + return err + } + if err := s.checkChildScheduling(ctx); err != nil { + return err + } + if err := s.checkEntitlementResourceReferences(ctx); err != nil { + return err + } + if err := halt("invariants-I7-complete"); err != nil { + return err + } + if err := s.checkGrantResourceReferences(ctx); err != nil { + return err + } + if err := halt("invariants-I3-complete"); err != nil { + return err + } + if err := s.checkGrantEntitlementReferences(ctx); err != nil { + return err + } + if err := halt("invariants-I8-complete"); err != nil { + return err + } + return s.checkGrantPrincipalReferences(ctx) +} diff --git a/pkg/sync/ingest_invariants_granularity_test.go b/pkg/sync/ingest_invariants_granularity_test.go new file mode 100644 index 000000000..d3216a01d --- /dev/null +++ b/pkg/sync/ingest_invariants_granularity_test.go @@ -0,0 +1,190 @@ +package sync //nolint:revive,nolintlint // we can't change the package name for backwards compatibility + +// Granularity tests for the invariant exemptions and gates: +// +// - I8's InsertResourceGrants exemption is per GRANT (the ownership +// rule's unit), not per entitlement resource — one machinery-owned +// IRG grant must not launder connector-owned dangling references +// that share its resource. +// - I6 (scope/manifest consistency) is skipped for expand-grants-only +// syncs: compaction's fold output deliberately clears the manifest +// while keeping scope stamps, so on that path every stamped scope of +// a healthy artifact would read as an orphan. + +import ( + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + reader_v2 "github.com/conductorone/baton-sdk/pb/c1/reader/v2" + "github.com/conductorone/baton-sdk/pkg/connectorstore" + "github.com/conductorone/baton-sdk/pkg/dotc1z" + "github.com/conductorone/baton-sdk/pkg/logging" + "github.com/conductorone/baton-sdk/pkg/sourcecache" + "github.com/conductorone/baton-sdk/pkg/types/grant" + rs "github.com/conductorone/baton-sdk/pkg/types/resource" +) + +// TestIngestInvariantI8ExemptionIsPerGrant pins the granularity: a +// resource holding BOTH a machinery-owned IRG grant (dangling by design) +// and a connector-owned grant referencing a different missing entitlement +// must keep the former and drop (or fail on) the latter. +func TestIngestInvariantI8ExemptionIsPerGrant(t *testing.T) { + ctx, err := logging.Init(t.Context()) + require.NoError(t, err) + tmpDir := t.TempDir() + + repo, err := rs.NewResource("Repo r1", equivRepoRT, "r1") + require.NoError(t, err) + irgGrant := repairTestGrant(t, repo) // "admin" slug, carries InsertResourceGrants + ghost := v2.Resource_builder{ + Id: v2.ResourceId_builder{ResourceType: "user", Resource: "bob"}.Build(), + }.Build() + // Same resource, different missing entitlement, NO annotation: + // connector-owned dangling reference. + plainGrant := grant.NewGrant(repo, "viewer", ghost.GetId()) + + // NO resource-type row for eq_repo: a disabled-type configuration + // gap, which is the DROP arm of the replay policy (enabled-type + // danglings fail the sync instead — see the referential tests). + cur := newRepairTestStore(ctx, t, filepath.Join(tmpDir, "cur.c1z"), tmpDir) + _, err = cur.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + require.NoError(t, cur.PutResources(ctx, repo)) + require.NoError(t, cur.PutGrants(ctx, irgGrant, plainGrant)) + + s := &syncer{store: cur, syncType: connectorstore.SyncTypeFull} + + // Fail-fast: the connector-owned dangling grant must fail the sync — + // the sibling IRG grant on the same resource must not exempt it. + s.failFastInvariants = true + err = s.checkGrantEntitlementReferences(ctx) + require.Error(t, err, "an IRG grant on the resource must not exempt an unrelated dangling reference") + require.Contains(t, err.Error(), "ingest invariant I8") + require.Contains(t, err.Error(), "viewer") + require.NotErrorIs(t, err, ErrReplayIntegrity) + + // Default mode: the connector-owned grant drops, the IRG grant stays. + s.failFastInvariants = false + require.NoError(t, s.checkGrantEntitlementReferences(ctx)) + _, err = cur.GetGrant(ctx, reader_v2.GrantsReaderServiceGetGrantRequest_builder{ + GrantId: plainGrant.GetId(), + }.Build()) + require.Error(t, err, "the connector-owned dangling grant must be dropped") + _, err = cur.GetGrant(ctx, reader_v2.GrantsReaderServiceGetGrantRequest_builder{ + GrantId: irgGrant.GetId(), + }.Build()) + require.NoError(t, err, "the machinery-owned IRG grant must never be dropped") + + // Idempotent: after the drop, both modes find nothing. + require.NoError(t, s.checkGrantEntitlementReferences(ctx)) + s.failFastInvariants = true + require.NoError(t, s.checkGrantEntitlementReferences(ctx)) + + require.NoError(t, cur.Close(ctx)) +} + +// TestIngestInvariantI8MixedGrantsUnderOneEntitlement pins the per-grant +// rule within a single missing entitlement: IRG carriers and plain grants +// can share the SAME entitlement identity (an IRG page and a normal page +// both emitted grants for it); only the plain grants drop. +func TestIngestInvariantI8MixedGrantsUnderOneEntitlement(t *testing.T) { + ctx, err := logging.Init(t.Context()) + require.NoError(t, err) + tmpDir := t.TempDir() + + repo, err := rs.NewResource("Repo r1", equivRepoRT, "r1") + require.NoError(t, err) + irgGrant := repairTestGrant(t, repo) // "admin" entitlement, alice, annotated + ghost := v2.Resource_builder{ + Id: v2.ResourceId_builder{ResourceType: "user", Resource: "bob"}.Build(), + }.Build() + plainGrant := grant.NewGrant(repo, "admin", ghost.GetId()) // same entitlement, no annotation + + // Disabled-type gap (no eq_repo type row): the drop arm. + cur := newRepairTestStore(ctx, t, filepath.Join(tmpDir, "cur.c1z"), tmpDir) + _, err = cur.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + require.NoError(t, cur.PutResources(ctx, repo)) + require.NoError(t, cur.PutGrants(ctx, irgGrant, plainGrant)) + + s := &syncer{store: cur, syncType: connectorstore.SyncTypeFull} + + s.failFastInvariants = true + err = s.checkGrantEntitlementReferences(ctx) + require.Error(t, err, "a mixed entitlement (IRG + plain grants) must fail fast on the plain grant") + + s.failFastInvariants = false + require.NoError(t, s.checkGrantEntitlementReferences(ctx)) + _, err = cur.GetGrant(ctx, reader_v2.GrantsReaderServiceGetGrantRequest_builder{ + GrantId: plainGrant.GetId(), + }.Build()) + require.Error(t, err, "the plain grant under the mixed entitlement must be dropped") + _, err = cur.GetGrant(ctx, reader_v2.GrantsReaderServiceGetGrantRequest_builder{ + GrantId: irgGrant.GetId(), + }.Build()) + require.NoError(t, err, "the IRG carrier under the mixed entitlement must be kept") + + require.NoError(t, cur.Close(ctx)) +} + +// TestIngestInvariantI6SkippedForExpandOnlySyncs pins the expand-grants +// gate: an orphan scope (stamped rows, no manifest entry) fails a normal +// sealed sync's I6, but an expansion-only run — compaction's pass over a +// fold output whose manifest was deliberately cleared — must not evaluate +// it at all. +func TestIngestInvariantI6SkippedForExpandOnlySyncs(t *testing.T) { + ctx, err := logging.Init(t.Context()) + require.NoError(t, err) + tmpDir := t.TempDir() + + repo, err := rs.NewResource("Repo r1", equivRepoRT, "r1") + require.NoError(t, err) + ghost := v2.Resource_builder{ + Id: v2.ResourceId_builder{ResourceType: "user", Resource: "alice"}.Build(), + }.Build() + g := grant.NewGrant(repo, "admin", ghost.GetId()) + + // Stamped rows with NO manifest entry: the orphan shape. + cur := newRepairTestStore(ctx, t, filepath.Join(tmpDir, "cur.c1z"), tmpDir) + _, err = cur.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + require.NoError(t, cur.PutResourceTypes(ctx, equivRepoRT)) + require.NoError(t, cur.PutResources(ctx, repo)) + scope := sourcecache.HashScope("grants:r1") + require.NoError(t, cur.PutGrants(sourcecache.WithScope(ctx, scope), g)) + + s := &syncer{store: cur, syncType: connectorstore.SyncTypeFull, failFastInvariants: true} + + // Sanity: a normal sync detects the orphan and fails in EVERY mode + // (replay policy: a stale replay index is never sealed away). This + // syncer is COLD (no previous sync), so the failure is attributed + // plainly — the replay-integrity sentinel rides only warm syncs, + // where a discarded output and a cold retry can remediate. + err = s.checkSourceCacheScopeConsistency(ctx) + require.Error(t, err) + require.NotErrorIs(t, err, ErrReplayIntegrity) + require.Contains(t, err.Error(), "ingest invariant I6") + + // Warm shape: the same evidence carries the sentinel so the runners + // discard the output and retry cold. + s.sourceCache.prev = struct{ dotc1z.SourceCacheStore }{} + err = s.checkSourceCacheScopeConsistency(ctx) + require.ErrorIs(t, err, ErrReplayIntegrity) + s.sourceCache.prev = nil + + // Default mode is no longer warn-and-seal: the same cold failure. + s.failFastInvariants = false + err = s.checkSourceCacheScopeConsistency(ctx) + require.Error(t, err, "I6 must fail in default mode too — warn-and-seal would publish a poisoned replay source") + s.failFastInvariants = true + + // The expansion-only pass must skip the check entirely. + s.onlyExpandGrants = true + require.NoError(t, s.checkSourceCacheScopeConsistency(ctx), + "expand-grants-only syncs must not evaluate I6: fold outputs legitimately hold stamps without a manifest") + + require.NoError(t, cur.Close(ctx)) +} diff --git a/pkg/sync/ingest_invariants_i9_mixed_test.go b/pkg/sync/ingest_invariants_i9_mixed_test.go new file mode 100644 index 000000000..32ca0fb4b --- /dev/null +++ b/pkg/sync/ingest_invariants_i9_mixed_test.go @@ -0,0 +1,96 @@ +package sync //nolint:revive,nolintlint // we can't change the package name for backwards compatibility + +// Mixed-population pin for I9's exemption: exemptions must be judged per +// GRANT, not per principal. One dangling principal carrying BOTH an +// unprocessed external-match carrier and a plain grant must keep exactly +// the carrier and drop exactly the plain grant in default mode, and must +// hard-fail (dropping nothing) in fail-fast mode. This is the fixture +// shape that homogeneous exemption tests structurally cannot check — the +// escape route the per-entitlement-resource I8 exemption granularity bug +// took. + +import ( + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + reader_v2 "github.com/conductorone/baton-sdk/pb/c1/reader/v2" + "github.com/conductorone/baton-sdk/pkg/annotations" + "github.com/conductorone/baton-sdk/pkg/connectorstore" + "github.com/conductorone/baton-sdk/pkg/logging" + et "github.com/conductorone/baton-sdk/pkg/types/entitlement" + "github.com/conductorone/baton-sdk/pkg/types/grant" + rs "github.com/conductorone/baton-sdk/pkg/types/resource" +) + +func TestIngestInvariantI9MixedPrincipalPopulation(t *testing.T) { + ctx, err := logging.Init(t.Context()) + require.NoError(t, err) + tmpDir := t.TempDir() + + repo, err := rs.NewResource("Repo r1", equivRepoRT, "r1") + require.NoError(t, err) + adminEnt := et.NewAssignmentEntitlement(repo, "admin") + viewerEnt := et.NewAssignmentEntitlement(repo, "viewer") + ghost := v2.Resource_builder{ + Id: v2.ResourceId_builder{ResourceType: "user", Resource: "ghost"}.Build(), + }.Build() + + // The SAME dangling principal carries one match carrier and one + // plain grant. + carrier := grant.NewGrant(repo, "admin", ghost.GetId()) + annos := annotations.Annotations(carrier.GetAnnotations()) + annos.Update(v2.ExternalResourceMatch_builder{Key: "email", Value: "ghost@example.com"}.Build()) + carrier.SetAnnotations(annos) + plain := grant.NewGrant(repo, "viewer", ghost.GetId()) + + cur := newRepairTestStore(ctx, t, filepath.Join(tmpDir, "cur.c1z"), tmpDir) + _, err = cur.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + require.NoError(t, cur.PutResourceTypes(ctx, equivRepoRT)) + require.NoError(t, cur.PutResources(ctx, repo)) + require.NoError(t, cur.PutEntitlements(ctx, adminEnt, viewerEnt)) + require.NoError(t, cur.PutGrants(ctx, carrier, plain)) + + s := &syncer{store: cur, syncType: connectorstore.SyncTypeFull} + + requireGrant := func(id string, wantPresent bool, msg string) { + t.Helper() + _, err := cur.GetGrant(ctx, reader_v2.GrantsReaderServiceGetGrantRequest_builder{ + GrantId: id, + }.Build()) + if wantPresent { + require.NoError(t, err, msg) + } else { + require.Error(t, err, msg) + } + } + + // Fail-fast: a mixed principal is NOT the exempt shape — it must be + // named as a violation, and nothing may be dropped. + s.failFastInvariants = true + err = s.checkGrantPrincipalReferences(ctx) + require.Error(t, err) + require.Contains(t, err.Error(), "ingest invariant I9") + requireGrant(carrier.GetId(), true, "fail-fast must not drop the carrier") + requireGrant(plain.GetId(), true, "fail-fast must not drop the plain grant") + + // Default mode: per-grant judgment — the carrier survives, the plain + // grant is dropped. + s.failFastInvariants = false + require.NoError(t, s.checkGrantPrincipalReferences(ctx)) + requireGrant(carrier.GetId(), true, "the match carrier must be kept (config-gap evidence)") + requireGrant(plain.GetId(), false, "the plain grant under the dangling principal must be dropped") + + // After the drop, the principal's surviving population is + // carriers-only — the exempt shape — so BOTH modes now pass without + // further mutation (idempotence of the mixed case). + require.NoError(t, s.checkGrantPrincipalReferences(ctx)) + s.failFastInvariants = true + require.NoError(t, s.checkGrantPrincipalReferences(ctx)) + requireGrant(carrier.GetId(), true, "the carrier must survive repeated passes") + + require.NoError(t, cur.Close(ctx)) +} diff --git a/pkg/sync/ingest_invariants_probe_test.go b/pkg/sync/ingest_invariants_probe_test.go new file mode 100644 index 000000000..76b02f94e --- /dev/null +++ b/pkg/sync/ingest_invariants_probe_test.go @@ -0,0 +1,229 @@ +package sync //nolint:revive,nolintlint // we can't change the package name for backwards compatibility + +// Pins the dangling-type probe's error handling: a resource-type read +// failure during the I7/I8/I9 sweep must FAIL the check, never +// masquerade as "type never synced" — that verdict picks the DROP arm, +// so an IO error or canceled context would seal a sanitized artifact +// where the replay policy demands a loud failure. + +import ( + "context" + "errors" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + reader_v2 "github.com/conductorone/baton-sdk/pb/c1/reader/v2" + "github.com/conductorone/baton-sdk/pkg/annotations" + "github.com/conductorone/baton-sdk/pkg/connectorstore" + "github.com/conductorone/baton-sdk/pkg/dotc1z" + "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" + "github.com/conductorone/baton-sdk/pkg/logging" + "github.com/conductorone/baton-sdk/pkg/sourcecache" + et "github.com/conductorone/baton-sdk/pkg/types/entitlement" + "github.com/conductorone/baton-sdk/pkg/types/grant" + rs "github.com/conductorone/baton-sdk/pkg/types/resource" +) + +// typeProbeFaultStore wraps a real store and fails every GetResourceType +// with a chosen non-NotFound error, leaving the rest of the invariant +// surface (HasResourceRecord, the sweep iterators, the delete paths) +// intact — the exact seam the danglingTypeProbe reads. +type typeProbeFaultStore struct { + c1zstore.Store + dotc1z.IngestInvariantStore + err error +} + +func (s typeProbeFaultStore) GetResourceType( + context.Context, *reader_v2.ResourceTypesReaderServiceGetResourceTypeRequest, +) (*reader_v2.ResourceTypesReaderServiceGetResourceTypeResponse, error) { + return nil, s.err +} + +func TestDanglingTypeProbeErrorFailsInsteadOfDropping(t *testing.T) { + ctx, err := logging.Init(t.Context()) + require.NoError(t, err) + tmpDir := t.TempDir() + + repo, err := rs.NewResource("Repo r1", equivRepoRT, "r1") + require.NoError(t, err) + ent := et.NewAssignmentEntitlement(repo, "admin") + + // A dangling entitlement (no resource row, no type row): with a + // healthy probe this is a disabled-type gap and gets DROPPED. With + // the probe FAILING, the sweep must fail — before the fix, the + // probe's error was collapsed into "type never synced" and the row + // was silently dropped. + cur := newRepairTestStore(ctx, t, filepath.Join(tmpDir, "cur.c1z"), tmpDir) + _, err = cur.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + require.NoError(t, cur.PutEntitlements(ctx, ent)) + + inv, ok := cur.(dotc1z.IngestInvariantStore) + require.True(t, ok) + probeFault := errors.New("injected: resource-type read failed") + s := &syncer{ + store: typeProbeFaultStore{Store: cur, IngestInvariantStore: inv, err: probeFault}, + syncType: connectorstore.SyncTypeFull, + } + + err = s.checkEntitlementResourceReferences(ctx) + require.Error(t, err, "a probe read failure must fail the sweep, not pick a drop verdict") + require.ErrorIs(t, err, probeFault) + + _, err = cur.GetEntitlement(ctx, reader_v2.EntitlementsReaderServiceGetEntitlementRequest_builder{ + EntitlementId: ent.GetId(), + }.Build()) + require.NoError(t, err, "no row may be dropped on a probe failure") + + // Same seam for I8: a grant whose entitlement has no row. + err = s.checkGrantEntitlementReferences(ctx) + require.NoError(t, err, "no dangling grants planted; I8 must pass without probing") + + require.NoError(t, cur.Close(ctx)) +} + +// TestExclusionGroupVerdictRidesWarmColdLadder pins I5's membership in +// the warm/cold ErrReplayIntegrity ladder: replayed stale entitlement +// rows can manufacture warm-only exclusion-group conflicts (a stale +// group member beside a fresh one), so a warm verdict must carry the +// sentinel (runners discard + retry cold) while a cold verdict is +// attributed plainly to the connector — the same classification as +// I3/I6/enabled I7–I9. +func TestExclusionGroupVerdictRidesWarmColdLadder(t *testing.T) { + ctx, err := logging.Init(t.Context()) + require.NoError(t, err) + tmpDir := t.TempDir() + + group := v2.EntitlementExclusionGroup_builder{ExclusionGroupId: "eg-1", IsDefault: true}.Build() + res := v2.Resource_builder{ + Id: v2.ResourceId_builder{ResourceType: "group", Resource: "g1"}.Build(), + }.Build() + // Two defaults in one group: an I5 violation however the rows arrived. + ent1 := v2.Entitlement_builder{Id: "group:g1:member", Resource: res, Annotations: annotations.New(group)}.Build() + ent2 := v2.Entitlement_builder{Id: "group:g1:owner", Resource: res, Annotations: annotations.New(group)}.Build() + + cur := newRepairTestStore(ctx, t, filepath.Join(tmpDir, "cur.c1z"), tmpDir) + _, err = cur.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + require.NoError(t, cur.PutEntitlements(ctx, ent1, ent2)) + + s := &syncer{store: cur, syncType: connectorstore.SyncTypeFull} + + // Cold: plain connector attribution, no sentinel. + err = s.validateStoredExclusionGroups(ctx) + require.Error(t, err) + require.Contains(t, err.Error(), "ingest invariant I5") + require.Contains(t, err.Error(), "multiple default entitlements") + require.NotErrorIs(t, err, ErrReplayIntegrity, "a cold verdict is terminal, not a replay fault") + + // Warm: the same evidence may be replay-carried staleness. + s.sourceCache.prev = struct{ dotc1z.SourceCacheStore }{} + err = s.validateStoredExclusionGroups(ctx) + require.Error(t, err) + require.ErrorIs(t, err, ErrReplayIntegrity, "a warm verdict must ride the discard-and-retry-cold ladder") + + require.NoError(t, cur.Close(ctx)) +} + +// TestFailFastPromotesI3UnannotatedWarn pins the fail-fast contract for +// I3's tolerated arm: production WARNS on grants referencing a +// never-synced entitlement resource without InsertResourceGrants +// (tolerated connector behavior), but fail-fast mode — the harness's +// whole mechanism for catching engineered violations — must promote +// that warn to a hard failure, or the harness silently passes connector +// shapes production would flag. +func TestFailFastPromotesI3UnannotatedWarn(t *testing.T) { + ctx, err := logging.Init(t.Context()) + require.NoError(t, err) + tmpDir := t.TempDir() + + repo, err := rs.NewResource("Repo r1", equivRepoRT, "r1") + require.NoError(t, err) + principal := v2.Resource_builder{ + Id: v2.ResourceId_builder{ResourceType: "user", Resource: "alice"}.Build(), + }.Build() + g := grant.NewGrant(repo, "admin", principal.GetId()) + + // The tolerated shape: a grant whose entitlement RESOURCE has no row + // and no InsertResourceGrants annotation. + cur := newRepairTestStore(ctx, t, filepath.Join(tmpDir, "cur.c1z"), tmpDir) + _, err = cur.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + require.NoError(t, cur.PutGrants(ctx, g)) + + // Default mode: warn only, no error (production tolerance). + s := &syncer{store: cur, syncType: connectorstore.SyncTypeFull} + require.NoError(t, s.checkGrantResourceReferences(ctx), + "default mode tolerates unannotated danglings with a warning") + + // Fail-fast: the warn must become a named failure. + s.failFastInvariants = true + err = s.checkGrantResourceReferences(ctx) + require.Error(t, err, "fail-fast must promote the tolerated warn to a hard failure") + require.Contains(t, err.Error(), "ingest invariant I3") + require.NotErrorIs(t, err, ErrReplayIntegrity, "fail-fast verdicts are plain, matching I7-I9") + + require.NoError(t, cur.Close(ctx)) +} + +// TestPartialSyncSkipsStoredInvariants pins the partial-sync gate for +// I5 and I6, matching I3/I7/I8/I9: a partial sync writes a deliberate +// subset, so store-derived verdicts computed over it are not +// evaluable — a partial must neither hard-fail on exclusion-group +// evidence nor on orphan-scope evidence while the referential arms are +// inert. +func TestPartialSyncSkipsStoredInvariants(t *testing.T) { + ctx, err := logging.Init(t.Context()) + require.NoError(t, err) + tmpDir := t.TempDir() + + group := v2.EntitlementExclusionGroup_builder{ExclusionGroupId: "eg-1", IsDefault: true}.Build() + res := v2.Resource_builder{ + Id: v2.ResourceId_builder{ResourceType: "group", Resource: "g1"}.Build(), + }.Build() + ent1 := v2.Entitlement_builder{Id: "group:g1:member", Resource: res, Annotations: annotations.New(group)}.Build() + ent2 := v2.Entitlement_builder{Id: "group:g1:owner", Resource: res, Annotations: annotations.New(group)}.Build() + + cur := newRepairTestStore(ctx, t, filepath.Join(tmpDir, "cur.c1z"), tmpDir) + _, err = cur.StartNewSync(ctx, connectorstore.SyncTypePartial, "") + require.NoError(t, err) + require.NoError(t, cur.PutEntitlements(ctx, ent1, ent2)) + // An orphan scope shape: stamped rows, no manifest entry. + repo, err := rs.NewResource("Repo r1", equivRepoRT, "r1") + require.NoError(t, err) + principal := v2.Resource_builder{ + Id: v2.ResourceId_builder{ResourceType: "user", Resource: "alice"}.Build(), + }.Build() + require.NoError(t, cur.PutGrants(sourcecache.WithScope(ctx, "scope-partial"), grant.NewGrant(repo, "admin", principal.GetId()))) + + s := &syncer{store: cur, syncType: connectorstore.SyncTypePartial} + require.NoError(t, s.validateStoredExclusionGroups(ctx), + "I5 must not run on a partial sync (subset store; referential arms are inert too)") + require.NoError(t, s.checkSourceCacheScopeConsistency(ctx), + "I6 must not run on a partial sync") + + // The same evidence on a FULL sync still fails both. + s.syncType = connectorstore.SyncTypeFull + require.Error(t, s.validateStoredExclusionGroups(ctx)) + require.Error(t, s.checkSourceCacheScopeConsistency(ctx)) + + require.NoError(t, cur.Close(ctx)) +} + +// TestDanglingAttributionBuckets pins the drop-arm attribution wording: +// the drop arm only ever sees config gaps and compaction-manufactured +// gaps (enabled-type danglings in a normal sync FAIL before any drop), +// so no drop warning may blame connector magic-id construction. +func TestDanglingAttributionBuckets(t *testing.T) { + require.Contains(t, danglingAttribution(3, 0), "config gap") + require.Contains(t, danglingAttribution(0, 2), "compaction") + require.Contains(t, danglingAttribution(1, 1), "mixed") + for _, s := range []string{danglingAttribution(3, 0), danglingAttribution(0, 2), danglingAttribution(1, 1)} { + require.NotContains(t, s, "magic-id", + "drop-arm attribution must not blame connector id construction — those cases fail the sync instead") + } +} diff --git a/pkg/sync/ingest_invariants_referential_test.go b/pkg/sync/ingest_invariants_referential_test.go new file mode 100644 index 000000000..9373cae08 --- /dev/null +++ b/pkg/sync/ingest_invariants_referential_test.go @@ -0,0 +1,425 @@ +package sync //nolint:revive,nolintlint // we can't change the package name for backwards compatibility + +// I7/I8/I9 referential-invariant tests, under the replay policy's +// attribution split: +// +// - DISABLED-TYPE gaps (the referent's resource type was never +// synced): expected configuration shapes. Default mode DROPS the +// dependent rows with one aggregated warning and seals a +// referentially consistent artifact; fail-fast mode fails. The drop +// fixtures below deliberately omit the referent's type row. +// - ENABLED-TYPE danglings (the type IS synced, the row is not): +// never sealed away. Warm syncs fail with ErrReplayIntegrity +// (discard + cold retry); cold syncs fail plainly (connector +// data/behavior). See the *_EnabledTypeFails tests. + +import ( + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + reader_v2 "github.com/conductorone/baton-sdk/pb/c1/reader/v2" + "github.com/conductorone/baton-sdk/pkg/annotations" + "github.com/conductorone/baton-sdk/pkg/connectorstore" + "github.com/conductorone/baton-sdk/pkg/dotc1z" + "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" + "github.com/conductorone/baton-sdk/pkg/logging" + et "github.com/conductorone/baton-sdk/pkg/types/entitlement" + "github.com/conductorone/baton-sdk/pkg/types/grant" + rs "github.com/conductorone/baton-sdk/pkg/types/resource" +) + +func TestIngestInvariantI7DropsDanglingEntitlements(t *testing.T) { + ctx, err := logging.Init(t.Context()) + require.NoError(t, err) + tmpDir := t.TempDir() + + repo, err := rs.NewResource("Repo r1", equivRepoRT, "r1") + require.NoError(t, err) + ent := et.NewAssignmentEntitlement(repo, "admin") + + // The violating shape: an entitlement row whose resource has no row — + // and no TYPE row either (disabled-type gap: the drop arm). + cur := newRepairTestStore(ctx, t, filepath.Join(tmpDir, "cur.c1z"), tmpDir) + _, err = cur.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + require.NoError(t, cur.PutEntitlements(ctx, ent)) + + s := &syncer{store: cur, syncType: connectorstore.SyncTypeFull} + + // Fail-fast: named failure, no sentinel, no drop. + s.failFastInvariants = true + err = s.checkEntitlementResourceReferences(ctx) + require.Error(t, err) + require.Contains(t, err.Error(), "ingest invariant I7") + require.NotErrorIs(t, err, ErrReplayIntegrity) + _, err = cur.GetEntitlement(ctx, reader_v2.EntitlementsReaderServiceGetEntitlementRequest_builder{ + EntitlementId: ent.GetId(), + }.Build()) + require.NoError(t, err, "fail-fast mode must not drop rows") + + // Default mode: the row is DROPPED, and the pass is idempotent. + s.failFastInvariants = false + require.NoError(t, s.checkEntitlementResourceReferences(ctx)) + _, err = cur.GetEntitlement(ctx, reader_v2.EntitlementsReaderServiceGetEntitlementRequest_builder{ + EntitlementId: ent.GetId(), + }.Build()) + require.Error(t, err, "default mode must drop the dangling entitlement row") + require.NoError(t, s.checkEntitlementResourceReferences(ctx)) + + // After the drop, fail-fast finds nothing. + s.failFastInvariants = true + require.NoError(t, s.checkEntitlementResourceReferences(ctx)) + + require.NoError(t, cur.Close(ctx)) +} + +func TestIngestInvariantI8DropsDanglingGrants(t *testing.T) { + ctx, err := logging.Init(t.Context()) + require.NoError(t, err) + tmpDir := t.TempDir() + + repo, err := rs.NewResource("Repo r1", equivRepoRT, "r1") + require.NoError(t, err) + ent := et.NewAssignmentEntitlement(repo, "admin") + principal := v2.Resource_builder{ + Id: v2.ResourceId_builder{ResourceType: "user", Resource: "alice"}.Build(), + }.Build() + g := grant.NewGrant(repo, "admin", principal.GetId()) + + // The violating shape: a grant row whose entitlement has no row — + // and no TYPE row for the entitlement's resource type (disabled-type + // gap: the drop arm). + cur := newRepairTestStore(ctx, t, filepath.Join(tmpDir, "cur.c1z"), tmpDir) + _, err = cur.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + require.NoError(t, cur.PutResources(ctx, repo)) + require.NoError(t, cur.PutGrants(ctx, g)) + + s := &syncer{store: cur, syncType: connectorstore.SyncTypeFull} + + // Fail-fast: named failure carrying the exact reconstructed id; no drop. + s.failFastInvariants = true + err = s.checkGrantEntitlementReferences(ctx) + require.Error(t, err) + require.Contains(t, err.Error(), "ingest invariant I8") + require.Contains(t, err.Error(), ent.GetId()) + require.NotErrorIs(t, err, ErrReplayIntegrity) + + // Default mode: the grant is DROPPED; the pass is idempotent; fail-fast then + // finds nothing. + s.failFastInvariants = false + require.NoError(t, s.checkGrantEntitlementReferences(ctx)) + _, err = cur.GetGrant(ctx, reader_v2.GrantsReaderServiceGetGrantRequest_builder{ + GrantId: g.GetId(), + }.Build()) + require.Error(t, err, "default mode must drop the dangling grant row") + require.NoError(t, s.checkGrantEntitlementReferences(ctx)) + s.failFastInvariants = true + require.NoError(t, s.checkGrantEntitlementReferences(ctx)) + + require.NoError(t, cur.Close(ctx)) +} + +func TestIngestInvariantI8ExemptsInsertResourceGrants(t *testing.T) { + ctx, err := logging.Init(t.Context()) + require.NoError(t, err) + tmpDir := t.TempDir() + + repo, err := rs.NewResource("Repo r1", equivRepoRT, "r1") + require.NoError(t, err) + g := repairTestGrant(t, repo) // carries InsertResourceGrants + + cur := newRepairTestStore(ctx, t, filepath.Join(tmpDir, "cur.c1z"), tmpDir) + _, err = cur.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + require.NoError(t, cur.PutResourceTypes(ctx, equivRepoRT)) + require.NoError(t, cur.PutResources(ctx, repo)) + require.NoError(t, cur.PutGrants(ctx, g)) + + s := &syncer{store: cur, syncType: connectorstore.SyncTypeFull} + + // Exempt in both modes: no failure, no drop. + s.failFastInvariants = true + require.NoError(t, s.checkGrantEntitlementReferences(ctx)) + s.failFastInvariants = false + require.NoError(t, s.checkGrantEntitlementReferences(ctx)) + _, err = cur.GetGrant(ctx, reader_v2.GrantsReaderServiceGetGrantRequest_builder{ + GrantId: g.GetId(), + }.Build()) + require.NoError(t, err, "InsertResourceGrants shape must never be dropped") + + require.NoError(t, cur.Close(ctx)) +} + +func TestIngestInvariantI9DropsDanglingPrincipalGrants(t *testing.T) { + ctx, err := logging.Init(t.Context()) + require.NoError(t, err) + tmpDir := t.TempDir() + + repo, err := rs.NewResource("Repo r1", equivRepoRT, "r1") + require.NoError(t, err) + ent := et.NewAssignmentEntitlement(repo, "admin") + ghost := v2.Resource_builder{ + Id: v2.ResourceId_builder{ResourceType: "user", Resource: "ghost"}.Build(), + }.Build() + g := grant.NewGrant(repo, "admin", ghost.GetId()) + + // The violating shape: resource + entitlement exist, principal not. + cur := newRepairTestStore(ctx, t, filepath.Join(tmpDir, "cur.c1z"), tmpDir) + _, err = cur.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + require.NoError(t, cur.PutResourceTypes(ctx, equivRepoRT)) + require.NoError(t, cur.PutResources(ctx, repo)) + require.NoError(t, cur.PutEntitlements(ctx, ent)) + require.NoError(t, cur.PutGrants(ctx, g)) + + s := &syncer{store: cur, syncType: connectorstore.SyncTypeFull} + + // Fail-fast: named failure, no drop. + s.failFastInvariants = true + err = s.checkGrantPrincipalReferences(ctx) + require.Error(t, err) + require.Contains(t, err.Error(), "ingest invariant I9") + require.Contains(t, err.Error(), "user/ghost") + require.NotErrorIs(t, err, ErrReplayIntegrity) + + // Default mode: the grant is DROPPED; the pass is idempotent; fail-fast then + // finds nothing. + s.failFastInvariants = false + require.NoError(t, s.checkGrantPrincipalReferences(ctx)) + _, err = cur.GetGrant(ctx, reader_v2.GrantsReaderServiceGetGrantRequest_builder{ + GrantId: g.GetId(), + }.Build()) + require.Error(t, err, "default mode must drop the dangling-principal grant") + require.NoError(t, s.checkGrantPrincipalReferences(ctx)) + s.failFastInvariants = true + require.NoError(t, s.checkGrantPrincipalReferences(ctx)) + + require.NoError(t, cur.Close(ctx)) +} + +func TestIngestInvariantI9ExemptsExternalMatchCarriers(t *testing.T) { + ctx, err := logging.Init(t.Context()) + require.NoError(t, err) + tmpDir := t.TempDir() + + repo, err := rs.NewResource("Repo r1", equivRepoRT, "r1") + require.NoError(t, err) + ent := et.NewAssignmentEntitlement(repo, "admin") + viewerEnt := et.NewAssignmentEntitlement(repo, "viewer") + carrier := v2.Resource_builder{ + Id: v2.ResourceId_builder{ResourceType: "external-user", Resource: "someone@example.com"}.Build(), + }.Build() + // Two carrier grants for the SAME dangling principal, so the + // carrier-grant count is distinguishable from the principal count. + newCarrierGrant := func(slug string) *v2.Grant { + g := grant.NewGrant(repo, slug, carrier.GetId()) + annos := annotations.Annotations(g.GetAnnotations()) + annos.Update(v2.ExternalResourceMatch_builder{Key: "email", Value: "someone@example.com"}.Build()) + g.SetAnnotations(annos) + return g + } + g := newCarrierGrant("admin") + g2 := newCarrierGrant("viewer") + + // The carrier shape: a match-annotated grant whose stub principal has + // no row because no external resource file was configured (the match + // op never ran to transform-and-delete it). + cur := newRepairTestStore(ctx, t, filepath.Join(tmpDir, "cur.c1z"), tmpDir) + _, err = cur.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + require.NoError(t, cur.PutResourceTypes(ctx, equivRepoRT)) + require.NoError(t, cur.PutResources(ctx, repo)) + require.NoError(t, cur.PutEntitlements(ctx, ent, viewerEnt)) + require.NoError(t, cur.PutGrants(ctx, g, g2)) + + s := &syncer{store: cur, syncType: connectorstore.SyncTypeFull} + + // Exempt in both modes: evidence of a config gap, not bad data. + s.failFastInvariants = true + require.NoError(t, s.checkGrantPrincipalReferences(ctx)) + s.failFastInvariants = false + require.NoError(t, s.checkGrantPrincipalReferences(ctx)) + for _, kept := range []string{g.GetId(), g2.GetId()} { + _, err = cur.GetGrant(ctx, reader_v2.GrantsReaderServiceGetGrantRequest_builder{ + GrantId: kept, + }.Build()) + require.NoError(t, err, "unprocessed external-match carriers must never be dropped") + } + + // The carrier count is per GRANT, not per principal: one dangling + // principal with two carrier grants must report carrierGrants == 2 + // (units consistent with the mixed-principal path's skip count). + facts, ok := cur.(dotc1z.IngestInvariantStore) + require.True(t, ok) + require.NoError(t, facts.EnsureGrantIndexes(ctx)) + visits := 0 + require.NoError(t, facts.ForEachDanglingGrantPrincipal(ctx, func(rt, rid string, matchOnly bool, carrierGrants int64) error { + visits++ + require.True(t, matchOnly) + require.Equal(t, int64(2), carrierGrants) + return nil + })) + require.Equal(t, 1, visits, "one dangling principal expected") + + require.NoError(t, cur.Close(ctx)) +} + +// TestDishonestValidatorStrandsGrantEntitlements is the end-to-end I8 +// scenario: a connector whose grants validator says "unchanged" (so the +// scope replays) while its entitlement listing was refreshed and dropped +// the entitlement. The stranded grants dangle under an ENABLED type, so +// the replay policy applies: the warm output is DISCARDED +// (ErrReplayIntegrity — the runners' cold-retry ladder) and the cold +// re-run, whose fresh fetch has no stranded grants, seals clean. The +// artifact is never silently sanitized. +func TestDishonestValidatorStrandsGrantEntitlements(t *testing.T) { + ctx, err := logging.Init(t.Context()) + require.NoError(t, err) + tmpDir := t.TempDir() + + mc := newSourceCacheMockConnector() + group, memberEnt, err := mc.AddGroup(ctx, "g1") + require.NoError(t, err) + user, err := mc.AddUser(ctx, "u1") + require.NoError(t, err) + stranded := mc.AddGroupMember(ctx, group, user) + mc.etagByResource[group.GetId().GetResource()] = "v1" + + sync1 := filepath.Join(tmpDir, "s1.c1z") + runSourceCacheSync(ctx, t, mc, sync1, "", tmpDir) + + // The dishonesty: the entitlement listing drops the member + // entitlement while the grants validator still reports "unchanged". + // The membership grants also vanish upstream (the fresh fetch has + // none) — only the REPLAY carries the stranded grants forward. + mc.entDB[group.GetId().GetResource()] = nil + mc.grantDB[group.GetId().GetResource()] = nil + + // Warm sync (default mode): the stranded grants dangle under the + // SYNCED group type — replay is implicated, the output must be + // discarded and retried cold, never sealed sanitized. + store2, err := dotc1z.NewStore(ctx, filepath.Join(tmpDir, "s2.c1z"), + dotc1z.WithEngine(c1zstore.EnginePebble), + dotc1z.WithTmpDir(tmpDir), + ) + require.NoError(t, err) + warm, err := NewSyncer(ctx, mc, + WithConnectorStore(store2), + WithTmpDir(tmpDir), + WithPreviousSyncC1ZPath(sync1), + ) + require.NoError(t, err) + err = warm.Sync(ctx) + require.Error(t, err) + require.Contains(t, err.Error(), "ingest invariant I8") + require.ErrorIs(t, err, ErrReplayIntegrity, + "stranded grants under an enabled type on a warm sync must route to the cold-retry ladder") + require.NoError(t, warm.Close(ctx)) + + // The cold retry (what the runners do with ErrReplayIntegrity): + // clean — the fresh fetch never had the stranded grants. + sync3 := filepath.Join(tmpDir, "s3.c1z") + runSourceCacheSync(ctx, t, mc, sync3, "", tmpDir) + grants3 := listGrantsInFile(ctx, t, sync3) + require.NotContains(t, grants3, stranded.GetId(), + "the cold retry's artifact must not contain the stranded grant") + _ = memberEnt +} + +// TestIngestInvariantEnabledTypeDanglingsFail pins the replay policy's +// strict arm for all three referential invariants: a missing referent +// whose resource TYPE is synced is never sealed away. Cold syncs fail +// plainly (connector data/behavior — the retry could not help); warm +// syncs fail with ErrReplayIntegrity (discard + cold retry); the +// compaction expand pass is exempt (keep-newer merges legitimately +// manufacture dangling refs, and its drop pass converges the artifact). +func TestIngestInvariantEnabledTypeDanglingsFail(t *testing.T) { + ctx, err := logging.Init(t.Context()) + require.NoError(t, err) + tmpDir := t.TempDir() + + repo, err := rs.NewResource("Repo r1", equivRepoRT, "r1") + require.NoError(t, err) + ent := et.NewAssignmentEntitlement(repo, "admin") + userRT := v2.ResourceType_builder{Id: "user", DisplayName: "User"}.Build() + ghost := v2.Resource_builder{ + Id: v2.ResourceId_builder{ResourceType: "user", Resource: "ghost"}.Build(), + }.Build() + + // All three shapes, every referent under a SYNCED type: an + // entitlement whose resource row is missing (I7), a grant whose + // entitlement row is missing (I8), and a grant whose principal row + // is missing (I9). + cur := newRepairTestStore(ctx, t, filepath.Join(tmpDir, "cur.c1z"), tmpDir) + _, err = cur.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + require.NoError(t, cur.PutResourceTypes(ctx, equivRepoRT, userRT)) + + // I7 shape: entitlement row, no resource row, type row present. + require.NoError(t, cur.PutEntitlements(ctx, ent)) + + s := &syncer{store: cur, syncType: connectorstore.SyncTypeFull} + + // COLD: plain failure, no sentinel, nothing dropped. + err = s.checkEntitlementResourceReferences(ctx) + require.Error(t, err) + require.Contains(t, err.Error(), "ingest invariant I7") + require.Contains(t, err.Error(), "SYNCED type") + require.NotErrorIs(t, err, ErrReplayIntegrity) + _, err = cur.GetEntitlement(ctx, reader_v2.EntitlementsReaderServiceGetEntitlementRequest_builder{ + EntitlementId: ent.GetId(), + }.Build()) + require.NoError(t, err, "an enabled-type dangling must never be dropped") + + // WARM: same evidence carries the replay-integrity sentinel. + s.sourceCache.prev = struct{ dotc1z.SourceCacheStore }{} + err = s.checkEntitlementResourceReferences(ctx) + require.ErrorIs(t, err, ErrReplayIntegrity) + s.sourceCache.prev = nil + + // COMPACTION EXPAND PASS: exempt — the drop arm converges the + // merged artifact (keep-newer merges manufacture these by design). + s.onlyExpandGrants = true + require.NoError(t, s.checkEntitlementResourceReferences(ctx), + "the compaction expand pass must drop, not fail: merges manufacture dangling refs no input contained") + s.onlyExpandGrants = false + + // I8 + I9 shapes: resource row present now; a grant under a missing + // entitlement (enabled repo type) and a grant to a missing principal + // (enabled user type). + require.NoError(t, cur.PutResources(ctx, repo)) + gEnt := grant.NewGrant(repo, "missing-ent", ghost.GetId()) + require.NoError(t, cur.PutGrants(ctx, gEnt)) + + err = s.checkGrantEntitlementReferences(ctx) + require.Error(t, err) + require.Contains(t, err.Error(), "ingest invariant I8") + require.Contains(t, err.Error(), "SYNCED type") + require.NotErrorIs(t, err, ErrReplayIntegrity) + s.sourceCache.prev = struct{ dotc1z.SourceCacheStore }{} + require.ErrorIs(t, s.checkGrantEntitlementReferences(ctx), ErrReplayIntegrity) + s.sourceCache.prev = nil + + // Heal I8's shape (put its entitlement) so I9 is isolated: the + // grant's principal "user/ghost" has a type row but no resource row. + missingEnt := et.NewAssignmentEntitlement(repo, "missing-ent") + require.NoError(t, cur.PutEntitlements(ctx, missingEnt)) + err = s.checkGrantPrincipalReferences(ctx) + require.Error(t, err) + require.Contains(t, err.Error(), "ingest invariant I9") + require.Contains(t, err.Error(), "SYNCED type") + require.NotErrorIs(t, err, ErrReplayIntegrity) + s.sourceCache.prev = struct{ dotc1z.SourceCacheStore }{} + require.ErrorIs(t, s.checkGrantPrincipalReferences(ctx), ErrReplayIntegrity) + s.sourceCache.prev = nil + _, err = cur.GetGrant(ctx, reader_v2.GrantsReaderServiceGetGrantRequest_builder{ + GrantId: gEnt.GetId(), + }.Build()) + require.NoError(t, err, "enabled-type danglings must never be dropped in any failing mode") + + require.NoError(t, cur.Close(ctx)) +} diff --git a/pkg/sync/ingest_invariants_repair_test.go b/pkg/sync/ingest_invariants_repair_test.go new file mode 100644 index 000000000..8a0e06eb3 --- /dev/null +++ b/pkg/sync/ingest_invariants_repair_test.go @@ -0,0 +1,234 @@ +package sync //nolint:revive,nolintlint // we can't change the package name for backwards compatibility + +// I3 repair-arm tests: a grant-inserted resource lost from the current +// sync (the replay-loss class) is repaired by copying the previous +// sync's row in default mode, and hard-fails under ErrReplayIntegrity in +// fail-fast mode (so mutation tests still name the machinery bug) or when +// no previous source can supply the row. + +import ( + "context" + "fmt" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + reader_v2 "github.com/conductorone/baton-sdk/pb/c1/reader/v2" + "github.com/conductorone/baton-sdk/pkg/annotations" + "github.com/conductorone/baton-sdk/pkg/connectorstore" + "github.com/conductorone/baton-sdk/pkg/dotc1z" + "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" + "github.com/conductorone/baton-sdk/pkg/logging" + "github.com/conductorone/baton-sdk/pkg/types/grant" + rs "github.com/conductorone/baton-sdk/pkg/types/resource" +) + +func repairTestGrant(t *testing.T, repo *v2.Resource) *v2.Grant { + t.Helper() + principal := v2.Resource_builder{ + Id: v2.ResourceId_builder{ResourceType: "user", Resource: "alice"}.Build(), + }.Build() + g := grant.NewGrant(repo, "admin", principal.GetId()) + annos := annotations.Annotations(g.GetAnnotations()) + annos.Update(&v2.InsertResourceGrants{}) + g.SetAnnotations(annos) + return g +} + +func newRepairTestStore(ctx context.Context, t *testing.T, path, tmpDir string) c1zstore.Store { + t.Helper() + store, err := dotc1z.NewStore(ctx, path, + dotc1z.WithEngine(c1zstore.EnginePebble), + dotc1z.WithTmpDir(tmpDir), + ) + require.NoError(t, err) + return store +} + +func TestIngestInvariantI3RepairsLostRelatedResource(t *testing.T) { + ctx, err := logging.Init(t.Context()) + require.NoError(t, err) + tmpDir := t.TempDir() + + repo, err := rs.NewResource("Repo r1", equivRepoRT, "r1") + require.NoError(t, err) + g := repairTestGrant(t, repo) + + // Previous sync: healthy state — annotated grant AND its resource. + prev := newRepairTestStore(ctx, t, filepath.Join(tmpDir, "prev.c1z"), tmpDir) + _, err = prev.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + require.NoError(t, prev.PutResourceTypes(ctx, equivRepoRT)) + require.NoError(t, prev.PutResources(ctx, repo)) + require.NoError(t, prev.PutGrants(ctx, g)) + require.NoError(t, prev.EndSync(ctx)) + + // Current sync: the replay-loss shape — annotated grant, NO resource. + cur := newRepairTestStore(ctx, t, filepath.Join(tmpDir, "cur.c1z"), tmpDir) + _, err = cur.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + require.NoError(t, cur.PutResourceTypes(ctx, equivRepoRT)) + require.NoError(t, cur.PutGrants(ctx, g)) + + s := &syncer{store: cur, syncType: connectorstore.SyncTypeFull} + prevSC, ok := any(prev).(dotc1z.SourceCacheStore) + require.True(t, ok) + s.sourceCache.prev = prevSC + s.sourceCache.prevReader = prev + + // Fail-fast mode: no repair — the machinery bug must be NAMED, and the + // failure classified as replay-integrity. + s.failFastInvariants = true + err = s.checkGrantResourceReferences(ctx) + require.Error(t, err) + require.ErrorIs(t, err, ErrReplayIntegrity) + + // Default mode: repaired from the previous sync's row, full fidelity. + s.failFastInvariants = false + require.NoError(t, s.checkGrantResourceReferences(ctx)) + got, err := cur.GetResource(ctx, reader_v2.ResourcesReaderServiceGetResourceRequest_builder{ + ResourceId: repo.GetId(), + }.Build()) + require.NoError(t, err) + require.Equal(t, "Repo r1", got.GetResource().GetDisplayName(), + "repair must restore the previous sync's full-fidelity row, not a stub") + + // Idempotent: a second pass finds nothing dangling. + require.NoError(t, s.checkGrantResourceReferences(ctx)) + + require.NoError(t, cur.Close(ctx)) + require.NoError(t, prev.Close(ctx)) +} + +// TestReplayIntegrityErrorPropagatesFromSync pins the propagation the +// runners' cold-retry depends on: an ErrReplayIntegrity-classified +// failure raised at the replay seam must survive the action loop and +// parallel workers and satisfy errors.Is at syncer.Sync()'s return. +func TestReplayIntegrityErrorPropagatesFromSync(t *testing.T) { + ctx, err := logging.Init(t.Context()) + require.NoError(t, err) + tmpDir := t.TempDir() + + mc := newSourceCacheMockConnector() + res, _, err := mc.AddGroup(ctx, "g1") + require.NoError(t, err) + _, err = mc.AddUser(ctx, "u1") + require.NoError(t, err) + mc.etagByResource[res.GetId().GetResource()] = "v1" + + sync1 := filepath.Join(tmpDir, "s1.c1z") + runSourceCacheSync(ctx, t, mc, sync1, "", tmpDir) + + // Warm sync whose replay seam raises a classified failure — the same + // return path a real replay copy/count-check failure takes. + store, err := dotc1z.NewStore(ctx, filepath.Join(tmpDir, "s2.c1z"), + dotc1z.WithEngine(c1zstore.EnginePebble), + dotc1z.WithTmpDir(tmpDir), + ) + require.NoError(t, err) + syncer, err := NewSyncer(ctx, mc, + WithConnectorStore(store), + WithTmpDir(tmpDir), + WithPreviousSyncC1ZPath(sync1), + func(s *syncer) { + s.testSourceCacheHaltHook = func(stage, scopeKey string) error { + if stage != "replay-copied" { + return nil + } + return fmt.Errorf("injected replay-seam failure for scope %q: %w", scopeKey, ErrReplayIntegrity) + } + }, + ) + require.NoError(t, err) + err = syncer.Sync(ctx) + require.Error(t, err) + require.ErrorIs(t, err, ErrReplayIntegrity, + "the sentinel must survive propagation to Sync()'s return, or the runners' cold retry never fires") + cctx, cancel := context.WithCancel(ctx) + cancel() + _ = syncer.Close(cctx) +} + +// TestWithoutPreviousSyncForcesCold pins the runners' cold-retry +// mechanism: appending WithoutPreviousSync AFTER a previous-sync path +// option must fully disable replay — the connector sees only lookup +// misses and the sync completes cold. +func TestWithoutPreviousSyncForcesCold(t *testing.T) { + ctx, err := logging.Init(t.Context()) + require.NoError(t, err) + tmpDir := t.TempDir() + + mc := newSourceCacheMockConnector() + res, _, err := mc.AddGroup(ctx, "g1") + require.NoError(t, err) + _, err = mc.AddUser(ctx, "u1") + require.NoError(t, err) + mc.etagByResource[res.GetId().GetResource()] = "v1" + + // Warm-capable chain: sync 1 stamps, sync 2 (control) replays. + sync1 := filepath.Join(tmpDir, "s1.c1z") + runSourceCacheSync(ctx, t, mc, sync1, "", tmpDir) + sync2 := filepath.Join(tmpDir, "s2.c1z") + runSourceCacheSync(ctx, t, mc, sync2, sync1, tmpDir) + require.Positive(t, mc.lookupHits, "control: the warm sync must have replayed") + + // The retry shape: previous path SET, WithoutPreviousSync appended + // last. The connector must see only misses. + hitsBefore := mc.lookupHits + sync3 := filepath.Join(tmpDir, "s3.c1z") + runSourceCacheSync(ctx, t, mc, sync3, sync1, tmpDir, WithoutPreviousSync()) + require.Equal(t, hitsBefore, mc.lookupHits, "WithoutPreviousSync must force every lookup to miss") +} + +func TestIngestInvariantI3UnrepairableFailsWithReplayIntegrity(t *testing.T) { + ctx, err := logging.Init(t.Context()) + require.NoError(t, err) + tmpDir := t.TempDir() + + repo, err := rs.NewResource("Repo r1", equivRepoRT, "r1") + require.NoError(t, err) + g := repairTestGrant(t, repo) + + // Previous sync exists but is ALSO missing the resource: repair has + // no source, so even default mode must fail — and classify as + // replay-integrity so the runners' cold retry can take over. + prev := newRepairTestStore(ctx, t, filepath.Join(tmpDir, "prev.c1z"), tmpDir) + _, err = prev.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + require.NoError(t, prev.PutResourceTypes(ctx, equivRepoRT)) + require.NoError(t, prev.EndSync(ctx)) + + cur := newRepairTestStore(ctx, t, filepath.Join(tmpDir, "cur.c1z"), tmpDir) + _, err = cur.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + require.NoError(t, cur.PutResourceTypes(ctx, equivRepoRT)) + require.NoError(t, cur.PutGrants(ctx, g)) + + s := &syncer{store: cur, syncType: connectorstore.SyncTypeFull} + prevSC, ok := any(prev).(dotc1z.SourceCacheStore) + require.True(t, ok) + s.sourceCache.prev = prevSC + s.sourceCache.prevReader = prev + + // WARM + unrepairable: replay is implicated → replay-integrity, so + // the runners retry cold (the attribution experiment). + err = s.checkGrantResourceReferences(ctx) + require.Error(t, err) + require.ErrorIs(t, err, ErrReplayIntegrity) + + // COLD (no previous source): replay cannot be the culprit — the + // error must blame connector data and must NOT carry the replay + // sentinel, or the runners would burn a pointless cold retry every + // sync on a recurring connector bug. + s.sourceCache.prev = nil + s.sourceCache.prevReader = nil + err = s.checkGrantResourceReferences(ctx) + require.Error(t, err) + require.NotErrorIs(t, err, ErrReplayIntegrity) + require.Contains(t, err.Error(), "COLD sync") + + require.NoError(t, cur.Close(ctx)) + require.NoError(t, prev.Close(ctx)) +} diff --git a/pkg/sync/ingest_invariants_scale_test.go b/pkg/sync/ingest_invariants_scale_test.go new file mode 100644 index 000000000..f2271dc9a --- /dev/null +++ b/pkg/sync/ingest_invariants_scale_test.go @@ -0,0 +1,172 @@ +package sync //nolint:revive,nolintlint // we can't change the package name for backwards compatibility + +// Scale and inertness pins for the dangling-reference drops: +// +// - the drop paths must stream chunked batches (a large dangling +// population costs seconds, not one fsync per row); +// - the invariants must be COMPLETELY inert on SQLite stores and on +// partial syncs, in both modes. + +import ( + "fmt" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/require" + + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + reader_v2 "github.com/conductorone/baton-sdk/pb/c1/reader/v2" + "github.com/conductorone/baton-sdk/pkg/connectorstore" + "github.com/conductorone/baton-sdk/pkg/dotc1z" + "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" + "github.com/conductorone/baton-sdk/pkg/logging" + et "github.com/conductorone/baton-sdk/pkg/types/entitlement" + "github.com/conductorone/baton-sdk/pkg/types/grant" + rs "github.com/conductorone/baton-sdk/pkg/types/resource" +) + +// TestIngestInvariantI8DropScale drops 100k grants under one dangling +// entitlement. With the streaming chunked-batch implementation this is +// seconds; a per-row-fsync implementation would blow the test timeout +// (100k synced commits), so completion itself pins the batching. +func TestIngestInvariantI8DropScale(t *testing.T) { + if testing.Short() { + t.Skip("scale test") + } + ctx, err := logging.Init(t.Context()) + require.NoError(t, err) + tmpDir := t.TempDir() + + repo, err := rs.NewResource("Repo r1", equivRepoRT, "r1") + require.NoError(t, err) + + // No eq_repo TYPE row: a disabled-type gap, which is the drop arm of + // the replay policy (an enabled-type dangling would fail instead). + cur := newRepairTestStore(ctx, t, filepath.Join(tmpDir, "cur.c1z"), tmpDir) + _, err = cur.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + require.NoError(t, cur.PutResources(ctx, repo)) + + const grantCount = 100_000 + batch := make([]*v2.Grant, 0, 1000) + for i := 0; i < grantCount; i++ { + principal := v2.Resource_builder{ + Id: v2.ResourceId_builder{ResourceType: "user", Resource: fmt.Sprintf("u-%06d", i)}.Build(), + }.Build() + // All grants reference the same entitlement, which has no row. + batch = append(batch, grant.NewGrant(repo, "ghost", principal.GetId())) + if len(batch) == cap(batch) { + require.NoError(t, cur.PutGrants(ctx, batch...)) + batch = batch[:0] + } + } + require.NoError(t, cur.PutGrants(ctx, batch...)) + + s := &syncer{store: cur, syncType: connectorstore.SyncTypeFull} + start := time.Now() + require.NoError(t, s.checkGrantEntitlementReferences(ctx)) + elapsed := time.Since(start) + t.Logf("dropped %d dangling grants in %v", grantCount, elapsed) + + resp, err := cur.ListGrants(ctx, v2.GrantsServiceListGrantsRequest_builder{PageSize: 1}.Build()) + require.NoError(t, err) + require.Empty(t, resp.GetList(), "every dangling grant must be dropped") + + // Idempotent second pass, and fail-fast agrees nothing remains. + require.NoError(t, s.checkGrantEntitlementReferences(ctx)) + s.failFastInvariants = true + require.NoError(t, s.checkGrantEntitlementReferences(ctx)) + + require.NoError(t, cur.Close(ctx)) +} + +// TestReferentialInvariantsInertOnSQLite pins the SQLite gate: the +// referential invariants require IngestInvariantStore (Pebble-only), so on a +// SQLite store they are complete no-ops in both modes — no error, no +// drop — even with flagrantly dangling data present. +func TestReferentialInvariantsInertOnSQLite(t *testing.T) { + ctx, err := logging.Init(t.Context()) + require.NoError(t, err) + tmpDir := t.TempDir() + + store, err := dotc1z.NewStore(ctx, filepath.Join(tmpDir, "sqlite.c1z"), + dotc1z.WithEngine(c1zstore.EngineSQLite), + dotc1z.WithTmpDir(tmpDir), + ) + require.NoError(t, err) + _, ok := any(store).(dotc1z.IngestInvariantStore) + require.False(t, ok, "SQLite store must not implement IngestInvariantStore; if it grows the capability, revisit the drop gates") + + _, err = store.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + repo, err := rs.NewResource("Repo r1", equivRepoRT, "r1") + require.NoError(t, err) + require.NoError(t, store.PutResourceTypes(ctx, equivRepoRT)) + // Dangling everything: entitlement without its resource, grant + // without its entitlement or principal. + danglingEnt := et.NewAssignmentEntitlement(repo, "admin") + require.NoError(t, store.PutEntitlements(ctx, danglingEnt)) + ghost := v2.Resource_builder{ + Id: v2.ResourceId_builder{ResourceType: "user", Resource: "ghost"}.Build(), + }.Build() + g := grant.NewGrant(repo, "ghost-slug", ghost.GetId()) + require.NoError(t, store.PutGrants(ctx, g)) + + for _, failFast := range []bool{false, true} { + s := &syncer{store: store, syncType: connectorstore.SyncTypeFull, failFastInvariants: failFast} + require.NoError(t, s.checkEntitlementResourceReferences(ctx)) + require.NoError(t, s.checkGrantResourceReferences(ctx)) + require.NoError(t, s.checkGrantEntitlementReferences(ctx)) + require.NoError(t, s.checkGrantPrincipalReferences(ctx)) + } + + // Nothing was dropped. + _, err = store.GetEntitlement(ctx, reader_v2.EntitlementsReaderServiceGetEntitlementRequest_builder{ + EntitlementId: danglingEnt.GetId(), + }.Build()) + require.NoError(t, err) + _, err = store.GetGrant(ctx, reader_v2.GrantsReaderServiceGetGrantRequest_builder{ + GrantId: g.GetId(), + }.Build()) + require.NoError(t, err) + + require.NoError(t, store.Close(ctx)) +} + +// TestReferentialInvariantsInertOnPartialSync pins the sync-type gate: +// a partial sync's store legitimately lacks rows that live in the base +// full sync, so the referential invariants must not evaluate at all — +// no error even in fail-fast mode, no drops. +func TestReferentialInvariantsInertOnPartialSync(t *testing.T) { + ctx, err := logging.Init(t.Context()) + require.NoError(t, err) + tmpDir := t.TempDir() + + cur := newRepairTestStore(ctx, t, filepath.Join(tmpDir, "cur.c1z"), tmpDir) + _, err = cur.StartNewSync(ctx, connectorstore.SyncTypePartial, "") + require.NoError(t, err) + repo, err := rs.NewResource("Repo r1", equivRepoRT, "r1") + require.NoError(t, err) + require.NoError(t, cur.PutResourceTypes(ctx, equivRepoRT)) + ghost := v2.Resource_builder{ + Id: v2.ResourceId_builder{ResourceType: "user", Resource: "ghost"}.Build(), + }.Build() + g := grant.NewGrant(repo, "ghost-slug", ghost.GetId()) + require.NoError(t, cur.PutGrants(ctx, g)) + + for _, failFast := range []bool{false, true} { + s := &syncer{store: cur, syncType: connectorstore.SyncTypePartial, failFastInvariants: failFast} + require.NoError(t, s.checkEntitlementResourceReferences(ctx)) + require.NoError(t, s.checkGrantResourceReferences(ctx)) + require.NoError(t, s.checkGrantEntitlementReferences(ctx)) + require.NoError(t, s.checkGrantPrincipalReferences(ctx)) + } + + _, err = cur.GetGrant(ctx, reader_v2.GrantsReaderServiceGetGrantRequest_builder{ + GrantId: g.GetId(), + }.Build()) + require.NoError(t, err, "partial syncs must never drop") + + require.NoError(t, cur.Close(ctx)) +} diff --git a/pkg/sync/ingest_invariants_test.go b/pkg/sync/ingest_invariants_test.go new file mode 100644 index 000000000..e8218047b --- /dev/null +++ b/pkg/sync/ingest_invariants_test.go @@ -0,0 +1,75 @@ +package sync //nolint:revive,nolintlint // we can't change the package name for backwards compatibility + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/proto" + + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" +) + +// TestPushChildResourceActionsDedupesPerSync pins the per-sync child +// scheduling dedupe: a parent discovered by several ingestion paths in +// one sync (replay copy + delta overlay re-emission) must schedule each +// (childType, parent) pair exactly once — the duplicate would double the +// child-listing connector work on every warm overlay. The dedupe rides +// the I4 evidence set (childScheduleSet.recordIfNew), so the evidence +// stays complete for every pair regardless of which discovery scheduled. +func TestPushChildResourceActionsDedupesPerSync(t *testing.T) { + ctx := context.Background() + s := &syncer{state: newState()} + + countActions := func() int { + st, ok := s.state.(*state) + require.True(t, ok) + return len(st.actions) + } + + s.pushChildResourceActions(ctx, []string{"project", "repo"}, "org", "org0") + require.Equal(t, 2, countActions()) + + // Second discovery of the same parent (e.g. the overlay re-emission + // after a replay copy already scheduled it): no new actions. + s.pushChildResourceActions(ctx, []string{"project", "repo"}, "org", "org0") + require.Equal(t, 2, countActions()) + + // A different parent still schedules. + s.pushChildResourceActions(ctx, []string{"project"}, "org", "org1") + require.Equal(t, 3, countActions()) + + // The I4 evidence is intact for every recorded pair. + require.True(t, s.childSchedule.has("project", "org", "org0")) + require.True(t, s.childSchedule.has("repo", "org", "org0")) + require.True(t, s.childSchedule.has("project", "org", "org1")) +} + +// TestIngestInvariantAnnotationCoverage is the completeness meta-test for +// the ingestion-invariant registry: every connector annotation known to +// imply a syncer side effect must map to the mechanism that guarantees it +// regardless of ingestion path (stream, replay, or whatever comes next). +// Adding a new side-effect-implying annotation without extending +// sideEffectAnnotationCoverage — and the invariant behind it — fails +// here instead of in review round eleven. +func TestIngestInvariantAnnotationCoverage(t *testing.T) { + sideEffectAnnotations := []proto.Message{ + &v2.GrantExpandable{}, + &v2.ExternalResourceMatch{}, + &v2.ExternalResourceMatchAll{}, + &v2.ExternalResourceMatchID{}, + &v2.InsertResourceGrants{}, + &v2.ChildResourceType{}, + &v2.EntitlementExclusionGroup{}, + &v2.TypeScopedEntitlements{}, + &v2.TypeScopedGrants{}, + } + for _, msg := range sideEffectAnnotations { + name := string(msg.ProtoReflect().Descriptor().FullName()) + require.Contains(t, sideEffectAnnotationCoverage, name, + "side-effect-implying annotation %s has no registered ingestion invariant; "+ + "see docs/tasks/source-cache-ingestion-invariants.md", name) + } + require.Len(t, sideEffectAnnotationCoverage, len(sideEffectAnnotations), + "coverage map and the enumerated annotation set drifted; update both together") +} diff --git a/pkg/sync/parallel_syncer.go b/pkg/sync/parallel_syncer.go index 387204852..3b2f59640 100644 --- a/pkg/sync/parallel_syncer.go +++ b/pkg/sync/parallel_syncer.go @@ -304,6 +304,24 @@ func (s *syncer) parallelSync( } } + if !s.dontExpandGrants && !s.state.NeedsExpansion() { + // A replay-only grants phase can arm expansion without + // processing a connector GrantExpandable row. If the + // process dies after its last grant action finishes but + // before that in-memory flag is checkpointed, resume + // reaches this action with NeedsExpansion=false even + // though the current store contains pending expansion + // rows. Reconcile from the authoritative store before + // deciding to skip; PendingExpansionPage is read-only and + // repeats the expansion loader's first page safely. + pending, _, pendingErr := s.store.Grants().PendingExpansionPage(ctx, stateAction.PageToken) + if pendingErr != nil { + return warnings, pendingErr + } + if len(pending) > 0 { + s.state.SetNeedsExpansion() + } + } if s.dontExpandGrants || !s.state.NeedsExpansion() { l.Debug("skipping grant expansion, no grants to expand") s.state.FinishAction(ctx, stateAction) diff --git a/pkg/sync/pebble_etag_replay_test.go b/pkg/sync/pebble_etag_replay_test.go index 045392bbd..cb832323f 100644 --- a/pkg/sync/pebble_etag_replay_test.go +++ b/pkg/sync/pebble_etag_replay_test.go @@ -354,6 +354,11 @@ func TestOptionalPreviousSyncC1ZPath_SoftFails(t *testing.T) { t.Run(name, func(t *testing.T) { mc := newEtagObservingMockConnector("etag-v1") mc.WithData(group, ent, grant) + // The grant's principal must exist: a missing row under the + // synced user type is an enabled-type dangling and fails the + // sync under the replay policy (this fixture was previously + // referentially dishonest and the drop arm hid it). + mc.AddResource(ctx, user) store, err := dotc1z.NewStore(ctx, filepath.Join(t.TempDir(), "out.c1z"), dotc1z.WithEngine(c1zstore.EnginePebble), dotc1z.WithTmpDir(tempDir), diff --git a/pkg/sync/progresslog/progresslog.go b/pkg/sync/progresslog/progresslog.go index 734ccd548..e30773825 100644 --- a/pkg/sync/progresslog/progresslog.go +++ b/pkg/sync/progresslog/progresslog.go @@ -32,15 +32,17 @@ const ( ) type ProgressLog struct { - resourceTypes int - resources map[string]int - entitlementsProgress map[string]int - lastEntitlementLog map[string]time.Time - grantsProgress map[string]int - lastGrantLog map[string]time.Time - mu sync.RWMutex - l *zap.Logger - maxLogFrequency time.Duration + resourceTypes int + resources map[string]int + entitlementsProgress map[string]int + lastEntitlementLog map[string]time.Time + entitlementsCountOnly map[string]bool + grantsProgress map[string]int + lastGrantLog map[string]time.Time + grantsCountOnly map[string]bool + mu sync.RWMutex + l *zap.Logger + maxLogFrequency time.Duration // Optional cross-step db-size tracking for LogExpandProgress. Populated // via WithDBSizeProvider at construction time or SetDBSizeProvider after @@ -145,15 +147,17 @@ func (p *ProgressLog) SetDBSizeProvider(provider connectorstore.DBSizeProvider) func NewProgressCounts(ctx context.Context, opts ...Option) *ProgressLog { p := &ProgressLog{ - resources: make(map[string]int), - entitlementsProgress: make(map[string]int), - lastEntitlementLog: make(map[string]time.Time), - grantsProgress: make(map[string]int), - lastGrantLog: make(map[string]time.Time), - l: ctxzap.Extract(ctx), - maxLogFrequency: defaultMaxLogFrequency, - mu: sync.RWMutex{}, - metricsHandler: metrics.NewNoOpHandler(ctx), + resources: make(map[string]int), + entitlementsProgress: make(map[string]int), + lastEntitlementLog: make(map[string]time.Time), + entitlementsCountOnly: make(map[string]bool), + grantsProgress: make(map[string]int), + lastGrantLog: make(map[string]time.Time), + grantsCountOnly: make(map[string]bool), + l: ctxzap.Extract(ctx), + maxLogFrequency: defaultMaxLogFrequency, + mu: sync.RWMutex{}, + metricsHandler: metrics.NewNoOpHandler(ctx), } for _, o := range opts { o(p) @@ -202,15 +206,19 @@ func (p *ProgressLog) LogResourcesProgress(ctx context.Context, resourceType str func (p *ProgressLog) LogEntitlementsProgress(ctx context.Context, resourceType string) { var entitlementsProgress, resources int var lastLogTime time.Time + var countOnly bool p.mu.RLock() entitlementsProgress = p.entitlementsProgress[resourceType] resources = p.resources[resourceType] lastLogTime = p.lastEntitlementLog[resourceType] + countOnly = p.entitlementsCountOnly[resourceType] p.mu.RUnlock() - if resources == 0 { - // if resuming sync, resource counts will be zero, so don't calculate percentage. just log every 10 seconds. + if resources == 0 || countOnly { + // Count-only: either a resumed sync (resource counts are zero) or a + // type-scoped entitlements type (no meaningful denominator). Log the + // raw synced count every log window; never compute a percentage. if time.Since(lastLogTime) > p.maxLogFrequency { p.l.Info("Syncing entitlements", zap.String("resource_type_id", resourceType), @@ -256,18 +264,66 @@ func (p *ProgressLog) LogEntitlementsProgress(ctx context.Context, resourceType } } +// SetEntitlementsCountOnly marks a resource type's entitlement progress as a +// plain count with no resources-covered denominator. Used for type-scoped +// entitlement enumeration (v2.TypeScopedEntitlements), where cursors don't +// map 1:1 to resources. +func (p *ProgressLog) SetEntitlementsCountOnly(resourceType string) { + p.mu.Lock() + defer p.mu.Unlock() + p.entitlementsCountOnly[resourceType] = true +} + +// EntitlementsProgress returns the current entitlement-coverage counter for a +// resource type. For per-resource types this is "resources covered"; for +// count-only (type-scoped) types it is the raw entitlement-row count. +// Exposed for tests pinning that accounting. +func (p *ProgressLog) EntitlementsProgress(resourceType string) int { + p.mu.RLock() + defer p.mu.RUnlock() + return p.entitlementsProgress[resourceType] +} + +// SetGrantsCountOnly marks a resource type's grant progress as a plain +// count with no resources-covered denominator. Used for type-scoped grant +// enumeration (v2.TypeScopedGrants), where cursors don't map 1:1 to +// resources: the per-cursor accounting would exceed the type's resource +// total and trip the "more grant resources than resources" warning for a +// perfectly healthy sync. Count-only types log synced row counts +// periodically and never compute the ratio. +func (p *ProgressLog) SetGrantsCountOnly(resourceType string) { + p.mu.Lock() + defer p.mu.Unlock() + p.grantsCountOnly[resourceType] = true +} + +// GrantsProgress returns the current grant-coverage counter for a resource +// type. For per-resource types this is "resources covered" and must never +// exceed the type's resource total — spawned sibling cursors +// (v2.EnqueuePageTokens) don't increment it, only the origin action's chain end +// does. Exposed for tests pinning that accounting. +func (p *ProgressLog) GrantsProgress(resourceType string) int { + p.mu.RLock() + defer p.mu.RUnlock() + return p.grantsProgress[resourceType] +} + func (p *ProgressLog) LogGrantsProgress(ctx context.Context, resourceType string) { var grantsProgress, resources int var lastLogTime time.Time + var countOnly bool p.mu.RLock() grantsProgress = p.grantsProgress[resourceType] resources = p.resources[resourceType] lastLogTime = p.lastGrantLog[resourceType] + countOnly = p.grantsCountOnly[resourceType] p.mu.RUnlock() - if resources == 0 { - // if resuming sync, resource counts will be zero, so don't calculate percentage. just log every 10 seconds. + if resources == 0 || countOnly { + // Count-only: either a resumed sync (resource counts are zero) or a + // type-scoped grants type (no meaningful denominator). Log the raw + // synced count every log window; never compute a percentage. if time.Since(lastLogTime) > p.maxLogFrequency { p.l.Info("Syncing grants", zap.String("resource_type_id", resourceType), diff --git a/pkg/sync/source_cache.go b/pkg/sync/source_cache.go new file mode 100644 index 000000000..69d4be2f5 --- /dev/null +++ b/pkg/sync/source_cache.go @@ -0,0 +1,665 @@ +package sync //nolint:revive,nolintlint // we can't change the package name for backwards compatibility + +import ( + "context" + "fmt" + stdsync "sync" + "time" + + "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" + "go.uber.org/zap" + + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + "github.com/conductorone/baton-sdk/pkg/annotations" + "github.com/conductorone/baton-sdk/pkg/connectorstore" + "github.com/conductorone/baton-sdk/pkg/dotc1z" + "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" + "github.com/conductorone/baton-sdk/pkg/sourcecache" +) + +// Source-cache replay, syncer side. See +// proto/c1/connector/v2/annotation_source_cache.proto for the contract. +// +// Setup degrades, replay fails loudly. Any setup problem (capability +// absent, store engine unsupported, no usable previous sync) installs the +// no-op lookup: the connector never sees a previous validator, never gets +// a conditional-request hit, and therefore never emits SourceCacheReplay +// — which is what makes it safe to treat a replay annotation arriving +// while degraded as a hard error (the connector already skipped row +// generation; there is nothing to fall back to). + +// syncerSourceCache is the per-sync source-cache state resolved by +// configureSourceCache. +// +// Write side and read side enable independently: the FIRST sync of a chain +// has no previous sync but must still stamp rows and write manifest +// entries, or the second sync would have nothing to replay. enabled covers +// the write side (capability declared + current store supports it); prev +// is non-nil only when a usable previous sync exists (read side — lookup +// hits and replay). +type syncerSourceCache struct { + enabled bool + // current is the writable output store's source-cache capability. + current dotc1z.SourceCacheStore + // prev is the previous sync's lookup/replay source (read-only). Nil + // when no usable previous sync exists; lookups then miss and replay + // annotations are hard errors. + prev dotc1z.SourceCacheStore + // prevReader is the same store as prev, typed for ReplaySourceCache. + prevReader connectorstore.Reader + // lookup is the connector-facing lookup built from prev (NoopLookup + // when prev is nil). The syncer also uses it to answer lookup asks + // from connectors on single-shot transports (the ask/answer + // continuation); both paths see identical results by construction. + lookup sourcecache.Lookup + // replayedScopes tracks scopes replayed this sync whose validator may + // arrive on a LATER page (multi-page overlay rounds), so the final + // page's manifest write is not also counted as a fresh stamp — one + // warm delta round would otherwise book as both a hit and a miss in + // the replay ratio. Only rounds that can span pages are recorded + // (overlay, or replay without an inline validator), keeping the set tiny: + // single-page 304 rounds never enter it. Pointer so copying the + // per-sync struct shares the set; guarded internally for parallel + // workers. Best-effort across resume (in-memory), stats-only. + replayedScopes *replayedScopeSet + // lookupInstalled records that THIS sync installed a lookup (real or + // no-op) into the connector's shared slot, so Close only clears what + // it owns. A syncer that failed before installing must not clear — + // on the shared subprocess slot that would decrement another live + // sync's install and blind it (see sourcecache.GRPCServer). + lookupInstalled bool + // compatKey is this sync's replay-compatibility key, computed by + // configureSourceCache and persisted into the artifact right after + // the sync starts (see Sync). The read side requires the previous + // artifact's recorded key to match it byte-exactly. + compatKey sourcecache.CompatKey +} + +type scopeRef struct { + kind sourcecache.RowKind + scopeKey string +} + +type replayedScopeSet struct { + mu stdsync.Mutex + m map[scopeRef]struct{} +} + +func (c *syncerSourceCache) markScopeReplayed(kind sourcecache.RowKind, scopeKey string) { + if c.replayedScopes == nil { + return + } + c.replayedScopes.mu.Lock() + defer c.replayedScopes.mu.Unlock() + if c.replayedScopes.m == nil { + c.replayedScopes.m = make(map[scopeRef]struct{}) + } + c.replayedScopes.m[scopeRef{kind: kind, scopeKey: scopeKey}] = struct{}{} +} + +func (c *syncerSourceCache) scopeReplayedThisSync(kind sourcecache.RowKind, scopeKey string) bool { + if c.replayedScopes == nil { + return false + } + c.replayedScopes.mu.Lock() + defer c.replayedScopes.mu.Unlock() + _, ok := c.replayedScopes.m[scopeRef{kind: kind, scopeKey: scopeKey}] + return ok +} + +// prevStoreLookup adapts the previous store's manifest to the +// connector-facing Lookup. Mid-sync read errors are logged once and +// treated as misses: at lookup time the connector can still fetch fresh, +// so degrading beats failing the sync. +type prevStoreLookup struct { + prev dotc1z.SourceCacheStore + logOnce *stdsync.Once +} + +var _ sourcecache.Lookup = prevStoreLookup{} + +func (p prevStoreLookup) Lookup(ctx context.Context, kind sourcecache.RowKind, scopeKey string) (sourcecache.Entry, bool, error) { + entry, found, err := p.prev.LookupSourceCacheEntry(ctx, kind, scopeKey) + if err != nil { + p.logOnce.Do(func() { + ctxzap.Extract(ctx).Warn("source cache lookup failed; treating as miss", zap.Error(err)) + }) + return sourcecache.Entry{}, false, nil //nolint:nilerr // intentional: a failed lookup degrades to a miss (connector fetches fresh) rather than failing the connector call + } + return entry, found, nil +} + +// configureSourceCache resolves per-sync source-cache state from the +// connector's Validate response and installs the connector-facing lookup. +// Called once per Sync, after Validate. +func (s *syncer) configureSourceCache(ctx context.Context, resp *v2.ConnectorServiceValidateResponse) error { + l := ctxzap.Extract(ctx) + s.sourceCache = syncerSourceCache{} + + setLookup, canSetLookup := s.connector.(sourcecache.SourceCacheSetter) + degrade := func(reason string) error { + if canSetLookup { + // The no-op install still occupies the shared slot: a + // degraded sync's connector must see misses even while + // another sync's real lookup is live, or its replay-illegal + // state could receive hits. + setLookup.SetSourceCache(ctx, sourcecache.NoopLookup{}) + s.sourceCache.lookupInstalled = true + } + // An operator who EXPLICITLY configured --previous-sync-c1z (not + // the runner's optional spare) is expecting replay; degrading to + // a cold sync at Info level left them silently replay-less — the + // default storage engine (SQLite) doesn't support source cache, + // so the flag alone quietly did nothing without + // --storage-engine pebble. Warn with the reason instead. + if s.previousSyncC1ZPath != "" && !s.previousSyncC1ZPathOptional { + if reason == "" { + reason = "connector does not declare the source-cache capability" + } + l.Warn("source cache disabled; --previous-sync-c1z will be ignored and the sync runs cold", + zap.String("reason", reason), + zap.String("previous_sync_c1z", s.previousSyncC1ZPath), + ) + return nil + } + if reason != "" { + l.Info("source cache disabled", zap.String("reason", reason)) + } + return nil + } + + capability := &v2.SourceCacheCapability{} + annos := annotations.Annotations(resp.GetAnnotations()) + ok, err := annos.Pick(capability) + if err != nil { + return fmt.Errorf("error parsing source cache capability annotation: %w", err) + } + if !ok || capability.GetMode() != v2.SourceCacheCapability_MODE_READ_WRITE { + // The common case; stay quiet. + return degrade("") + } + if s.syncType != connectorstore.SyncTypeFull { + // Replay is FULL-sync-only. A partial/targeted sync writes a + // targeted subset; replaying a whole scope could resurrect rows + // outside the targets, and the partial-sync related-resource + // fetch (ShouldFetchRelatedResources) only runs over response + // rows, so replayed grants could reference resources the partial + // store never received. Degrading installs the no-op lookup: + // every scope misses and the connector fetches fresh — + // cold-correct, and partial syncs are small by construction. + return degrade(fmt.Sprintf("source cache replay is only supported on full syncs (sync type %q)", s.syncType)) + } + current, ok := s.store.(dotc1z.SourceCacheStore) + if !ok { + return degrade("storage engine does not support source cache") + } + + // The replay-compatibility key: the exact conditions this sync's + // manifest is recorded under, and the conditions a previous artifact + // must have been recorded under to serve replay. Components are + // explicit declarations (never inferred from versions): the + // connector's cache generation and config/permission fingerprint + // (Validate annotations), the SDK's materialization-policy + // generation, and the sync-selection fingerprint. + compatKey := sourcecache.CompatKey{ + ConnectorCacheGeneration: capability.GetCacheGeneration(), + ConnectorConfigFingerprint: capability.GetConfigFingerprint(), + SDKMaterializationGeneration: sourcecache.MaterializationPolicyGeneration, + SyncSelectionFingerprint: sourcecache.SelectionFingerprint(s.syncResourceTypes, s.skipEntitlementsAndGrants, s.skipGrants), + } + + // Write side enabled: rows produced under a scope get stamped and + // manifest entries get written, so this sync is usable as the NEXT + // sync's replay source even when this one has nothing to replay from. + // The compat key is stashed here and persisted by Sync right after + // the sync starts (configureSourceCache runs before StartOrResumeSync, + // so the store has no current sync yet) — still BEFORE any page could + // write a manifest entry, so the artifact can never hold validators + // without the key that scopes their validity. + s.sourceCache = syncerSourceCache{enabled: true, current: current, replayedScopes: &replayedScopeSet{}, compatKey: compatKey} + + // Read side: a usable previous sync makes lookups hit and replay legal. + var readReason string + if s.previousSyncReader == nil { + readReason = "no previous-sync c1z configured" + } else if prev, ok := s.previousSyncReader.(dotc1z.SourceCacheStore); !ok { + readReason = "previous-sync store engine does not support source cache" + } else if reason, err := previousSyncReplayUnusableReason(ctx, s.previousSyncReader); err != nil { + return fmt.Errorf("error reading previous sync's run metadata: %w", err) + } else if reason != "" { + // The explicit metadata gate, independent of the manifest's + // contents (c1zstore.SyncRun.UsableAsReplaySource): compacted + // artifacts are keep-newer merges no input's validators describe, + // and partial/derived syncs are subsets whose rows do not cover + // the scopes any validator vouches for. Those paths also leave no + // manifest to hit (belt); the run-record gate is the contract + // (suspenders) — "what kind of sync is this" answers "can it be + // used for replay" without inspecting keyspaces. + readReason = reason + } else if compatReason := replayCompatMismatchReason(ctx, compatKey, prev); compatReason != "" { + // The replay-compatibility gate: exact key match or cold. An + // absent key (pre-key artifact, fold) and an unreadable key both + // count as mismatches — replay is optional, and "cannot prove + // compatible" means incompatible. + readReason = compatReason + } else { + s.sourceCache.prev = prev + s.sourceCache.prevReader = s.previousSyncReader + } + + lookup := sourcecache.Lookup(sourcecache.NoopLookup{}) + if s.sourceCache.prev != nil { + lookup = prevStoreLookup{prev: s.sourceCache.prev, logOnce: &stdsync.Once{}} + } + s.sourceCache.lookup = lookup + if canSetLookup { + setLookup.SetSourceCache(ctx, lookup) + s.sourceCache.lookupInstalled = true + } else { + // The connector declared the capability but the client offers no + // way to deliver lookups. Its own lookup stays no-op, so every + // scope misses and no replay annotations can legally arrive. + l.Warn("source cache capability declared but connector client cannot receive lookups") + } + l.Info("source cache enabled", + zap.Bool("replay_available", s.sourceCache.prev != nil), + zap.String("replay_unavailable_reason", readReason), + ) + return nil +} + +// replayCompatMismatchReason evaluates the replay-compatibility gate: +// the previous artifact's recorded key must match the current run's key +// byte-exactly on every component, or the sync runs cold. Absent +// (pre-key artifacts, compaction folds) and unreadable keys are +// mismatches, not errors: replay is a pure optimization, and a key we +// cannot verify is a key that does not match. +func replayCompatMismatchReason(ctx context.Context, current sourcecache.CompatKey, prev dotc1z.SourceCacheStore) string { + prevKey, found, err := prev.GetSourceCacheCompat(ctx) + if err != nil { + ctxzap.Extract(ctx).Warn("error reading previous sync's replay-compatibility key; running cold", zap.Error(err)) + return "previous sync's replay-compatibility key is unreadable" + } + if !found { + return "previous sync carries no replay-compatibility key (recorded by a pre-key SDK, or a compaction fold)" + } + return current.MismatchReason(prevKey) +} + +// ReplaySourceRunUnusableReason evaluates the sync-run metadata gate on +// one finished-run record: "" when the run qualifies as a replay source, +// or a human-readable reason. Exported as the single source of truth for +// the gate — the syncer's read-side setup and the `baton source-cache` +// audit command both consume it, so the two can never drift. +func ReplaySourceRunUnusableReason(run *c1zstore.SyncRun) string { + switch { + case run == nil: + return "previous-sync c1z has no finished sync run" + case run.UsableAsReplaySource(): + // The record can lie by omission: a compacted artifact produced + // by an SDK that predates the run record's compacted flag carries + // a full type and no flag, and its fold never cleared the base + // copy's source-cache manifest — so both the metadata gate and + // the manifest belt would pass, and the keep-newer merge would + // replay under validators that do not describe its contents. The + // sync TOKEN's compaction provenance section has been written by + // every compactor version, so it is the one signal a mixed-version + // artifact cannot omit. A token that fails to parse proves + // nothing either way and is ignored (the primary gates stand). + if comp, tokenErr := CompactionStatsFromToken(run.SyncToken); tokenErr == nil && comp != nil { + return "previous sync's token carries compaction provenance; the artifact is a merge no upstream validator describes (produced by a pre-compacted-flag compactor)" + } + return "" + case run.Compacted: + return "previous sync is a compaction artifact; upstream validators do not describe its merged contents" + default: + return fmt.Sprintf("previous sync is type %q, not a full sync; its rows do not cover the scopes any validator vouches for", run.Type) + } +} + +// previousSyncReplayUnusableReason inspects the previous-sync reader's +// newest finished run record against the replay-source predicate +// (ReplaySourceRunUnusableReason) and returns a human-readable reason +// when it fails, or "" when the sync qualifies. A reader without a +// SyncMeta surface degrades to "" — the manifest-based guards still +// apply — but a readable record is authoritative. +func previousSyncReplayUnusableReason(ctx context.Context, reader connectorstore.Reader) (string, error) { + metaHolder, ok := reader.(interface{ SyncMeta() c1zstore.SyncMeta }) + if !ok { + return "", nil + } + run, err := metaHolder.SyncMeta().LatestFinishedSyncOfAnyType(ctx) + if err != nil { + return "", err + } + return ReplaySourceRunUnusableReason(run), nil +} + +// clearSourceCacheLookup detaches the per-sync lookup so a late RPC from +// the connector cannot read a store the syncer no longer owns. Only the +// sync that actually installed a lookup clears: a syncer that failed +// before configureSourceCache's install must not decrement the shared +// slot's accounting or blind a concurrently live sync. +func (s *syncer) clearSourceCacheLookup(ctx context.Context) { + if !s.sourceCache.lookupInstalled { + return + } + if setLookup, ok := s.connector.(sourcecache.SourceCacheSetter); ok { + setLookup.SetSourceCache(ctx, nil) + s.sourceCache.lookupInstalled = false + } +} + +// sourceCachePage carries one list response's source-cache instructions +// from beginSourceCachePage (before rows are written) to +// finishSourceCachePage (after rows are written). +type sourceCachePage struct { + kind sourcecache.RowKind + scopeKey string + cacheValidator string + // replayed reports that beginSourceCachePage copied the previous + // sync's rows for this scope into the current sync BEFORE the page's + // own rows commit. Consumers that dedupe against "already synced this + // sync" state (the resources path) must not skip this page's rows: + // they are the overlay, and the already-present row is the stale + // replayed base they exist to overwrite. + replayed bool + // replayRows / overlayRows carry the replay's copy count and the + // page's own row count from begin to finish, where the stats are + // recorded — recording at begin would double-count a page that is + // re-run after failing between replay and finish. + replayRows int64 + overlayRows int + // deletedIDs are canonical-id tombstones (grant/entitlement ids, + // resource BIDs); deletedPrincipalIDs are bare-object-id tombstones + // applied scope-relatively. Both may arrive on any page of a scope + // (replay annotation on the first page, scope annotation on every + // page) and apply after the page's rows commit. + deletedIDs []string + deletedPrincipalIDs []string +} + +// beginSourceCachePage inspects a list response's annotations, performs +// any requested replay, and returns the context to write the page's rows +// under (stamped with the scope when one is present). A nil page means the +// response carried no source-cache instructions. +// +// rowCount is the number of rows in the response; a non-overlay replay +// that also returned rows gets a warning (the rows are upserted anyway). +func (s *syncer) beginSourceCachePage( + ctx context.Context, + kind sourcecache.RowKind, + respAnnos annotations.Annotations, + rowCount int, +) (context.Context, *sourceCachePage, error) { + replay := &v2.SourceCacheReplay{} + hasReplay, err := respAnnos.Pick(replay) + if err != nil { + return ctx, nil, fmt.Errorf("source cache: error parsing replay annotation: %w", err) + } + scope := &v2.SourceCacheRecord{} + hasScope, err := respAnnos.Pick(scope) + if err != nil { + return ctx, nil, fmt.Errorf("source cache: error parsing scope annotation: %w", err) + } + if !hasReplay && !hasScope { + return ctx, nil, nil + } + + if !s.sourceCache.enabled { + if hasReplay { + // The connector skipped row generation expecting a replay; with + // source cache disabled there is nothing to replay from. This is + // a connector bug (replay for a scope it never got a lookup hit + // on), not a degradable condition. + return ctx, nil, fmt.Errorf("source cache: connector requested replay for scope %q but source cache is disabled", replay.GetScopeKey()) + } + // Scope annotations without the capability handshake are ignored. + return ctx, nil, nil + } + + page := &sourceCachePage{kind: kind} + switch { + case hasReplay && hasScope: + if replay.GetScopeKey() != scope.GetScopeKey() { + return ctx, nil, fmt.Errorf("source cache: replay scope %q and page scope %q disagree", replay.GetScopeKey(), scope.GetScopeKey()) + } + page.scopeKey = replay.GetScopeKey() + case hasReplay: + page.scopeKey = replay.GetScopeKey() + default: + page.scopeKey = scope.GetScopeKey() + } + if err := sourcecache.ValidateScopeKey(page.scopeKey); err != nil { + return ctx, nil, fmt.Errorf("source cache: %w", err) + } + // Prefer the scope annotation's etag (the freshest validator on + // overlay pages); fall back to the replay's. + page.cacheValidator = scope.GetCacheValidator() + if page.cacheValidator == "" { + page.cacheValidator = replay.GetCacheValidator() + } + // Tombstones may ride either annotation — the replay annotation on a + // round's first page, the scope annotation on every page (so a + // multi-page delta round never buffers deletions). Fresh slices: an + // append onto the proto's returned slice could write into its backing + // array when it has spare capacity. + page.deletedIDs = append(append([]string{}, replay.GetDeletedIds()...), scope.GetDeletedIds()...) + page.deletedPrincipalIDs = append(append([]string{}, replay.GetDeletedPrincipalIds()...), scope.GetDeletedPrincipalIds()...) + + if hasReplay { + if err := s.executeScopeReplay(ctx, kind, page, replay, rowCount); err != nil { + return ctx, nil, err + } + } + + return sourcecache.WithScope(ctx, page.scopeKey), page, nil +} + +// executeScopeReplay is beginSourceCachePage's replay step: validate the +// replay request against this sync's state, copy the previous sync's +// rows for the scope, and record the page-level bookkeeping. The page's +// scope key, validator, and tombstones have already been parsed. +func (s *syncer) executeScopeReplay(ctx context.Context, kind sourcecache.RowKind, page *sourceCachePage, replay *v2.SourceCacheReplay, rowCount int) error { + if s.sourceCache.prev == nil { + // Same invariant violation as the disabled case: the connector + // can only have gotten a lookup hit if a previous source exists. + return fmt.Errorf("source cache: connector requested replay for scope %q but no previous sync is available", replay.GetScopeKey()) + } + page.replayed = true + if !replay.GetOverlay() && rowCount > 0 { + // The contract says a 304-style replay page is empty, but rows + // arriving here are more data, not less — upsert them on top of + // the replayed base (overlay semantics) rather than failing the + // sync. Kept lenient while the model is proven against real + // providers. + ctxzap.Extract(ctx).Warn("source cache: non-overlay replay returned rows; treating them as an overlay", + zap.String("scope_key", page.scopeKey), + zap.Int("rows", rowCount), + ) + } + // Advisory check: a well-behaved connector only replays a scope + // whose validator came from this sync's lookup, so a missing + // previous manifest entry is suspicious — but not by itself data + // loss (the stamped rows may still exist, e.g. a partially carried + // file). The hard error below is reserved for a replay that + // produces nothing. + _, entryFound, err := s.sourceCache.prev.LookupSourceCacheEntry(ctx, kind, page.scopeKey) + if err != nil { + // The PREVIOUS file failing to serve reads mid-sync means the + // replay source itself is sick; retiring it and re-running cold + // (the runners' ErrReplayIntegrity handling) is the definitive + // remediation. + return fmt.Errorf("source cache: error reading previous manifest for scope %q: %w: %w", page.scopeKey, err, ErrReplayIntegrity) + } + if !entryFound { + ctxzap.Extract(ctx).Warn("source cache: replay requested for scope with no previous manifest entry", + zap.String("scope_key", page.scopeKey)) + } + replayStart := time.Now() + res, err := s.sourceCache.current.ReplaySourceCache(ctx, s.sourceCache.prevReader, kind, page.scopeKey) + s.state.AddStepDuration("source_cache_replay", time.Since(replayStart)) + if err != nil { + // Classified as a replay-integrity failure: whether the copy + // tripped its own count check or plain I/O died mid-copy, the + // replay state can't be trusted and a cold re-run is the safe + // remediation (see ErrReplayIntegrity). + return fmt.Errorf("source cache: replay for scope %q failed: %w: %w", page.scopeKey, err, ErrReplayIntegrity) + } + // Stats are stashed on the page and recorded in + // finishSourceCachePage so a page re-run after failing between + // replay and finish doesn't double-count. + page.replayRows = res.Rows + page.overlayRows = rowCount + if replay.GetOverlay() || page.cacheValidator == "" { + // This replay round can span pages: the validator arrives on + // a LATER page's scope annotation (delta rounds), and that + // page must count toward ScopesReplayed's scope, not as a + // freshly stamped one — see finishSourceCachePage. + s.sourceCache.markScopeReplayed(kind, page.scopeKey) + } + if res.Rows == 0 && !entryFound { + // The connector skipped row generation expecting a base that + // does not exist anywhere in the previous file — this sync + // would silently drop the scope's rows. + return fmt.Errorf("source cache: replay for scope %q found no previous rows and no manifest entry; the connector replayed a scope it never looked up", page.scopeKey) + } + // Empty-vs-dropped discrimination via the previous file's scope + // index: a legitimately empty scope has NO index entries + // (StaleSkipped == 0, fine to proceed); a scope whose index says + // rows existed but none carried a matching value stamp means the + // rows were rewritten without index cleanup — proceeding would + // silently drop the scope's contents from this sync. + if res.Rows == 0 && res.StaleSkipped > 0 { + return fmt.Errorf( + "source cache: replay for scope %q copied no rows but the previous file's scope index holds %d entries "+ + "whose rows no longer carry this scope's stamp; refusing to silently drop the scope: %w", + page.scopeKey, res.StaleSkipped, ErrReplayIntegrity) + } + if res.StaleSkipped > 0 { + // Partial staleness: some rows copied, some index entries were + // dead — a writer skipped index cleanup somewhere upstream. The + // replay policy treats a stale replay index as untrustworthy + // wholesale: discard the warm output and re-run cold rather than + // guess which side of the divergence is right. + return fmt.Errorf( + "source cache: replay for scope %q skipped %d stale scope-index entries (rows copied: %d); "+ + "the previous file's replay index cannot be trusted: %w", + page.scopeKey, res.StaleSkipped, res.Rows, ErrReplayIntegrity) + } + if res.ResumedRowsCleared > 0 { + // A re-run over a crashed attempt's committed rows: the engine + // cleared them before re-copying so the copy is idempotent. + // Expected after a resume; noteworthy otherwise. + ctxzap.Extract(ctx).Info("source cache: replay re-run cleared rows committed by a previous attempt", + zap.String("scope_key", page.scopeKey), + zap.Int64("rows_cleared", res.ResumedRowsCleared), + zap.Int64("rows_copied", res.Rows), + ) + } + // Side effects for replayed rows are STORE-DERIVED, not armed + // here — the engine maintains the needs_expansion index and the + // external-match existence bit for replayed rows exactly as for + // fresh ones, and the ingestion invariants read the store at + // their consuming seams (expansion: the PendingExpansion probe at + // SyncGrantExpansionOp; external match: the fact probe at + // SyncExternalResourcesOp; exclusion groups: the post-collection + // keyspace validation). See + // docs/tasks/source-cache-ingestion-invariants.md. The one + // replay-carried side effect is child scheduling, which cannot be + // derived after the fact (a zero-row child listing leaves no + // store evidence), so replay carries the parents' child types: + if kind == sourcecache.RowKindResources { + for _, parent := range res.ChildResources { + s.pushChildResourceActions(ctx, parent.ChildTypeIDs, parent.ResourceTypeID, parent.ResourceID) + } + } + ctxzap.Extract(ctx).Debug("source cache replayed scope", + zap.String("row_kind", string(kind)), + zap.String("scope_key", page.scopeKey), + zap.Int64("rows", res.Rows), + zap.Int("deleted_ids", len(page.deletedIDs)), + zap.Int("deleted_principal_ids", len(page.deletedPrincipalIDs)), + ) + if s.testSourceCacheHaltHook != nil { + if err := s.testSourceCacheHaltHook("replay-copied", page.scopeKey); err != nil { + return err + } + } + return nil +} + +// finishSourceCachePage runs after the page's rows committed: applies +// delta tombstones and, when the page carried a validator, writes the +// current sync's manifest entry. The entry write is last so a failed page +// can never leave a phantom hit for the next sync. +func (s *syncer) finishSourceCachePage(ctx context.Context, page *sourceCachePage) error { + if page == nil { + return nil + } + if s.testSourceCacheHaltHook != nil { + if err := s.testSourceCacheHaltHook("rows-committed", page.scopeKey); err != nil { + return err + } + } + var rowsDeleted int64 + if len(page.deletedIDs)+len(page.deletedPrincipalIDs) > 0 { + tombstoneStart := time.Now() + defer func() { + s.state.AddStepDuration("source_cache_tombstones", time.Since(tombstoneStart)) + }() + deleted, err := s.sourceCache.current.ApplyTombstones(ctx, page.kind, page.scopeKey, page.deletedIDs, page.deletedPrincipalIDs) + if err != nil { + return fmt.Errorf("source cache: error applying tombstones for scope %q: %w", page.scopeKey, err) + } + rowsDeleted = deleted + ctxzap.Extract(ctx).Debug("source cache applied tombstones", + zap.String("row_kind", string(page.kind)), + zap.String("scope_key", page.scopeKey), + zap.Int("tombstone_ids", len(page.deletedIDs)), + zap.Int("tombstone_principals", len(page.deletedPrincipalIDs)), + zap.Int64("rows_deleted", deleted), + ) + } + if s.testSourceCacheHaltHook != nil { + if err := s.testSourceCacheHaltHook("tombstones-applied", page.scopeKey); err != nil { + return err + } + } + if page.cacheValidator != "" { + if err := s.sourceCache.current.PutSourceCacheEntry(ctx, page.kind, page.scopeKey, page.cacheValidator); err != nil { + return fmt.Errorf("source cache: error writing manifest entry for scope %q: %w", page.scopeKey, err) + } + if s.testSourceCacheHaltHook != nil { + if err := s.testSourceCacheHaltHook("manifest-written", page.scopeKey); err != nil { + return err + } + } + } + + stats := SourceCacheStats{ + TombstoneIDs: int64(len(page.deletedIDs)), + TombstonePrincipals: int64(len(page.deletedPrincipalIDs)), + RowsDeleted: rowsDeleted, + } + if page.replayed { + // Recorded here rather than in beginSourceCachePage so a page + // re-run after failing between replay and finish doesn't + // double-count the scope. + stats.ScopesReplayed = 1 + stats.RowsReplayed = map[sourcecache.RowKind]int64{page.kind: page.replayRows} + stats.OverlayRows = int64(page.overlayRows) + } + if page.cacheValidator != "" && !page.replayed && !s.sourceCache.scopeReplayedThisSync(page.kind, page.scopeKey) { + // A cold/fresh page that persisted a validator: the denominator + // of the replay hit ratio. The replayed-scope check keeps a + // multi-page overlay round — whose new validator legally arrives + // on the final page, without a replay annotation — from booking + // its own hit as a miss too. + stats.ScopesStamped = 1 + } + if stats.TombstoneIDs > 0 || stats.TombstonePrincipals > 0 || stats.RowsDeleted > 0 || stats.ScopesStamped > 0 || page.replayed { + s.state.AddSourceCacheStats(stats) + } + return nil +} diff --git a/pkg/sync/source_cache_bench_test.go b/pkg/sync/source_cache_bench_test.go new file mode 100644 index 000000000..b53e99ca5 --- /dev/null +++ b/pkg/sync/source_cache_bench_test.go @@ -0,0 +1,134 @@ +package sync //nolint:revive,nolintlint // we can't change the package name for backwards compatibility + +// Source-cache path benchmarks: the write-side cost of scope stamping +// (inline by_source_scope index + manifest writes + the seal-time I6 +// scan) and the replay path (engine copy + the replay-time count check), +// against an unstamped cold sync of the same rows as the baseline. +// +// go test -bench=BenchmarkSourceCacheSync -benchtime=1x -count=3 \ +// -run='^$' ./pkg/sync/ +// +// Shape: nGroups groups × membersPerGroup member grants, one grants +// scope per group (the per-resource conditional-request shape), users +// listed unstamped — a realistic mixed sync. The warm variant replays +// every scope (all 304s): the maximum-replay / maximum-count-check case. + +import ( + "context" + "fmt" + "os" + "path/filepath" + "testing" + "time" + + "github.com/conductorone/baton-sdk/pkg/dotc1z" + "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" + "github.com/conductorone/baton-sdk/pkg/logging" +) + +const ( + benchGroups = 500 + benchUsers = 5_000 + benchMembersPerGroup = 50 // 500 × 50 = 25k grants, 500 scopes +) + +func buildSourceCacheBenchConnector(b *testing.B, ctx context.Context, stamped bool) *sourceCacheMockConnector { + b.Helper() + mc := newSourceCacheMockConnector() + users := make([]string, benchUsers) + for i := range benchUsers { + u, err := mc.AddUser(ctx, fmt.Sprintf("u-%05d", i)) + if err != nil { + b.Fatalf("AddUser: %v", err) + } + users[i] = u.GetId().GetResource() + _ = users[i] + } + userRes := mc.resourceDB[userResourceType.GetId()] + for gi := range benchGroups { + g, _, err := mc.AddGroup(ctx, fmt.Sprintf("g-%04d", gi)) + if err != nil { + b.Fatalf("AddGroup: %v", err) + } + for m := range benchMembersPerGroup { + mc.AddGroupMember(ctx, g, userRes[(gi*benchMembersPerGroup+m)%len(userRes)]) + } + if stamped { + mc.etagByResource[g.GetId().GetResource()] = "v1" + } + } + return mc +} + +func benchOneSync(b *testing.B, ctx context.Context, cc *sourceCacheMockConnector, path, prevPath, tmpDir string) { + b.Helper() + store, err := dotc1z.NewStore(ctx, path, + dotc1z.WithEngine(c1zstore.EnginePebble), + dotc1z.WithTmpDir(tmpDir), + ) + if err != nil { + b.Fatalf("NewStore: %v", err) + } + opts := []SyncOpt{WithConnectorStore(store), WithTmpDir(tmpDir)} + if prevPath != "" { + opts = append(opts, WithPreviousSyncC1ZPath(prevPath)) + } + syncer, nerr := NewSyncer(ctx, cc, opts...) + if nerr != nil { + b.Fatalf("NewSyncer: %v", nerr) + } + if err := syncer.Sync(ctx); err != nil { + b.Fatalf("Sync: %v", err) + } + if err := syncer.Close(ctx); err != nil { + b.Fatalf("Close: %v", err) + } +} + +func BenchmarkSourceCacheSync(b *testing.B) { + ctx, err := logging.Init(context.Background(), logging.WithLogLevel("error")) + if err != nil { + b.Fatalf("logging.Init: %v", err) + } + + b.Run("cold_unstamped", func(b *testing.B) { + mc := buildSourceCacheBenchConnector(b, ctx, false) + tmpDir := b.TempDir() + b.ResetTimer() + for i := 0; i < b.N; i++ { + start := time.Now() + benchOneSync(b, ctx, mc, filepath.Join(tmpDir, fmt.Sprintf("cold-un-%d.c1z", i)), "", tmpDir) + b.ReportMetric(float64(time.Since(start).Milliseconds()), "ms/sync") + } + }) + + b.Run("cold_stamped", func(b *testing.B) { + mc := buildSourceCacheBenchConnector(b, ctx, true) + tmpDir := b.TempDir() + b.ResetTimer() + for i := 0; i < b.N; i++ { + start := time.Now() + path := filepath.Join(tmpDir, fmt.Sprintf("cold-st-%d.c1z", i)) + benchOneSync(b, ctx, mc, path, "", tmpDir) + b.ReportMetric(float64(time.Since(start).Milliseconds()), "ms/sync") + if i == 0 { + if fi, err := os.Stat(path); err == nil { + b.ReportMetric(float64(fi.Size())/(1<<20), "MB/c1z") + } + } + } + }) + + b.Run("warm_replay", func(b *testing.B) { + mc := buildSourceCacheBenchConnector(b, ctx, true) + tmpDir := b.TempDir() + prevPath := filepath.Join(tmpDir, "prev.c1z") + benchOneSync(b, ctx, mc, prevPath, "", tmpDir) // seed: fresh fetch, stamps + manifest + b.ResetTimer() + for i := 0; i < b.N; i++ { + start := time.Now() + benchOneSync(b, ctx, mc, filepath.Join(tmpDir, fmt.Sprintf("warm-%d.c1z", i)), prevPath, tmpDir) + b.ReportMetric(float64(time.Since(start).Milliseconds()), "ms/sync") + } + }) +} diff --git a/pkg/sync/source_cache_continuation.go b/pkg/sync/source_cache_continuation.go new file mode 100644 index 000000000..2cd64b62d --- /dev/null +++ b/pkg/sync/source_cache_continuation.go @@ -0,0 +1,269 @@ +package sync //nolint:revive,nolintlint // we can't change the package name for backwards compatibility + +// Syncer side of the source-cache lookup continuation (ask/answer). On +// single-shot transports (gRPC-over-Lambda) the connector cannot call the +// lookup service mid-request, so it answers a list RPC with a +// SourceCacheLookupAsk instead of rows; the syncer resolves the queries +// against its LOCAL previous-sync store — the same store replay copies +// from, so lookup and replay can never disagree — and re-invokes the same +// request with SourceCacheLookupAnswers attached. See +// docs/tasks/source-cache-lambda-lookup.md and the annotation contract in +// proto/c1/connector/v2/annotation_source_cache.proto. + +import ( + "context" + "fmt" + + "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" + "go.uber.org/zap" + + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + "github.com/conductorone/baton-sdk/pkg/annotations" + "github.com/conductorone/baton-sdk/pkg/sourcecache" +) + +const ( + // sourceCacheBounceCap bounds consecutive asks for the SAME request + // (same page token; only the answers annotation differs between + // re-invokes). A connector that keeps asking without progressing is + // broken (most commonly: swallowing ErrLookupDeferred and re-asking + // for scopes it already "handled"), and silence would be the + // stale-data failure mode — so fail loudly. + // + // Deliberately NOT per action: a multi-page action that asks once per + // page (e.g. a delta planner pre-resolving each planning page's chunk + // scopes) bounces once per page with monotonic progress — every + // NextPageToken advance is a new request and resets the counter. + // + // Shared with pkg/sourcecache so the connector-side answer-set cap + // (bounces × ask size) stays coherent with this loop's enforcement. + sourceCacheBounceCap = sourcecache.MaxLookupBouncesPerRequest + + // maxEnqueuePageTokensPerResponse bounds EnqueuePageTokens fan-out per response + // (each spawned action persists in the checkpointed state token, so + // unbounded fan-out bloats every checkpoint). Matches the proto + // max_items validate rule on EnqueuePageTokens.page_tokens. + maxEnqueuePageTokensPerResponse = 1024 + // maxEnqueuedPageTokenBytes matches the page-token request field and + // the EnqueuePageTokens item validation. Enforced before creating Actions: + // action tokens are checkpointed before they are sent through the + // later request field's validator. + maxEnqueuedPageTokenBytes = 1 << 20 + + // maxEnqueuedPageTokenTotalBytes bounds the AGGREGATE token bytes of + // one EnqueuePageTokens response. The per-item caps alone still admit + // 1024 × 1MiB = 1GiB of tokens in a single legal response — all of it + // persisted into every subsequent checkpoint token until the actions + // drain. Real planner tokens are tiny (chunk ids, cursor strings); + // 16MiB of aggregate budget is orders of magnitude above any sane + // fan-out while keeping worst-case checkpoints bounded. + maxEnqueuedPageTokenTotalBytes = 16 << 20 + + // sourceCacheAnswerBudget caps the TOTAL answers payload attached to + // one re-invoke — validator bytes AND per-answer identity bytes + // (row_kind, scope_key, proto framing) — keeping the request under + // single-shot transport payload limits (Lambda invokes cap at 6MB; + // the dual-encoded frame at 5MiB). Identity bytes must be charged + // too: at the proto caps (16384 accumulated answers × 256-byte scope + // keys) identities alone reach ~5MiB with zero validator bytes, so a + // validator-only budget could not actually bound the request. + // + // A found answer whose etag does not fit the remaining budget is + // DEGRADED to a not-found answer: the connector fetches that scope + // fresh (cold-correct, just slower). Leaving it absent-and-re-askable + // would livelock — answers accumulate across bounces (the connector + // re-executes from scratch each phase, so every re-invoke carries the + // union), so the budget only shrinks and an etag that did not fit once + // can never fit later; re-asking would burn straight into the bounce + // cap and hard-fail a sync that cold-degrades correctly. Identity + // bytes are NOT degradable (a not-found answer carries the same + // identity, and an absent answer livelocks), so identity bytes alone + // exceeding the budget is a hard failure — the connector asked for an + // answer set no re-invoke could ever carry. + sourceCacheAnswerBudget = 2 << 20 + + // sourceCacheAnswerOverheadBytes approximates one answer's non-string + // proto framing (field tags, length prefixes, the found bool, the Any + // wrapping amortized across the message). Deliberately generous: the + // budget exists to stay under transport limits, not to be exact. + sourceCacheAnswerOverheadBytes = 16 +) + +// Continuation counters live in SourceCacheStats on the sync state (see +// source_cache_stats.go): token-persisted so a resumed sync keeps them, +// reported once in the source-cache summary line. + +// listAttempt is one list-RPC attempt as observed by the continuation +// loop: enough of the response to detect and validate an ask. +type listAttempt struct { + annos annotations.Annotations + rows int + nextToken string +} + +// continuationCallMethod labels connector-call latency stats: re-invoked +// attempts (answer turns) get a ":cont" suffix so millisecond protocol +// turns don't blend into the base method's upstream-fetch distribution. +func continuationCallMethod(method string, attempt int) string { + if attempt > 0 { + return method + ":cont" + } + return method +} + +// withSourceCacheContinuation drives the ask/answer loop around one list +// RPC. issue performs the RPC with extra request annotations (the lookup +// offer, plus accumulated answers on re-invokes) and reports the response +// surface; the loop returns once a response carries no ask — that final +// response is the one the caller processes. attempt is 0 for the first +// issue and increments per re-invoke, so callers can label protocol turns +// separately in connector-call latency stats (a re-invoke that replays is +// not comparable to a cold upstream fetch). +// +// The offer is attached only when the syncer can actually answer (warm +// previous-sync lookup). Old or cold syncers never send it, and a +// compliant connector never asks without it — that pairing is what makes +// version skew degrade to a cold sync instead of a misread response. +func (s *syncer) withSourceCacheContinuation(ctx context.Context, op string, issue func(extra annotations.Annotations, attempt int) (listAttempt, error)) error { + l := ctxzap.Extract(ctx) + + warm := s.sourceCache.prev != nil + extra := annotations.Annotations{} + if warm { + extra.Update(&v2.SourceCacheLookupOffer{}) + } + + // Answers accumulate across bounces: the connector re-executes from + // scratch each phase, so every re-invoke must carry the union of all + // resolved queries, in first-resolved order (deterministic requests). + var ordered []sourcecache.Answer + seen := map[sourcecache.Query]bool{} + + asked, found, notFound, truncated := 0, 0, 0, 0 + for bounce := 0; ; bounce++ { + attempt, err := issue(extra, bounce) + if err != nil { + return err + } + + ask := &v2.SourceCacheLookupAsk{} + hasAsk, err := attempt.annos.Pick(ask) + if err != nil { + return fmt.Errorf("%s: error parsing source-cache lookup ask: %w", op, err) + } + if !hasAsk { + if bounce > 0 { + s.state.AddSourceCacheStats(SourceCacheStats{ + LookupBounces: int64(bounce), + LookupRequestsBounced: 1, + LookupScopesAsked: int64(asked), + LookupAnsweredFound: int64(found), + LookupAnsweredNotFound: int64(notFound), + LookupAnswersTruncated: int64(truncated), + LookupBouncesByOp: map[string]int64{op: int64(bounce)}, + }) + } + return nil + } + + // Ask legality. Failing loudly here is deliberate: every branch + // is a connector bug that would otherwise surface as silently + // wrong data or an unexplained stall. + if !warm { + return fmt.Errorf("%s: connector sent a source-cache lookup ask on a request that carried no offer (connector must gate asks on SourceCacheLookupOffer)", op) + } + if attempt.rows > 0 || attempt.nextToken != "" || + attempt.annos.Contains(&v2.SourceCacheRecord{}) || attempt.annos.Contains(&v2.SourceCacheReplay{}) || + attempt.annos.Contains(&v2.EnqueuePageTokens{}) { + return fmt.Errorf("%s: source-cache lookup ask response must carry ONLY the ask: "+ + "no rows, no next page token, no scope/replay annotations, no spawned cursors "+ + "(spawn on the re-invoked request's real response instead)", op) + } + if bounce >= sourceCacheBounceCap { + s.state.AddSourceCacheStats(SourceCacheStats{LookupCapFailures: 1}) + return fmt.Errorf("%s: source-cache lookup bounce cap (%d) exceeded for one request; "+ + "connector kept asking without progressing (%d scopes still unresolved) — "+ + "check for swallowed ErrLookupDeferred or nondeterministic scope computation", + op, sourceCacheBounceCap, len(ask.GetQueries())) + } + + queries, err := sourcecache.QueriesFromProto(ask) + if err != nil { + return fmt.Errorf("%s: invalid source-cache lookup ask: %w", op, err) + } + + answerIdentityCost := func(q sourcecache.Query) int { + return len(q.RowKind) + len(q.ScopeKey) + sourceCacheAnswerOverheadBytes + } + budget := sourceCacheAnswerBudget + for _, a := range ordered { + budget -= answerIdentityCost(a.Query) + len(a.CacheValidator) + } + newAsked, newFound, newNotFound, newTruncated := 0, 0, 0, 0 + for _, q := range queries { + if seen[q] { + continue + } + newAsked++ + // Identity bytes are spent for EVERY answered query — found, + // not-found, and degraded alike (the answer must exist or the + // connector livelocks re-asking it). If identities alone blow + // the budget, no degradation can produce a carryable request. + identityCost := answerIdentityCost(q) + if identityCost > budget { + s.state.AddSourceCacheStats(SourceCacheStats{LookupCapFailures: 1}) + return fmt.Errorf("%s: source-cache lookup answers exceed the %d-byte request budget on identity bytes alone "+ + "(%d answers accumulated); the connector's asks name more scope bytes than any re-invoke can carry", + op, sourceCacheAnswerBudget, len(ordered)) + } + budget -= identityCost + entry, ok, err := s.sourceCache.lookup.Lookup(ctx, q.RowKind, q.ScopeKey) + if err != nil { + return fmt.Errorf("%s: resolving source-cache lookup ask: %w", op, err) + } + if !ok { + seen[q] = true + ordered = append(ordered, sourcecache.Answer{Query: q, Found: false}) + newNotFound++ + continue + } + if len(entry.CacheValidator) > budget { + // Found, but the etag does not fit the remaining budget. + // Degrade to an explicit not-found so the connector + // fetches this scope fresh (cold-correct). See the + // sourceCacheAnswerBudget comment: an absent answer could + // never be resolved on a later bounce, only livelock into + // the bounce cap. + seen[q] = true + ordered = append(ordered, sourcecache.Answer{Query: q, Found: false}) + newTruncated++ + continue + } + budget -= len(entry.CacheValidator) + seen[q] = true + ordered = append(ordered, sourcecache.Answer{Query: q, Found: true, CacheValidator: entry.CacheValidator}) + newFound++ + } + asked += newAsked + found += newFound + notFound += newNotFound + truncated += newTruncated + + if newAsked == 0 { + // Every query was already answered on the request the + // connector just saw; re-invoking cannot make progress. + return fmt.Errorf("%s: connector re-asked only already-answered scopes (%d queries); connector lookup handling is broken", op, len(queries)) + } + + extra.Update(sourcecache.AnswersProto(ordered)) + + l.Debug("source-cache lookup bounce", + zap.String("op", op), + zap.Int("bounce", bounce+1), + zap.Int("asked", newAsked), + zap.Int("found", newFound), + zap.Int("not_found", newNotFound), + zap.Int("degraded_to_cold_for_budget", newTruncated), + ) + } +} diff --git a/pkg/sync/source_cache_continuation_e2e_test.go b/pkg/sync/source_cache_continuation_e2e_test.go new file mode 100644 index 000000000..5a50a7fc4 --- /dev/null +++ b/pkg/sync/source_cache_continuation_e2e_test.go @@ -0,0 +1,257 @@ +package sync //nolint:revive,nolintlint // we can't change the package name for backwards compatibility + +// Composed end-to-end test for the lookup continuation over the REAL +// Lambda stack: a connectorbuilder-built connector (unchanged connector +// code, lookup-before-fetch, errors propagated) registered on the actual +// gRPC-over-Lambda server, reached through the actual transport encodings +// (Request/Response MarshalJSON/UnmarshalJSON — dual-encoded requests, +// frame/legacy responses), driven by the real syncer. This is the +// production Lambda topology minus AWS: every layer between the syncer's +// bounce loop and the connector's SyncOpAttrs lookup is the shipping code +// path, including the wire round trip of the Offer/Ask/Answers +// annotations. + +import ( + "context" + "encoding/json" + "fmt" + "path/filepath" + "sync" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/conductorone/baton-sdk/internal/connector" + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + "github.com/conductorone/baton-sdk/pkg/annotations" + "github.com/conductorone/baton-sdk/pkg/connectorbuilder" + "github.com/conductorone/baton-sdk/pkg/connectorclient" + c1lambda "github.com/conductorone/baton-sdk/pkg/lambda/grpc" + "github.com/conductorone/baton-sdk/pkg/logging" + "github.com/conductorone/baton-sdk/pkg/sourcecache" + et "github.com/conductorone/baton-sdk/pkg/types/entitlement" + gt "github.com/conductorone/baton-sdk/pkg/types/grant" + rs "github.com/conductorone/baton-sdk/pkg/types/resource" +) + +// jsonWireTransport is the Lambda invoke minus AWS: the request is +// marshaled to the invoke payload and unmarshaled server-side, the +// response likewise back — both through the exact MarshalJSON / +// UnmarshalJSON paths the production transport uses. +type jsonWireTransport struct { + server *c1lambda.Server +} + +func (t jsonWireTransport) RoundTrip(ctx context.Context, req *c1lambda.Request) (*c1lambda.Response, error) { + payload, err := json.Marshal(req) + if err != nil { + return nil, fmt.Errorf("wire: marshal request: %w", err) + } + serverReq := &c1lambda.Request{} + if err := json.Unmarshal(payload, serverReq); err != nil { + return nil, fmt.Errorf("wire: unmarshal request: %w", err) + } + resp, err := t.server.Handler(ctx, serverReq) + if err != nil { + return nil, err + } + respPayload, err := json.Marshal(resp) + if err != nil { + return nil, fmt.Errorf("wire: marshal response: %w", err) + } + clientResp := &c1lambda.Response{} + if err := json.Unmarshal(respPayload, clientResp); err != nil { + return nil, fmt.Errorf("wire: unmarshal response: %w", err) + } + return clientResp, nil +} + +// lambdaE2EBuilder is a plain ConnectorBuilderV2 — the shape a real +// connector ships. No SetLookup, no test hooks in the sync path. +type lambdaE2EBuilder struct { + syncer *lambdaE2ESyncer +} + +func (b *lambdaE2EBuilder) Metadata(context.Context) (*v2.ConnectorMetadata, error) { + return v2.ConnectorMetadata_builder{DisplayName: "lambda-e2e"}.Build(), nil +} + +func (b *lambdaE2EBuilder) Validate(context.Context) (annotations.Annotations, error) { + return annotations.New(v2.SourceCacheCapability_builder{ + Mode: v2.SourceCacheCapability_MODE_READ_WRITE, + }.Build()), nil +} + +func (b *lambdaE2EBuilder) ResourceSyncers(context.Context) []connectorbuilder.ResourceSyncerV2 { + return []connectorbuilder.ResourceSyncerV2{b.syncer, &lambdaE2EUserSyncer{}} +} + +type lambdaE2EUserSyncer struct{} + +func (s *lambdaE2EUserSyncer) ResourceType(context.Context) *v2.ResourceType { + return userResourceType +} + +func (s *lambdaE2EUserSyncer) List(_ context.Context, _ *v2.ResourceId, _ rs.SyncOpAttrs) ([]*v2.Resource, *rs.SyncOpResults, error) { + var out []*v2.Resource + for i := 0; i < 4; i++ { + u, err := rs.NewUserResource(fmt.Sprintf("user-%02d", i), userResourceType, fmt.Sprintf("user-%02d", i), nil, + rs.WithAnnotation(&v2.SkipEntitlementsAndGrants{})) + if err != nil { + return nil, nil, err + } + out = append(out, u) + } + return out, &rs.SyncOpResults{}, nil +} + +func (s *lambdaE2EUserSyncer) Entitlements(context.Context, *v2.Resource, rs.SyncOpAttrs) ([]*v2.Entitlement, *rs.SyncOpResults, error) { + return nil, &rs.SyncOpResults{}, nil +} + +func (s *lambdaE2EUserSyncer) Grants(context.Context, *v2.Resource, rs.SyncOpAttrs) ([]*v2.Grant, *rs.SyncOpResults, error) { + return nil, &rs.SyncOpResults{}, nil +} + +// lambdaE2ESyncer serves group member grants with per-group scopes, +// written exactly the way the contract tells connector authors to: +// lookup FIRST (propagating errors with %w), replay on hit, cold + scope +// on miss. +type lambdaE2ESyncer struct { + mu sync.Mutex + members map[string][]string + etagByGroup map[string]string + lookups int + replays int + colds int +} + +func (s *lambdaE2ESyncer) ResourceType(context.Context) *v2.ResourceType { + return groupResourceType +} + +func (s *lambdaE2ESyncer) group(gid string) (*v2.Resource, error) { + return rs.NewGroupResource(gid, groupResourceType, gid, nil) +} + +func (s *lambdaE2ESyncer) List(_ context.Context, _ *v2.ResourceId, _ rs.SyncOpAttrs) ([]*v2.Resource, *rs.SyncOpResults, error) { + s.mu.Lock() + defer s.mu.Unlock() + var out []*v2.Resource + for gid := range s.members { + g, err := s.group(gid) + if err != nil { + return nil, nil, err + } + out = append(out, g) + } + return out, &rs.SyncOpResults{}, nil +} + +func (s *lambdaE2ESyncer) Entitlements(_ context.Context, r *v2.Resource, _ rs.SyncOpAttrs) ([]*v2.Entitlement, *rs.SyncOpResults, error) { + ent := et.NewAssignmentEntitlement(r, "member", et.WithGrantableTo(userResourceType)) + ent.SetSlug("member") + return []*v2.Entitlement{ent}, &rs.SyncOpResults{}, nil +} + +func (s *lambdaE2ESyncer) Grants(ctx context.Context, r *v2.Resource, opts rs.SyncOpAttrs) ([]*v2.Grant, *rs.SyncOpResults, error) { + gid := r.GetId().GetResource() + scope := "e2e/groups/" + gid + "/members" + + s.mu.Lock() + s.lookups++ + etag := s.etagByGroup[gid] + uids := append([]string(nil), s.members[gid]...) + s.mu.Unlock() + + entry, found, err := opts.SourceCache.Lookup(ctx, sourcecache.RowKindGrants, scope) + if err != nil { + return nil, nil, fmt.Errorf("revalidating %s: %w", gid, err) + } + ret := &rs.SyncOpResults{Annotations: annotations.Annotations{}} + if found && entry.CacheValidator == etag { + s.mu.Lock() + s.replays++ + s.mu.Unlock() + ret.Annotations.Update(v2.SourceCacheReplay_builder{ScopeKey: scope, CacheValidator: etag}.Build()) + return nil, ret, nil + } + + s.mu.Lock() + s.colds++ + s.mu.Unlock() + grants := make([]*v2.Grant, 0, len(uids)) + for _, uid := range uids { + u, err := rs.NewUserResource(uid, userResourceType, uid, nil) + if err != nil { + return nil, nil, err + } + grants = append(grants, gt.NewGrant(r, "member", u.GetId())) + } + ret.Annotations.Update(v2.SourceCacheRecord_builder{ScopeKey: scope, CacheValidator: etag}.Build()) + return grants, ret, nil +} + +// TestSourceCacheContinuation_LambdaStackE2E composes the shipping layers: +// connectorbuilder (per-request ContinuationLookup + deferral interception) +// behind internal/connector.Register on the gRPC-over-Lambda server, +// through the real JSON wire encodings, under the real syncer bounce loop. +func TestSourceCacheContinuation_LambdaStackE2E(t *testing.T) { + ctx, err := logging.Init(context.Background()) + require.NoError(t, err) + tmpDir := t.TempDir() + + es := &lambdaE2ESyncer{ + members: map[string][]string{ + "group-00": {"user-00", "user-01"}, + "group-01": {"user-02", "user-03"}, + }, + etagByGroup: map[string]string{"group-00": "e1", "group-01": "e1"}, + } + cb := &lambdaE2EBuilder{syncer: es} + + // The Lambda server path: connectorbuilder WITHOUT any runner-supplied + // lookup (RunTimeOpts.SourceCacheLookup is not wired on Lambda), so + // the continuation is the only lookup transport available. + cs, err := connectorbuilder.NewConnector(ctx, cb) + require.NoError(t, err) + server := c1lambda.NewServer(nil) + connector.Register(ctx, server, cs, nil) + client := connectorclient.NewConnectorClient(ctx, c1lambda.NewClientConn(jsonWireTransport{server: server})) + + // Sync 1: cold. No previous c1z → no offer → the connector's lookup is + // NoopLookup; everything fetches cold with zero bounces. + sync1 := filepath.Join(tmpDir, "sync1.c1z") + require.NoError(t, runContinuationSync(ctx, t, client, sync1, "", tmpDir, 0)) + require.Equal(t, 2, es.colds) + require.Zero(t, es.replays) + grants1 := listGrantsInFile(ctx, t, sync1) + require.Len(t, grants1, 4) + + // Sync 2: warm. The offer rides the wire, the connector's first + // execution defers (ErrLookupDeferred through the builder, ask through + // the transport), the re-invoke carries answers, and both groups + // replay. lookups counts handler EXECUTIONS: 2 phase-1 + 2 phase-2. + sync2 := filepath.Join(tmpDir, "sync2.c1z") + require.NoError(t, runContinuationSync(ctx, t, client, sync2, sync1, tmpDir, 0)) + require.Equal(t, 2, es.replays, "both groups replay via ask/answer over the wire") + require.Equal(t, 2, es.colds, "no cold fetches on the warm sync") + require.Equal(t, 6, es.lookups, "2 cold + 2 deferred phase-1 + 2 answered phase-2 handler executions") + requireSameGrantIDs(t, grants1, listGrantsInFile(ctx, t, sync2), "lambda-stack warm vs cold") + + // Churn one group; warm sync 3: stale answer → cold refetch for it, + // replay for the other; equivalence against a cold control. + es.mu.Lock() + es.members["group-01"] = []string{"user-02"} + es.etagByGroup["group-01"] = "e2" + es.mu.Unlock() + + sync3 := filepath.Join(tmpDir, "sync3.c1z") + require.NoError(t, runContinuationSync(ctx, t, client, sync3, sync2, tmpDir, 0)) + require.Equal(t, 3, es.replays) + require.Equal(t, 3, es.colds) + + control := filepath.Join(tmpDir, "control.c1z") + require.NoError(t, runContinuationSync(ctx, t, client, control, "", tmpDir, 0)) + requireSameGrantIDs(t, listGrantsInFile(ctx, t, control), listGrantsInFile(ctx, t, sync3), "churned lambda-stack warm vs cold control") +} diff --git a/pkg/sync/source_cache_continuation_test.go b/pkg/sync/source_cache_continuation_test.go new file mode 100644 index 000000000..91d146771 --- /dev/null +++ b/pkg/sync/source_cache_continuation_test.go @@ -0,0 +1,935 @@ +package sync //nolint:revive,nolintlint // we can't change the package name for backwards compatibility + +// Syncer-side tests for the source-cache lookup continuation (ask/answer). +// The mocks here play the LAMBDA side of the protocol by hand: they do NOT +// implement sourcecache.SourceCacheSetter (no direct lookup install — exactly the +// single-shot-transport topology), and instead answer list RPCs with +// SourceCacheLookupAsk and consume SourceCacheLookupAnswers from the +// re-invoked request. + +import ( + "fmt" + "path/filepath" + "strings" + "sync" + "testing" + + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + "github.com/conductorone/baton-sdk/pkg/annotations" + "github.com/conductorone/baton-sdk/pkg/dotc1z" + "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" + "github.com/conductorone/baton-sdk/pkg/logging" + "github.com/conductorone/baton-sdk/pkg/sourcecache" + "github.com/conductorone/baton-sdk/pkg/types" + et "github.com/conductorone/baton-sdk/pkg/types/entitlement" + gt "github.com/conductorone/baton-sdk/pkg/types/grant" + rs "github.com/conductorone/baton-sdk/pkg/types/resource" + + "context" +) + +func runContinuationSync(ctx context.Context, t *testing.T, mc types.ConnectorClient, path, prevPath, tmpDir string, workers int) error { + t.Helper() + store, err := dotc1z.NewStore(ctx, path, + dotc1z.WithEngine(c1zstore.EnginePebble), + dotc1z.WithTmpDir(tmpDir), + ) + require.NoError(t, err) + opts := []SyncOpt{WithConnectorStore(store), WithTmpDir(tmpDir)} + if prevPath != "" { + opts = append(opts, WithPreviousSyncC1ZPath(prevPath)) + } + if workers > 0 { + opts = append(opts, WithWorkerCount(workers)) + } + syncer, err := NewSyncer(ctx, mc, opts...) + require.NoError(t, err) + syncErr := syncer.Sync(ctx) + if syncErr != nil { + cctx, cancel := context.WithCancel(ctx) + cancel() + _ = syncer.Close(cctx) + return syncErr + } + require.NoError(t, syncer.Close(ctx)) + return nil +} + +// continuationMockConnector: one grants page per group, one scope per +// group. Lambda-side protocol by hand; misuse modes for the loud-failure +// paths. +type continuationMockConnector struct { + *mockConnector + + mu sync.Mutex + etagByGroup map[string]string + grantsByGroup map[string][]*v2.Grant + + askAlways bool // never satisfied: bounce-cap test + askWithRows bool // protocol violation: ask + rows + askNoOffer bool // protocol violation: ask on offerless request + askWithSpawn bool // protocol violation: ask + EnqueuePageTokens + + asks int + replayPages int + coldPages int + offerSeen bool +} + +func (mc *continuationMockConnector) Validate(context.Context, *v2.ConnectorServiceValidateRequest, ...grpc.CallOption) (*v2.ConnectorServiceValidateResponse, error) { + return v2.ConnectorServiceValidateResponse_builder{ + Annotations: annotations.New(v2.SourceCacheCapability_builder{ + Mode: v2.SourceCacheCapability_MODE_READ_WRITE, + }.Build()), + }.Build(), nil +} + +func groupScope(gid string) string { return "groups/" + gid + "/members" } + +func (mc *continuationMockConnector) askResponse(scope string, rows []*v2.Grant) *v2.GrantsServiceListGrantsResponse { + mc.mu.Lock() + mc.asks++ + mc.mu.Unlock() + return v2.GrantsServiceListGrantsResponse_builder{ + List: rows, + Annotations: annotations.New(sourcecache.AskProto([]sourcecache.Query{ + {RowKind: sourcecache.RowKindGrants, ScopeKey: scope}, + })), + }.Build() +} + +func (mc *continuationMockConnector) ListGrants(ctx context.Context, in *v2.GrantsServiceListGrantsRequest, _ ...grpc.CallOption) (*v2.GrantsServiceListGrantsResponse, error) { + gid := in.GetResource().GetId().GetResource() + if in.GetResource().GetId().GetResourceType() != groupResourceType.GetId() { + return v2.GrantsServiceListGrantsResponse_builder{}.Build(), nil + } + scope := groupScope(gid) + reqAnnos := annotations.Annotations(in.GetAnnotations()) + hasOffer := reqAnnos.Contains(&v2.SourceCacheLookupOffer{}) + answersMsg := &v2.SourceCacheLookupAnswers{} + hasAnswers, err := reqAnnos.Pick(answersMsg) + if err != nil { + return nil, err + } + mc.mu.Lock() + if hasOffer { + mc.offerSeen = true + } + etag := mc.etagByGroup[gid] + rows := mc.grantsByGroup[gid] + mc.mu.Unlock() + + if mc.askAlways { + return mc.askResponse(scope, nil), nil + } + if mc.askNoOffer && !hasAnswers { + return mc.askResponse(scope, nil), nil + } + if mc.askWithRows { + return mc.askResponse(scope, rows), nil + } + if mc.askWithSpawn && !hasAnswers && hasOffer { + resp := mc.askResponse(scope, nil) + annos := annotations.Annotations(resp.GetAnnotations()) + annos.Update(v2.EnqueuePageTokens_builder{PageTokens: []string{"bogus"}}.Build()) + resp.SetAnnotations(annos) + return resp, nil + } + + if hasAnswers { + answers, answersErr := sourcecache.AnswersFromProto(answersMsg) + if answersErr != nil { + return nil, answersErr + } + for _, a := range answers { + if a.ScopeKey != scope || a.RowKind != sourcecache.RowKindGrants { + continue + } + if a.Found && a.CacheValidator == etag { + mc.mu.Lock() + mc.replayPages++ + mc.mu.Unlock() + return v2.GrantsServiceListGrantsResponse_builder{ + Annotations: annotations.New(v2.SourceCacheReplay_builder{ + ScopeKey: scope, + CacheValidator: etag, + }.Build()), + }.Build(), nil + } + break // answered but stale/missing: cold below + } + } else if hasOffer { + // Lookup-before-fetch, deferred: phase 1 does no upstream work. + return mc.askResponse(scope, nil), nil + } + + mc.mu.Lock() + mc.coldPages++ + mc.mu.Unlock() + return v2.GrantsServiceListGrantsResponse_builder{ + List: rows, + Annotations: annotations.New(v2.SourceCacheRecord_builder{ + ScopeKey: scope, + CacheValidator: etag, + }.Build()), + }.Build(), nil +} + +func newContinuationMockConnector(t *testing.T, groups int) *continuationMockConnector { + t.Helper() + ctx := context.Background() + mc := &continuationMockConnector{ + mockConnector: newMockConnector(), + etagByGroup: map[string]string{}, + grantsByGroup: map[string][]*v2.Grant{}, + } + mc.rtDB = append(mc.rtDB, groupResourceType, userResourceType) + for i := 0; i < groups*2; i++ { + uid := fmt.Sprintf("user-%02d", i) + u, err := rs.NewUserResource(uid, userResourceType, uid, nil, rs.WithAnnotation(&v2.SkipEntitlementsAndGrants{})) + require.NoError(t, err) + mc.AddResource(ctx, u) + } + for i := 0; i < groups; i++ { + gid := fmt.Sprintf("group-%02d", i) + g, err := rs.NewGroupResource(gid, groupResourceType, gid, nil) + require.NoError(t, err) + mc.AddResource(ctx, g) + mc.entDB[gid] = []*v2.Entitlement{mkMemberEnt(g)} + mc.etagByGroup[gid] = "etag-v1" + for m := 0; m < 2; m++ { + uid := fmt.Sprintf("user-%02d", (i*2+m)%(groups*2)) + ur, err := rs.NewUserResource(uid, userResourceType, uid, nil) + require.NoError(t, err) + mc.grantsByGroup[gid] = append(mc.grantsByGroup[gid], gt.NewGrant(g, "member", ur.GetId())) + } + } + return mc +} + +// TestSourceCacheContinuation_WarmReplay is the whole protocol, end to +// end, against the real syncer loop: cold sync (no offer → no asks), warm +// sync (offer → one ask per group → answers → replay), and a third sync +// chained off the replay-only file (manifest carried forward through +// ask-path pages). +func TestSourceCacheContinuation_WarmReplay(t *testing.T) { + for _, workers := range []int{0, 4} { + t.Run(fmt.Sprintf("workers=%d", workers), func(t *testing.T) { + ctx, err := logging.Init(context.Background()) + require.NoError(t, err) + tmpDir := t.TempDir() + mc := newContinuationMockConnector(t, 5) + + sync1 := filepath.Join(tmpDir, "sync1.c1z") + require.NoError(t, runContinuationSync(ctx, t, mc, sync1, "", tmpDir, workers)) + require.False(t, mc.offerSeen, "cold sync (no previous c1z) must not offer lookups") + require.Zero(t, mc.asks, "no offer → a compliant connector never asks") + require.Equal(t, 5, mc.coldPages) + grants1 := listGrantsInFile(ctx, t, sync1) + require.Len(t, grants1, 10) + + sync2 := filepath.Join(tmpDir, "sync2.c1z") + require.NoError(t, runContinuationSync(ctx, t, mc, sync2, sync1, tmpDir, workers)) + require.True(t, mc.offerSeen) + require.Equal(t, 5, mc.asks, "one ask (bounce) per group page") + require.Equal(t, 5, mc.replayPages, "every page replays after its answer") + require.Equal(t, 5, mc.coldPages, "no cold fetches on the warm sync") + requireSameGrantIDs(t, grants1, listGrantsInFile(ctx, t, sync2), "warm-via-continuation vs cold") + + // Replay-only file remains a valid replay source. + sync3 := filepath.Join(tmpDir, "sync3.c1z") + require.NoError(t, runContinuationSync(ctx, t, mc, sync3, sync2, tmpDir, workers)) + require.Equal(t, 10, mc.replayPages) + requireSameGrantIDs(t, grants1, listGrantsInFile(ctx, t, sync3), "chained continuation sync") + + // Churn one group: its answer goes stale → cold refetch. + mc.mu.Lock() + g0 := mc.grantsByGroup["group-00"] + mc.grantsByGroup["group-00"] = g0[:1] + mc.etagByGroup["group-00"] = "etag-v2" + mc.mu.Unlock() + sync4 := filepath.Join(tmpDir, "sync4.c1z") + require.NoError(t, runContinuationSync(ctx, t, mc, sync4, sync3, tmpDir, workers)) + require.Equal(t, 6, mc.coldPages, "stale answer → exactly one cold refetch") + grants4 := listGrantsInFile(ctx, t, sync4) + require.Len(t, grants4, 9) + require.NotContains(t, grants4, g0[1].GetId()) + }) + } +} + +// verdictSpawnMock is the planner shape both consumer POCs asked for: the +// ORIGIN call batch-asks for every stored page's validator in ONE bounce +// (LookupMany semantics), then its phase-2 response serves page 0 and +// spawns sibling cursors whose tokens CARRY the verdicts. Siblings serve +// replay/cold from their token alone — zero asks, zero bounces. This pins +// the two contract commitments from the review briefs: the ask/answer loop +// composes with a phase-2 EnqueuePageTokens response, and a validator resolved +// in an EARLIER action of the same sync legally rides page tokens. +type verdictSpawnMock struct { + *mockConnector + + mu sync.Mutex + members map[string][]string // 2 rows per page + prevPages map[string]int + resByID map[string]*v2.Resource + + originAsks int + siblingAsks int + replays int + colds int +} + +func (mc *verdictSpawnMock) Validate(context.Context, *v2.ConnectorServiceValidateRequest, ...grpc.CallOption) (*v2.ConnectorServiceValidateResponse, error) { + return v2.ConnectorServiceValidateResponse_builder{ + Annotations: annotations.New(v2.SourceCacheCapability_builder{ + Mode: v2.SourceCacheCapability_MODE_READ_WRITE, + }.Build()), + }.Build(), nil +} + +func vsPageScope(gid string, page int) string { return fmt.Sprintf("g/%s/members?page=%d", gid, page) } + +func (mc *verdictSpawnMock) pageRows(gid string, page int) []string { + all := mc.members[gid] + lo, hi := page*2, page*2+2 + if lo >= len(all) { + return nil + } + if hi > len(all) { + hi = len(all) + } + return all[lo:hi] +} + +func vsPageEtag(gid string, page int, rows []string) string { + return fmt.Sprintf("v|%s|%d|%v", gid, page, rows) +} + +func (mc *verdictSpawnMock) servePage(gid string, page int, chain bool) *v2.GrantsServiceListGrantsResponse { + rows := mc.pageRows(gid, page) + grants := make([]*v2.Grant, 0, len(rows)) + g := mc.resByID[gid] + for _, uid := range rows { + grants = append(grants, gt.NewGrant(g, "member", mc.resByID[uid].GetId())) + } + next := "" + if chain && len(mc.pageRows(gid, page+1)) > 0 { + next = fmt.Sprintf("page=%d", page+1) + } + mc.colds++ + return v2.GrantsServiceListGrantsResponse_builder{ + List: grants, + Annotations: annotations.New(v2.SourceCacheRecord_builder{ + ScopeKey: vsPageScope(gid, page), + CacheValidator: vsPageEtag(gid, page, rows), + }.Build()), + NextPageToken: next, + }.Build() +} + +func (mc *verdictSpawnMock) ListGrants(_ context.Context, in *v2.GrantsServiceListGrantsRequest, _ ...grpc.CallOption) (*v2.GrantsServiceListGrantsResponse, error) { + gid := in.GetResource().GetId().GetResource() + if in.GetResource().GetId().GetResourceType() != groupResourceType.GetId() { + return v2.GrantsServiceListGrantsResponse_builder{}.Build(), nil + } + mc.mu.Lock() + defer mc.mu.Unlock() + + if tok := in.GetPageToken(); tok != "" { + // Sibling with a verdict-carrying token: "page=N|1|". + var page int + var found int + var etag string + if n, _ := fmt.Sscanf(tok, "page=%d|%d|", &page, &found); n == 2 { + if i := strings.Index(tok, "|"); i >= 0 { + if j := strings.Index(tok[i+1:], "|"); j >= 0 { + etag = tok[i+1+j+1:] + } + } + rows := mc.pageRows(gid, page) + if found == 1 && etag == vsPageEtag(gid, page, rows) { + mc.replays++ + return v2.GrantsServiceListGrantsResponse_builder{ + Annotations: annotations.New(v2.SourceCacheReplay_builder{ + ScopeKey: vsPageScope(gid, page), + CacheValidator: etag, + }.Build()), + }.Build(), nil + } + // Stale/missing verdict: cold, still no ask. + return mc.servePage(gid, page, false), nil + } + // Plain cold chain token: "page=N". + if _, err := fmt.Sscanf(tok, "page=%d", &page); err != nil { + return nil, fmt.Errorf("bad token %q", tok) + } + return mc.servePage(gid, page, true), nil + } + + // Origin call. + reqAnnos := annotations.Annotations(in.GetAnnotations()) + answersMsg := &v2.SourceCacheLookupAnswers{} + hasAnswers, err := reqAnnos.Pick(answersMsg) + if err != nil { + return nil, err + } + prev := mc.prevPages[gid] + + if !hasAnswers { + if reqAnnos.Contains(&v2.SourceCacheLookupOffer{}) && prev > 0 { + // ONE batch ask for the whole stored chain. + mc.originAsks++ + queries := make([]sourcecache.Query, 0, prev) + for p := 0; p < prev; p++ { + queries = append(queries, sourcecache.Query{RowKind: sourcecache.RowKindGrants, ScopeKey: vsPageScope(gid, p)}) + } + return v2.GrantsServiceListGrantsResponse_builder{ + Annotations: annotations.New(sourcecache.AskProto(queries)), + }.Build(), nil + } + // Cold first sync: serial chain. + return mc.servePage(gid, 0, true), nil + } + + // Phase 2: serve page 0 per its answer, spawn verdict-token siblings. + byScope := map[string]sourcecache.Answer{} + answers, answersErr := sourcecache.AnswersFromProto(answersMsg) + if answersErr != nil { + return nil, answersErr + } + for _, a := range answers { + byScope[a.ScopeKey] = a + } + var resp *v2.GrantsServiceListGrantsResponse + if a, ok := byScope[vsPageScope(gid, 0)]; ok && a.Found && a.CacheValidator == vsPageEtag(gid, 0, mc.pageRows(gid, 0)) { + mc.replays++ + resp = v2.GrantsServiceListGrantsResponse_builder{ + Annotations: annotations.New(v2.SourceCacheReplay_builder{ + ScopeKey: vsPageScope(gid, 0), + CacheValidator: a.CacheValidator, + }.Build()), + }.Build() + } else { + resp = mc.servePage(gid, 0, false) + } + var tokens []string + for p := 1; p < prev; p++ { + a := byScope[vsPageScope(gid, p)] + found := 0 + if a.Found { + found = 1 + } + tokens = append(tokens, fmt.Sprintf("page=%d|%d|%s", p, found, a.CacheValidator)) + } + if len(tokens) > 0 { + annos := annotations.Annotations(resp.GetAnnotations()) + annos.Update(v2.EnqueuePageTokens_builder{PageTokens: tokens}.Build()) + resp.SetAnnotations(annos) + resp.SetNextPageToken("") + } + return resp, nil +} + +func TestSourceCacheContinuation_Phase2EnqueuePageTokens(t *testing.T) { + for _, workers := range []int{0, 4} { + t.Run(fmt.Sprintf("workers=%d", workers), func(t *testing.T) { + ctx, err := logging.Init(context.Background()) + require.NoError(t, err) + tmpDir := t.TempDir() + + mc := &verdictSpawnMock{ + mockConnector: newMockConnector(), + members: map[string][]string{}, + prevPages: map[string]int{}, + resByID: map[string]*v2.Resource{}, + } + mc.rtDB = append(mc.rtDB, groupResourceType, userResourceType) + for i := 0; i < 8; i++ { + uid := fmt.Sprintf("user-%02d", i) + u, err := rs.NewUserResource(uid, userResourceType, uid, nil, rs.WithAnnotation(&v2.SkipEntitlementsAndGrants{})) + require.NoError(t, err) + mc.AddResource(ctx, u) + mc.resByID[uid] = u + } + for i := 0; i < 2; i++ { + gid := fmt.Sprintf("group-%02d", i) + g, err := rs.NewGroupResource(gid, groupResourceType, gid, nil) + require.NoError(t, err) + mc.AddResource(ctx, g) + mc.resByID[gid] = g + mc.entDB[gid] = []*v2.Entitlement{mkMemberEnt(g)} + for m := 0; m < 4; m++ { + mc.members[gid] = append(mc.members[gid], fmt.Sprintf("user-%02d", (i*4+m)%8)) + } + } + + sync1 := filepath.Join(tmpDir, "sync1.c1z") + require.NoError(t, runContinuationSync(ctx, t, mc, sync1, "", tmpDir, workers)) + require.Zero(t, mc.originAsks) + require.Equal(t, 4, mc.colds, "cold: 2 groups × 2 chained pages") + grants1 := listGrantsInFile(ctx, t, sync1) + require.Len(t, grants1, 8) + mc.mu.Lock() + for gid := range mc.members { + mc.prevPages[gid] = 2 + } + mc.mu.Unlock() + + sync2 := filepath.Join(tmpDir, "sync2.c1z") + require.NoError(t, runContinuationSync(ctx, t, mc, sync2, sync1, tmpDir, workers)) + require.Equal(t, 2, mc.originAsks, "ONE batch ask per group's origin action") + require.Zero(t, mc.siblingAsks, "verdict-carrying siblings never ask") + require.Equal(t, 4, mc.replays, "page 0 (phase 2) + spawned page 1, per group") + require.Equal(t, 4, mc.colds, "no cold fetches on the warm sync") + requireSameGrantIDs(t, grants1, listGrantsInFile(ctx, t, sync2), "planner-batch warm vs cold") + + // Churn group-01's tail page: its verdict goes stale, the + // sibling refetches cold from the token alone — still no ask. + mc.mu.Lock() + mc.members["group-01"][3] = "user-00" + mc.mu.Unlock() + sync3 := filepath.Join(tmpDir, "sync3.c1z") + require.NoError(t, runContinuationSync(ctx, t, mc, sync3, sync2, tmpDir, workers)) + require.Zero(t, mc.siblingAsks) + + control := filepath.Join(tmpDir, "control.c1z") + require.NoError(t, runContinuationSync(ctx, t, mc, control, "", tmpDir, workers)) + requireSameGrantIDs(t, listGrantsInFile(ctx, t, control), listGrantsInFile(ctx, t, sync3), "churned warm vs cold control") + }) + } +} + +func TestSourceCacheContinuation_BounceCapFails(t *testing.T) { + ctx, err := logging.Init(context.Background()) + require.NoError(t, err) + tmpDir := t.TempDir() + mc := newContinuationMockConnector(t, 1) + + sync1 := filepath.Join(tmpDir, "sync1.c1z") + require.NoError(t, runContinuationSync(ctx, t, mc, sync1, "", tmpDir, 0)) + + mc.askAlways = true + err = runContinuationSync(ctx, t, mc, filepath.Join(tmpDir, "sync2.c1z"), sync1, tmpDir, 0) + require.Error(t, err) + require.True(t, + strings.Contains(err.Error(), "bounce cap") || strings.Contains(err.Error(), "re-asked only already-answered"), + "a connector that never stops asking on ONE request must fail loudly, got: %v", err) +} + +// multiPagePlannerMock is the Entra planner shape: a TYPE-SCOPED grants +// action whose planning walk spans SIX pages, each of which asks once +// (LookupMany over the chunk scopes it is about to spawn) before serving. +// Six asks exceed the per-request bounce cap of four — the sync must still +// complete, because the cap is per REQUEST (same page token) and every +// NextPageToken advance is a fresh request. The final planning page spawns +// the real cursor; it must spawn exactly once (the ask response never +// reaches EnqueuePageTokens processing; the re-invoke serves the same page). +type multiPagePlannerMock struct { + *mockConnector + + mu sync.Mutex + groupIDs []string + resByID map[string]*v2.Resource + asks int + cursorServes int + spawns int + markerMissing bool // TypeScopedGrants marker absent on any request + answersScope string + answersPresent int // planning pages served with answers attached +} + +func (mc *multiPagePlannerMock) Validate(context.Context, *v2.ConnectorServiceValidateRequest, ...grpc.CallOption) (*v2.ConnectorServiceValidateResponse, error) { + return v2.ConnectorServiceValidateResponse_builder{ + Annotations: annotations.New(v2.SourceCacheCapability_builder{ + Mode: v2.SourceCacheCapability_MODE_READ_WRITE, + }.Build()), + }.Build(), nil +} + +const plannerPages = 6 + +func (mc *multiPagePlannerMock) ListGrants(_ context.Context, in *v2.GrantsServiceListGrantsRequest, _ ...grpc.CallOption) (*v2.GrantsServiceListGrantsResponse, error) { + if in.GetResource().GetId().GetResourceType() != typeScopedGroupRT.GetId() { + return v2.GrantsServiceListGrantsResponse_builder{}.Build(), nil + } + reqAnnos := annotations.Annotations(in.GetAnnotations()) + mc.mu.Lock() + defer mc.mu.Unlock() + if !reqAnnos.Contains(&v2.TypeScopedGrants{}) { + mc.markerMissing = true + } + + tok := in.GetPageToken() + if tok == "cursor" { + mc.cursorServes++ + grants := make([]*v2.Grant, 0, len(mc.groupIDs)) + for _, gid := range mc.groupIDs { + grants = append(grants, gt.NewGrant(mc.resByID[gid], "member", mc.resByID["user-00"].GetId())) + } + return v2.GrantsServiceListGrantsResponse_builder{List: grants}.Build(), nil + } + + page := 0 + if tok != "" { + if _, err := fmt.Sscanf(tok, "plan=%d", &page); err != nil { + return nil, fmt.Errorf("bad token %q", tok) + } + } + scope := fmt.Sprintf("chunks/page-%d", page) + + answersMsg := &v2.SourceCacheLookupAnswers{} + hasAnswers, err := reqAnnos.Pick(answersMsg) + if err != nil { + return nil, err + } + if !hasAnswers && reqAnnos.Contains(&v2.SourceCacheLookupOffer{}) { + mc.asks++ + return v2.GrantsServiceListGrantsResponse_builder{ + Annotations: annotations.New(sourcecache.AskProto([]sourcecache.Query{ + {RowKind: sourcecache.RowKindGrants, ScopeKey: scope}, + })), + }.Build(), nil + } + if hasAnswers { + mc.answersPresent++ + answers, answersErr := sourcecache.AnswersFromProto(answersMsg) + if answersErr != nil { + return nil, answersErr + } + for _, a := range answers { + if a.ScopeKey == scope { + mc.answersScope = scope + } + } + } + + // Serve the planning page: no rows; chain or spawn on the last page. + if page == plannerPages-1 { + mc.spawns++ + return v2.GrantsServiceListGrantsResponse_builder{ + Annotations: annotations.New(v2.EnqueuePageTokens_builder{PageTokens: []string{"cursor"}}.Build()), + }.Build(), nil + } + return v2.GrantsServiceListGrantsResponse_builder{ + NextPageToken: fmt.Sprintf("plan=%d", page+1), + }.Build(), nil +} + +func TestSourceCacheContinuation_MultiPagePlannerBouncesPerRequest(t *testing.T) { + ctx, err := logging.Init(context.Background()) + require.NoError(t, err) + tmpDir := t.TempDir() + + mc := &multiPagePlannerMock{ + mockConnector: newMockConnector(), + resByID: map[string]*v2.Resource{}, + } + mc.rtDB = append(mc.rtDB, typeScopedGroupRT, userResourceType) + u, err := rs.NewUserResource("user-00", userResourceType, "user-00", nil, rs.WithAnnotation(&v2.SkipEntitlementsAndGrants{})) + require.NoError(t, err) + mc.AddResource(ctx, u) + mc.resByID["user-00"] = u + for i := 0; i < 2; i++ { + gid := fmt.Sprintf("group-%02d", i) + g, err := rs.NewGroupResource(gid, typeScopedGroupRT, gid, nil) + require.NoError(t, err) + mc.AddResource(ctx, g) + mc.resByID[gid] = g + mc.groupIDs = append(mc.groupIDs, gid) + mc.entDB[gid] = []*v2.Entitlement{mkMemberEnt(g)} + } + + // Cold: no offer, no asks; the planner walks its pages and spawns. + sync1 := filepath.Join(tmpDir, "sync1.c1z") + require.NoError(t, runContinuationSync(ctx, t, mc, sync1, "", tmpDir, 0)) + require.Zero(t, mc.asks) + require.Equal(t, 1, mc.spawns) + require.Equal(t, 1, mc.cursorServes) + grants1 := listGrantsInFile(ctx, t, sync1) + require.Len(t, grants1, 2) + + // Warm: every planning page asks once — 6 bounces in ONE action, + // exceeding the per-request cap of 4. Must complete: each page-token + // advance is a fresh request. + sync2 := filepath.Join(tmpDir, "sync2.c1z") + require.NoError(t, runContinuationSync(ctx, t, mc, sync2, sync1, tmpDir, 0), + "a multi-page planner asking once per page must not trip the per-request bounce cap") + require.Equal(t, plannerPages, mc.asks, "one ask per planning page") + require.Equal(t, plannerPages, mc.answersPresent, "every re-invoked planning page carries answers") + require.NotEmpty(t, mc.answersScope, "answers must cover the asked scope") + require.False(t, mc.markerMissing, "TypeScopedGrants routing marker must survive on every request, including re-invokes") + require.Equal(t, 2, mc.spawns, "the asking planner page spawns exactly once per sync") + require.Equal(t, 2, mc.cursorServes, "spawned cursor executes exactly once per sync") + requireSameGrantIDs(t, grants1, listGrantsInFile(ctx, t, sync2), "warm planner sync vs cold") +} + +// fatEtagLookup finds every scope with the same (large) etag. +type fatEtagLookup struct{ etag string } + +func (f fatEtagLookup) Lookup(context.Context, sourcecache.RowKind, string) (sourcecache.Entry, bool, error) { + return sourcecache.Entry{CacheValidator: f.etag}, true, nil +} + +// TestSourceCacheContinuation_AnswerBudgetDegradesToCold pins the answer +// size budget's failure mode: one ask whose FOUND etags total more than +// sourceCacheAnswerBudget. Answers accumulate across bounces and the +// budget only shrinks, so an over-budget answer left absent could never +// be resolved by a re-ask — it must instead be degraded to an explicit +// not-found (the scope goes cold) and the request must complete in one +// answered turn, not livelock into the bounce cap. +func TestSourceCacheContinuation_AnswerBudgetDegradesToCold(t *testing.T) { + ctx, err := logging.Init(context.Background()) + require.NoError(t, err) + + const queryCount = 40 + etag := strings.Repeat("e", 65536) // the wire cap for one etag + // Mirror the budget arithmetic: every answered query spends its + // identity bytes (row_kind + scope_key + framing overhead) whether + // found or degraded; a found answer additionally needs its etag to + // fit the remaining budget. + identityCost := len(sourcecache.RowKindGrants) + len("scope-000") + sourceCacheAnswerOverheadBytes + fitting := 0 + for i, budget := 0, sourceCacheAnswerBudget; i < queryCount; i++ { + budget -= identityCost + if len(etag) <= budget { + budget -= len(etag) + fitting++ + } + } + require.Positive(t, fitting, "test must fit some answers") + require.Less(t, fitting, queryCount, "test must overflow the budget") + + s := &syncer{ + state: newState(), + sourceCache: syncerSourceCache{ + prev: struct{ dotc1z.SourceCacheStore }{}, // non-nil: warm + lookup: fatEtagLookup{etag: etag}, + }, + } + + queries := make([]sourcecache.Query, 0, queryCount) + for i := 0; i < queryCount; i++ { + queries = append(queries, sourcecache.Query{RowKind: sourcecache.RowKindGrants, ScopeKey: fmt.Sprintf("scope-%03d", i)}) + } + + turns := 0 + err = s.withSourceCacheContinuation(ctx, "grants", func(extra annotations.Annotations, attempt int) (listAttempt, error) { + turns++ + if attempt == 0 { + return listAttempt{annos: annotations.New(sourcecache.AskProto(queries))}, nil + } + answersMsg := &v2.SourceCacheLookupAnswers{} + hasAnswers, pickErr := extra.Pick(answersMsg) + require.NoError(t, pickErr) + require.True(t, hasAnswers) + answers, convErr := sourcecache.AnswersFromProto(answersMsg) + require.NoError(t, convErr) + require.Len(t, answers, queryCount, "every asked query must be answered; an absent answer can never resolve later") + found := 0 + for _, a := range answers { + if a.Found { + found++ + } + } + require.Equal(t, fitting, found, "found answers up to the budget") + return listAttempt{}, nil + }) + require.NoError(t, err, "an over-budget answer set must degrade to cold, not die at the bounce cap") + require.Equal(t, 2, turns, "one ask turn + one answered turn") + + stats := s.state.SourceCacheStatsSnapshot() + require.NotNil(t, stats) + require.Equal(t, int64(fitting), stats.LookupAnsweredFound) + require.Equal(t, int64(queryCount-fitting), stats.LookupAnswersTruncated) + require.Zero(t, stats.LookupAnsweredNotFound, "budget degradation counts as truncated, not as a genuine miss") +} + +func TestSourceCacheContinuation_AskWithEnqueuePageTokensFails(t *testing.T) { + ctx, err := logging.Init(context.Background()) + require.NoError(t, err) + tmpDir := t.TempDir() + mc := newContinuationMockConnector(t, 1) + + sync1 := filepath.Join(tmpDir, "sync1.c1z") + require.NoError(t, runContinuationSync(ctx, t, mc, sync1, "", tmpDir, 0)) + + mc.askWithSpawn = true + err = runContinuationSync(ctx, t, mc, filepath.Join(tmpDir, "sync2.c1z"), sync1, tmpDir, 0) + require.Error(t, err) + require.Contains(t, err.Error(), "no spawned cursors", + "EnqueuePageTokens alongside an ask is a connector bug and must fail loudly") +} + +func TestSourceCacheContinuation_AskWithRowsFails(t *testing.T) { + ctx, err := logging.Init(context.Background()) + require.NoError(t, err) + tmpDir := t.TempDir() + mc := newContinuationMockConnector(t, 1) + + sync1 := filepath.Join(tmpDir, "sync1.c1z") + require.NoError(t, runContinuationSync(ctx, t, mc, sync1, "", tmpDir, 0)) + + mc.askWithRows = true + err = runContinuationSync(ctx, t, mc, filepath.Join(tmpDir, "sync2.c1z"), sync1, tmpDir, 0) + require.Error(t, err) + require.Contains(t, err.Error(), "must carry ONLY the ask") +} + +func TestSourceCacheContinuation_AskWithoutOfferFails(t *testing.T) { + ctx, err := logging.Init(context.Background()) + require.NoError(t, err) + tmpDir := t.TempDir() + mc := newContinuationMockConnector(t, 1) + mc.askNoOffer = true + + // Cold sync: no previous c1z → no offer → the ask is a protocol + // violation and must fail, not be misread as an empty page. + err = runContinuationSync(ctx, t, mc, filepath.Join(tmpDir, "sync1.c1z"), "", tmpDir, 0) + require.Error(t, err) + require.Contains(t, err.Error(), "no offer") +} + +// multiPagePlannerEntitlementsMock mirrors multiPagePlannerMock for +// TypeScopedEntitlements: a multi-page planner that asks once per page +// before spawning a real cursor, asserting the routing marker survives +// every re-invocation. +type multiPagePlannerEntitlementsMock struct { + *mockConnector + + mu sync.Mutex + groupIDs []string + resByID map[string]*v2.Resource + asks int + cursorServes int + spawns int + markerMissing bool + answersScope string + answersPresent int +} + +func (mc *multiPagePlannerEntitlementsMock) Validate(context.Context, *v2.ConnectorServiceValidateRequest, ...grpc.CallOption) (*v2.ConnectorServiceValidateResponse, error) { + return v2.ConnectorServiceValidateResponse_builder{ + Annotations: annotations.New(v2.SourceCacheCapability_builder{ + Mode: v2.SourceCacheCapability_MODE_READ_WRITE, + }.Build()), + }.Build(), nil +} + +func (mc *multiPagePlannerEntitlementsMock) ListEntitlements( + _ context.Context, in *v2.EntitlementsServiceListEntitlementsRequest, _ ...grpc.CallOption, +) (*v2.EntitlementsServiceListEntitlementsResponse, error) { + if in.GetResource().GetId().GetResourceType() != typeScopedEntGroupRT.GetId() { + return v2.EntitlementsServiceListEntitlementsResponse_builder{}.Build(), nil + } + reqAnnos := annotations.Annotations(in.GetAnnotations()) + mc.mu.Lock() + defer mc.mu.Unlock() + if !reqAnnos.Contains(&v2.TypeScopedEntitlements{}) { + mc.markerMissing = true + } + + tok := in.GetPageToken() + if tok == "cursor" { + mc.cursorServes++ + ents := make([]*v2.Entitlement, 0, len(mc.groupIDs)) + for _, gid := range mc.groupIDs { + ents = append(ents, et.NewAssignmentEntitlement(mc.resByID[gid], "member", et.WithGrantableTo(userResourceType))) + } + return v2.EntitlementsServiceListEntitlementsResponse_builder{List: ents}.Build(), nil + } + + page := 0 + if tok != "" { + if _, err := fmt.Sscanf(tok, "plan=%d", &page); err != nil { + return nil, fmt.Errorf("bad token %q", tok) + } + } + scope := fmt.Sprintf("ent-chunks/page-%d", page) + + answersMsg := &v2.SourceCacheLookupAnswers{} + hasAnswers, err := reqAnnos.Pick(answersMsg) + if err != nil { + return nil, err + } + if !hasAnswers && reqAnnos.Contains(&v2.SourceCacheLookupOffer{}) { + mc.asks++ + return v2.EntitlementsServiceListEntitlementsResponse_builder{ + Annotations: annotations.New(sourcecache.AskProto([]sourcecache.Query{ + {RowKind: sourcecache.RowKindEntitlements, ScopeKey: scope}, + })), + }.Build(), nil + } + if hasAnswers { + mc.answersPresent++ + answers, answersErr := sourcecache.AnswersFromProto(answersMsg) + if answersErr != nil { + return nil, answersErr + } + for _, a := range answers { + if a.ScopeKey == scope { + mc.answersScope = scope + } + } + } + + if page == plannerPages-1 { + mc.spawns++ + return v2.EntitlementsServiceListEntitlementsResponse_builder{ + Annotations: annotations.New(v2.EnqueuePageTokens_builder{PageTokens: []string{"cursor"}}.Build()), + }.Build(), nil + } + return v2.EntitlementsServiceListEntitlementsResponse_builder{ + NextPageToken: fmt.Sprintf("plan=%d", page+1), + }.Build(), nil +} + +func (mc *multiPagePlannerEntitlementsMock) ListGrants(context.Context, *v2.GrantsServiceListGrantsRequest, ...grpc.CallOption) (*v2.GrantsServiceListGrantsResponse, error) { + return v2.GrantsServiceListGrantsResponse_builder{}.Build(), nil +} + +func TestSourceCacheContinuation_MultiPagePlannerEntitlementsBouncesPerRequest(t *testing.T) { + ctx, err := logging.Init(context.Background()) + require.NoError(t, err) + tmpDir := t.TempDir() + + mc := &multiPagePlannerEntitlementsMock{ + mockConnector: newMockConnector(), + resByID: map[string]*v2.Resource{}, + } + mc.rtDB = append(mc.rtDB, typeScopedEntGroupRT, userResourceType) + u, err := rs.NewUserResource("user-00", userResourceType, "user-00", nil, rs.WithAnnotation(&v2.SkipEntitlementsAndGrants{})) + require.NoError(t, err) + mc.AddResource(ctx, u) + mc.resByID["user-00"] = u + for i := 0; i < 2; i++ { + gid := fmt.Sprintf("group-%02d", i) + g, err := rs.NewGroupResource(gid, typeScopedEntGroupRT, gid, nil) + require.NoError(t, err) + mc.AddResource(ctx, g) + mc.resByID[gid] = g + mc.groupIDs = append(mc.groupIDs, gid) + } + + sync1 := filepath.Join(tmpDir, "sync1.c1z") + require.NoError(t, runContinuationSync(ctx, t, mc, sync1, "", tmpDir, 0)) + require.Zero(t, mc.asks) + require.Equal(t, 1, mc.spawns) + require.Equal(t, 1, mc.cursorServes) + ents1 := listEntitlementsInFile(ctx, t, sync1) + require.Len(t, ents1, 2) + + sync2 := filepath.Join(tmpDir, "sync2.c1z") + require.NoError(t, runContinuationSync(ctx, t, mc, sync2, sync1, tmpDir, 0), + "a multi-page entitlements planner asking once per page must not trip the per-request bounce cap") + require.Equal(t, plannerPages, mc.asks, "one ask per planning page") + require.Equal(t, plannerPages, mc.answersPresent, "every re-invoked planning page carries answers") + require.NotEmpty(t, mc.answersScope, "answers must cover the asked scope") + require.False(t, mc.markerMissing, "TypeScopedEntitlements routing marker must survive on every request, including re-invokes") + require.Equal(t, 2, mc.spawns, "the asking planner page spawns exactly once per sync") + require.Equal(t, 2, mc.cursorServes, "spawned cursor executes exactly once per sync") + requireSameEntitlementIDs(t, ents1, listEntitlementsInFile(ctx, t, sync2), "warm planner sync vs cold") +} diff --git a/pkg/sync/source_cache_equivalence_dirty_test.go b/pkg/sync/source_cache_equivalence_dirty_test.go new file mode 100644 index 000000000..250824cec --- /dev/null +++ b/pkg/sync/source_cache_equivalence_dirty_test.go @@ -0,0 +1,538 @@ +package sync //nolint:revive,nolintlint // we can't change the package name for backwards compatibility + +// Dirty-connector replay-equivalence scenarios. +// +// THE PROPERTY: default-mode warm ≡ default-mode cold, for connectors +// whose data carries EXPECTED configuration gaps — dangling references +// whose referent's resource TYPE is not synced (disabled). Under the +// replay policy those are the DROP arm of the referential invariants +// (I7/I8/I9): dependent rows drop with one aggregated warning, the +// artifact seals referentially consistent, and every dropped-from +// scope's manifest entry is INVALIDATED so the scope re-fetches cold +// next round. Enabled-type danglings are NOT equivalence material — they +// fail the sync (warm: ErrReplayIntegrity + cold retry; cold: plain) and +// are pinned by TestIngestInvariantEnabledTypeDanglingsFail and +// TestDishonestValidatorStrandsGrantEntitlements. +// +// The clean harness (source_cache_equivalence_test.go) runs fail-fast, +// so the drop arms never execute inside its property; this family runs +// default mode on both legs. The manifest-invalidation-on-drop contract +// is what the dirty→clean transition pins: without it, a later honest +// 304 replays the sanitized subset while a cold sync fetches the full +// set — warm ≠ cold exactly when a dropped row's referent appears. +// +// DIRTY SHAPES (all disabled-type; forced, every scenario): +// - I9: groups hold members of the SHADOW type (ids "shadow-*"), +// which is disabled at round 0 — member grants drop; and of the +// ghost2 type (ids "ghost2-*"), which NEVER enables — steady dirty. +// - I8: gd8's grants page emits a grant on the shadow HOST's "admin" +// entitlement, which exists only once the shadow type is enabled. +// - I7 (type-scoped): team chunk 0's entitlements page emits an +// entitlement for the shadow host2 resource (and chunk 0's grants +// page a grant under it — the I8 cascade). +// - Exempt carriers: the ext group's unprocessed ExternalResourceMatch +// carriers (no external c1z configured in this family) are KEPT in +// both artifacts every round. +// +// TRANSITIONS (round schedule; see mutateDirty): +// r0 initial dirty — shadow type disabled: I7, I8, and I9 drops fire. +// r1 dirty→clean (THE BUG CELL): the shadow type is ENABLED — its +// type row, resources, and host entitlements appear — while every +// dropped-from GRANTS/ENTS scope's validator stays put. Without +// manifest invalidation the warm side 304s the sanitized subsets +// and diverges from cold here. +// r2 no-op + crash/resume: the warm sync is crashed mid-grants on a +// steady-dirty scope and resumed (drops + invalidation must +// behave identically through resume). +// r3 clean→dirty via OVERLAY: gdelta's delta round adds a ghost2 +// member — dangling data arriving on the overlay path. +// r4 no-op — steady dirty chain-decay probe: ghost2 scopes re-fetch +// cold (invalidated), re-drop, re-warn; all clean scopes 304; the +// output must STILL equal cold. +// +// ORACLES: the clean harness's full semantic diff + manifest and +// scope-index comparison (manifest entries carry the invalidation +// verdict); exact model-derived post-drop row sets asserted against +// BOTH artifacts; and the warning-silence oracle INVERTED — every round +// asserts the expected "ingest invariant ... DROPPED" warnings are +// present on both legs (the anti-vacuity proof that drops executed) and +// that nothing unexpected warned. A dirty scenario that never dropped +// anything fails itself. + +import ( + "fmt" + "math/rand" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/require" + + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + "github.com/conductorone/baton-sdk/pkg/logging" + "github.com/conductorone/baton-sdk/pkg/types" + gt "github.com/conductorone/baton-sdk/pkg/types/grant" +) + +// newDirtyEquivModel is the clean topology plus the dirty shapes. The +// clean groups/orgs/teams are kept so the dirty rounds still have a +// large clean surface that must keep 304ing correctly around the drops. +func newDirtyEquivModel() *equivModel { + m := newEquivModel() + // I9 dirty→clean: a shadow-typed member, missing until the shadow + // type is enabled in round 1. + m.groups["gd9"] = &equivGroup{id: "gd9", members: map[string]bool{"u0": true, "shadow-mem": true}} + // I9 steady dirty: a ghost2-typed member whose type never enables. + m.groups["gd"] = &equivGroup{id: "gd", members: map[string]bool{"u0": true, "ghost2-mem": true}} + // I9 clean→dirty via overlay: gains a ghost2 member in round 3. + m.groups["gdelta"] = &equivGroup{id: "gdelta", members: map[string]bool{"u3": true}} + // I8: the shadow-host admin grant is emitted every round; its + // entitlement row exists only from round 1. + m.groups["gd8"] = &equivGroup{id: "gd8", members: map[string]bool{"u1": true}, ghostAdminGrant: true} + // I7 (+ I8 cascade) in the type-scoped chunk scopes. + m.ghostTeamEnt = true + // The shadow type's row universe once enabled. + m.shadowUsers = map[string]bool{"shadow-mem": true} + return m +} + +// mutateDirty is the dirty family's forced round schedule (no seeded +// churn: every transition is load-bearing and must occur exactly where +// the oracles expect it). +func (m *equivModel) mutateDirty(round int) { + m.clearDeltas() + switch round { + case 1: + // Dirty→clean: ENABLE the shadow resource type. Its type row, + // resources, and host entitlements appear via their own pages — + // every previously dropped-from scope's validator stays put. + m.shadowEnabled = true + case 3: + // Clean→dirty via a delta overlay round (ghost2 never enables). + m.addMember("gdelta", "ghost2-d") + default: + // Rounds 2 and 4: no-op (r2 additionally crashes/resumes the + // warm leg; steady-dirty scopes re-fetch and re-drop each round). + } +} + +// expectedDirtyResourceKeys is expectedResourceKeys minus external +// principals (no external c1z in this family) plus the shadow universe +// once enabled. +func expectedDirtyResourceKeys(m *equivModel) []string { + set := map[string]struct{}{} + for uid := range m.users { + set[userResourceType.GetId()+"/"+uid] = struct{}{} + } + for gid := range m.groups { + set[groupResourceType.GetId()+"/"+gid] = struct{}{} + } + for orgID, org := range m.orgs { + set[equivOrgRT.GetId()+"/"+orgID] = struct{}{} + for projID := range org.projects { + set[equivProjectRT.GetId()+"/"+projID] = struct{}{} + } + for repoID := range org.repoGrants { + set[equivRepoRT.GetId()+"/"+repoID] = struct{}{} + } + } + for tid := range m.teams { + set[equivTeamRT.GetId()+"/"+tid] = struct{}{} + } + if m.shadowEnabled { + for id := range m.shadowUsers { + set[equivShadowRT.GetId()+"/"+id] = struct{}{} + } + set[equivShadowRT.GetId()+"/"+equivShadowHostID] = struct{}{} + set[equivShadowRT.GetId()+"/"+equivShadowHost2ID] = struct{}{} + } + return sortedKeys(set) +} + +// expectedDirtyEntitlementIDs is the POST-DROP entitlement set: the +// shadow hosts' entitlements exist only while the type is enabled (I7 +// drops the host2 entitlement otherwise; the host's admin entitlement +// is simply not listed). +func expectedDirtyEntitlementIDs(m *equivModel) []string { + set := map[string]struct{}{} + for gid, g := range m.groups { + set[equivMemberEnt(equivGroupResource(gid), g.exclusionDefault).GetId()] = struct{}{} + } + for tid := range m.teams { + set[equivMemberEnt(equivTeamResource(tid), false).GetId()] = struct{}{} + } + if m.shadowEnabled { + set[equivShadowAdminEnt().GetId()] = struct{}{} + if m.ghostTeamEnt { + set[equivShadowHost2Ent().GetId()] = struct{}{} + } + } + return sortedKeys(set) +} + +// expectedDirtyGrantIDs is the POST-DROP grant set: +// - member grants for user-typed members always; shadow members only +// while the type is enabled (I9 drops them otherwise); ghost2 +// members never (their type never enables); +// - gd8's shadow-admin grant and the chunk-0 host2 grant only while +// the shadow type is enabled (I8 / I7-cascade drops otherwise); +// - the ext group's match carriers are KEPT (exempt) every round. +func expectedDirtyGrantIDs(m *equivModel) []string { + set := map[string]struct{}{} + memberExpected := func(uid string) bool { + switch { + case strings.HasPrefix(uid, "shadow-"): + return m.shadowEnabled && m.shadowUsers[uid] + case strings.HasPrefix(uid, "ghost2-"): + return false + default: + return m.users[uid] + } + } + for gid, g := range m.groups { + if gid == m.extGroupID { + continue + } + for uid := range g.members { + if memberExpected(uid) { + set[equivMemberGrant(gid, uid).GetId()] = struct{}{} + } + } + if g.ghostAdminGrant && m.shadowEnabled { + set[equivShadowAdminGrant("u1").GetId()] = struct{}{} + } + } + // The expandable group-to-group membership and its derived grants. + parentRes := equivGroupResource(m.expandableParent) + parentEnt := equivMemberEnt(parentRes, false) + set[gt.NewGrantID(equivGroupResource(m.expandableChild), parentEnt)] = struct{}{} + for uid := range m.groups[m.expandableChild].members { + if m.users[uid] { + set[gt.NewGrantID( + v2.ResourceId_builder{ResourceType: userResourceType.GetId(), Resource: uid}.Build(), + parentEnt)] = struct{}{} + } + } + // Unprocessed match carriers survive whole (exempt in I9). + for _, extID := range m.extMatchIDs { + set[equivMemberGrant(m.extGroupID, "placeholder-"+extID).GetId()] = struct{}{} + } + set[equivMemberGrant(m.extGroupID, "placeholder-all").GetId()] = struct{}{} + for _, org := range m.orgs { + for repoID, uid := range org.repoGrants { + set[gt.NewGrant(equivRepoResource(repoID), "admin", + v2.ResourceId_builder{ResourceType: userResourceType.GetId(), Resource: uid}.Build()).GetId()] = struct{}{} + } + } + for tid, members := range m.teams { + for uid := range members { + set[equivTeamGrant(tid, uid).GetId()] = struct{}{} + } + } + if m.ghostTeamEnt && m.shadowEnabled { + set[equivShadowHost2Grant("u0").GetId()] = struct{}{} + } + return sortedKeys(set) +} + +// assertDirtyArtifacts asserts one artifact's row sets EXACTLY equal the +// model-derived post-drop expectation — dropped rows absent, exempt +// carriers and IRG shapes present. Applied to BOTH the cold and warm +// snapshots (the differential diff alone can't distinguish "both sides +// dropped" from "neither side dropped"). +func assertDirtyArtifacts(t *testing.T, round int, m *equivModel, label string, snap *c1zSnapshot) { + t.Helper() + require.Equal(t, expectedDirtyResourceKeys(m), sortedKeys(snap.resources), + "round %d: %s resource set must equal the post-drop expectation", round, label) + require.Equal(t, expectedDirtyEntitlementIDs(m), sortedKeys(snap.entitlements), + "round %d: %s entitlement set must equal the post-drop expectation", round, label) + require.Equal(t, expectedDirtyGrantIDs(m), sortedKeys(snap.grants), + "round %d: %s grant set must equal the post-drop expectation", round, label) + // Exempt carriers, named explicitly (also implied by the exact sets). + require.Contains(t, snap.grants, equivMemberGrant(m.extGroupID, "placeholder-ext-0").GetId(), + "round %d: %s must keep the exempt match carriers", round, label) +} + +// Invariant warning fingerprints (must match the aggregated warn +// messages in ingest_invariants.go). +const ( + dirtyWarnI7Drop = "ingest invariant I7: DROPPED entitlements" + dirtyWarnI8Drop = "ingest invariant I8: DROPPED grants" + dirtyWarnI9Drop = "ingest invariant I9: DROPPED grants" + dirtyWarnI9Kept = "ingest invariant I9: kept unprocessed external-match carrier grants" + dirtyWarnI3Never = "ingest invariant I3: grants reference a resource that was never synced" +) + +// dirtyExpectedWarnings returns the warning fingerprints REQUIRED on +// every leg of a round (warm and cold alike — drops are a pure function +// of the converged store). Anything captured that matches no required +// fingerprint fails the leg. +func dirtyExpectedWarnings(round int) []string { + if round == 0 { + // Shadow type disabled: the I7 host2 entitlement, the I8 + // shadow-admin + host2-cascade grants (I3 warns about their + // never-synced entitlement resources before I8 drops them), and + // the I9 shadow/ghost2 members, plus the kept-carriers report. + return []string{dirtyWarnI7Drop, dirtyWarnI8Drop, dirtyWarnI9Drop, dirtyWarnI9Kept, dirtyWarnI3Never} + } + // Rounds 1+: the shadow shapes are clean; only the ghost2 steady- + // dirty members keep dropping (I9) and the carriers keep being kept. + return []string{dirtyWarnI9Drop, dirtyWarnI9Kept} +} + +func assertDirtyWarnings(t *testing.T, round int, label string, msgs []string) { + t.Helper() + expected := dirtyExpectedWarnings(round) + for _, want := range expected { + found := false + for _, msg := range msgs { + if strings.Contains(msg, want) { + found = true + break + } + } + require.True(t, found, + "round %d (%s): expected invariant warning %q never fired — the drop arm did not execute (vacuous dirty round). captured: %v", + round, label, want, msgs) + } + for _, msg := range msgs { + allowed := false + for _, want := range expected { + if strings.Contains(msg, want) { + allowed = true + break + } + } + require.True(t, allowed, + "round %d (%s): unexpected ingestion-invariant warning %q", round, label, msg) + } +} + +const dirtyEquivRounds = 4 + +// dirtyCrashRound crashes the warm sync mid-grants and resumes it, in a +// round where the steady-dirty drops fire — drops and manifest +// invalidation must behave identically through resume. +const dirtyCrashRound = 2 + +// TestSourceCache_DirtyHaltAtInvariantSeam crashes a DIRTY sync at the +// "ingest-invariants-complete" seam — after the disabled-type drops (and +// their manifest invalidations) committed, before the checkpoint and +// EndSync — then resumes the same file. The resumed sync re-runs the +// whole invariant pass over already-dropped state, so this pins the +// post-drop crash contract end to end at the SYNCER level (the engine's +// chunk-boundary crash test pins the intra-drop window): drops are +// idempotent on re-run, the invalidations committed by the first attempt +// survive and are not misread (I6 must not report the invalidated scopes +// as orphans), and the resumed output equals a cold sync — including +// through round 1's dirty→clean replay, where a lost invalidation would +// 304 the sanitized subsets. +func TestSourceCache_DirtyHaltAtInvariantSeam(t *testing.T) { + runDirtyHaltAtStage(t, "ingest-invariants-complete") +} + +// TestSourceCache_DirtyHaltSweepInsideInvariantPass sweeps every seam +// INSIDE the invariant pass (the stage hooks in runIngestionInvariants): +// halt after each invariant's drops/repairs committed, resume the same +// file, and require the resumed output to equal a cold sync. This is +// what pins the stop-point matrix from the crash review — "I7's drops +// committed, I8 never ran" converges via the drop cascade on re-run; +// "deferred index built, marker cleared, I9 never scanned" re-scans +// without repeating the O(grants) build — instead of arguing those +// recoveries from code structure. +func TestSourceCache_DirtyHaltSweepInsideInvariantPass(t *testing.T) { + stages := []string{ + "invariants-I5-complete", + "invariants-I6-complete", + "invariants-I7-complete", + "invariants-I3-complete", + "invariants-I8-complete", + "invariants-I9-indexes-ensured", + } + for _, stage := range stages { + t.Run(stage, func(t *testing.T) { + runDirtyHaltAtStage(t, stage) + }) + } +} + +func runDirtyHaltAtStage(t *testing.T, stage string) { + ctx, err := logging.Init(t.Context()) + require.NoError(t, err) + tmpDir := t.TempDir() + + model := newDirtyEquivModel() + warmConn := newEquivConnector(model, false) + coldConn := newEquivConnector(model, true) + var warmClient types.ConnectorClient = warmConn + var coldClient types.ConnectorClient = coldConn + + halt := &haltOnce{stage: stage} + hookOpt := func(s *syncer) { s.testSourceCacheHaltHook = halt.hook } + + // Round 0: the warm leg crashes at the seam, then resumes the SAME + // file to completion. + warm0 := filepath.Join(tmpDir, "warm-r0.c1z") + haltErr := runEquivSyncMode(ctx, t, warmClient, warm0, "", tmpDir, false, hookOpt) + require.Error(t, haltErr, "the armed halt must crash the dirty sync at the invariant seam") + require.Contains(t, haltErr.Error(), "injected halt at "+stage) + require.NoError(t, runEquivSyncMode(ctx, t, warmClient, warm0, "", tmpDir, false, hookOpt), + "resume after the invariant-seam halt must complete (drops must be idempotent over already-dropped state)") + + cold0 := filepath.Join(tmpDir, "cold-r0.c1z") + require.NoError(t, runEquivSyncMode(ctx, t, coldClient, cold0, "", tmpDir, false)) + + warmSnap := snapshotC1z(ctx, t, warm0) + coldSnap := snapshotC1z(ctx, t, cold0) + require.Equal(t, 1, warmSnap.syncRuns, "the halted sync must RESUME (one sync run), not restart") + assertDirtyArtifacts(t, 0, model, "warm", warmSnap) + assertDirtyArtifacts(t, 0, model, "cold", coldSnap) + requireSnapshotsEqual(t, 0, coldSnap, warmSnap) + invalidated := 0 + for _, entry := range warmSnap.manifest { + if entry.Invalidated { + invalidated++ + } + } + require.Positive(t, invalidated, + "round 0's drops must leave invalidated manifest entries in the resumed artifact") + + // Round 1 (dirty→clean): warm replays from the resumed artifact. A + // lost invalidation would 304 the sanitized scopes here and diverge + // from cold. + model.mutateDirty(1) + warm1 := filepath.Join(tmpDir, "warm-r1.c1z") + cold1 := filepath.Join(tmpDir, "cold-r1.c1z") + require.NoError(t, runEquivSyncMode(ctx, t, warmClient, warm1, warm0, tmpDir, false)) + require.NoError(t, runEquivSyncMode(ctx, t, coldClient, cold1, "", tmpDir, false)) + warmSnap1 := snapshotC1z(ctx, t, warm1) + coldSnap1 := snapshotC1z(ctx, t, cold1) + assertDirtyArtifacts(t, 1, model, "warm", warmSnap1) + assertDirtyArtifacts(t, 1, model, "cold", coldSnap1) + requireSnapshotsEqual(t, 1, coldSnap1, warmSnap1) +} + +func TestSourceCache_ReplayEquivalenceDirty(t *testing.T) { + scenarios := []struct { + workers int + seed int64 + continuation bool + }{ + {workers: 0, seed: 11}, + {workers: 4, seed: 12}, + {workers: 0, seed: 13, continuation: true}, + } + for _, sc := range scenarios { + name := fmt.Sprintf("workers=%d/seed=%d", sc.workers, sc.seed) + if sc.continuation { + name += "/continuation" + } + t.Run(name, func(t *testing.T) { + runDirtyReplayEquivalenceScenario(t, sc.workers, sc.seed, sc.continuation) + }) + } +} + +func runDirtyReplayEquivalenceScenario(t *testing.T, workers int, seed int64, continuation bool) { + ctx, err := logging.Init(t.Context()) + require.NoError(t, err) + ctx, invariantWarnings := withInvariantWarningOracle(ctx) + tmpDir := t.TempDir() + // The schedule is fully forced; the seed only labels the scenario + // (kept for parity with the clean family and future churn). + _ = rand.New(rand.NewSource(seed)) //nolint:gosec // deterministic test randomness + t.Logf("dirty replay-equivalence scenario: workers=%d seed=%d rounds=%d continuation=%v", workers, seed, dirtyEquivRounds, continuation) + + model := newDirtyEquivModel() + warmConn := newEquivConnector(model, false) + coldConn := newEquivConnector(model, true) + warmConn.continuation = continuation + coldConn.continuation = continuation + var warmClient types.ConnectorClient = warmConn + var coldClient types.ConnectorClient = coldConn + if continuation { + warmClient = equivContinuationClient{warmConn} + coldClient = equivContinuationClient{coldConn} + } + + // NO external c1z: unprocessed match carriers must survive to the + // invariant seam (the exempt shape) — with a file configured the + // match op deletes every carrier it processes. + syncOpts := func() []SyncOpt { + var opts []SyncOpt + if workers > 0 { + opts = append(opts, WithWorkerCount(workers)) + } + return opts + } + + // runLeg runs one default-mode sync and asserts the leg's invariant + // warnings: the round's expected drops MUST have fired (anti- + // vacuity), and nothing else may warn. + runLeg := func(round int, label string, cc types.ConnectorClient, path, prevPath string) { + t.Helper() + before := len(invariantWarnings.captured()) + require.NoError(t, runEquivSyncMode(ctx, t, cc, path, prevPath, tmpDir, false, syncOpts()...)) + assertDirtyWarnings(t, round, label, invariantWarnings.captured()[before:]) + } + + prevWarm := "" + for round := 0; round <= dirtyEquivRounds; round++ { + if round > 0 { + model.mutateDirty(round) + } + warmPath := filepath.Join(tmpDir, fmt.Sprintf("warm-r%d.c1z", round)) + coldPath := filepath.Join(tmpDir, fmt.Sprintf("cold-r%d.c1z", round)) + + before := warmConn.snapshotCounters() + if round == dirtyCrashRound { + // Crash the warm sync mid-grants on a steady-dirty scope + // (gd re-fetches every round: its manifest entry was + // invalidated by the previous round's drop), then resume the + // SAME file. + warmConn.injectGrantsFailure("gd") + crashErr := runEquivSyncMode(ctx, t, warmClient, warmPath, prevWarm, tmpDir, false, syncOpts()...) + require.Error(t, crashErr, "round %d: the injected failure must crash the warm sync", round) + require.Contains(t, crashErr.Error(), "injected crash") + require.Equal(t, 1, warmConn.snapshotCounters().injectedFailures-before.injectedFailures, + "round %d: exactly one injected failure expected", round) + } + runLeg(round, "warm", warmClient, warmPath, prevWarm) + after := warmConn.snapshotCounters() + if round > 0 { + require.Positive(t, (after.pages304+after.pagesOverlay)-(before.pages304+before.pagesOverlay), + "round %d: VACUOUS RUN — the warm sync replayed nothing, so equivalence proves nothing", round) + } + runLeg(round, "cold", coldClient, coldPath, "") + + coldSnap := snapshotC1z(ctx, t, coldPath) + warmSnap := snapshotC1z(ctx, t, warmPath) + if round == dirtyCrashRound { + require.Equal(t, 1, warmSnap.syncRuns, + "round %d: the crashed warm sync must RESUME (one sync run), not restart", round) + } + assertDirtyArtifacts(t, round, model, "cold", coldSnap) + assertDirtyArtifacts(t, round, model, "warm", warmSnap) + requireSnapshotsEqual(t, round, coldSnap, warmSnap) + + prevWarm = warmPath + } + + // Scenario-level coverage: the dirty rounds must still have + // exercised the replay machinery they claim to run drops through. + final := warmConn.snapshotCounters() + require.Positive(t, final.pages304, "coverage: no 304 replay round ever happened") + require.Positive(t, final.pagesOverlay, "coverage: no delta-overlay round ever happened (gdelta round 3)") + require.Positive(t, final.pagesFresh, "coverage: no cold-miss page ever happened") + require.Positive(t, final.chunkPages304, "coverage: no spawned chunk cursor ever replayed") + require.Positive(t, final.chunkPagesFresh, "coverage: no spawned chunk cursor ever fetched fresh") + require.Positive(t, final.injectedFailures, "coverage: the crash/resume round never crashed") + if continuation { + require.Positive(t, final.askPages, "coverage: continuation topology never asked") + } else { + require.Zero(t, final.askPages, "harness bug: direct topology must never ask") + } + coldFinal := coldConn.snapshotCounters() + require.Zero(t, coldFinal.pages304+coldFinal.pagesOverlay, + "harness bug: the cold connector must never replay") +} diff --git a/pkg/sync/source_cache_equivalence_soak_test.go b/pkg/sync/source_cache_equivalence_soak_test.go new file mode 100644 index 000000000..8d12931a2 --- /dev/null +++ b/pkg/sync/source_cache_equivalence_soak_test.go @@ -0,0 +1,31 @@ +package sync //nolint:revive,nolintlint // we can't change the package name for backwards compatibility + +// Soak sweep for the replay-equivalence harness: 20 additional seeds +// beyond TestSourceCache_ReplayEquivalence's fixed five, alternating +// worker counts and continuation topology. Gated behind BATON_SOAK so +// ordinary CI runs don't pay ~20x harness cost. +// +// TEMPORARY VERIFICATION FILE — safe to delete; nothing references it. + +import ( + "fmt" + "os" + "testing" +) + +func TestSourceCache_ReplayEquivalenceSoak(t *testing.T) { + if os.Getenv("BATON_SOAK") == "" { + t.Skip("set BATON_SOAK=1 to run the seed soak") + } + for seed := int64(6); seed <= 25; seed++ { + workers := 0 + if seed%2 == 0 { + workers = 4 + } + continuation := seed%3 == 0 + name := fmt.Sprintf("workers=%d/seed=%d/continuation=%v", workers, seed, continuation) + t.Run(name, func(t *testing.T) { + runReplayEquivalenceScenario(t, workers, seed, continuation) + }) + } +} diff --git a/pkg/sync/source_cache_equivalence_test.go b/pkg/sync/source_cache_equivalence_test.go new file mode 100644 index 000000000..1681352ba --- /dev/null +++ b/pkg/sync/source_cache_equivalence_test.go @@ -0,0 +1,2232 @@ +package sync //nolint:revive,nolintlint // we can't change the package name for backwards compatibility + +// Replay-equivalence differential harness. +// +// THE PROPERTY: for any upstream state, a warm sync (replaying from the +// previous sync's c1z) must produce a c1z semantically identical to a +// cold sync of the same upstream state. This is the contract stated in +// pkg/sourcecache ("a cached sync must reproduce what a full resync +// would produce"), enforced here as a differential test instead of +// relying on humans to remember every response-row side effect. +// +// SHAPE: a fixed, feature-complete topology exercising every feature +// that interacts with replay — +// - parent/child resources (ChildResourceType scheduling), +// - grant expansion (GrantExpandable + needs_expansion re-arming), +// - external-resource matching (MatchID and MatchAll, with the +// external c1z churning UNDER a 304 of the internal scope — the one +// shape where a missed re-arm is observable), +// - InsertResourceGrants (grant-driven resource insertion), +// - exclusion-group annotations on replayed entitlements, +// - both delta tombstone channels (canonical grant ids on the replay +// annotation; bare principal ids on the scope annotation), +// - 304 rounds, delta-overlay rounds, cold refetch (validator +// rotation), zero-row scopes, and scopes appearing mid-chain — +// mutated across rounds by a forced schedule (guaranteeing each behavior +// occurs regardless of seed) plus seeded random churn (interleavings). +// The warm side CHAINS: round r replays from round r-1's warm output, so +// decay compounds instead of hiding. +// +// ANTI-VACUITY: the harness fails itself if it did not actually exercise +// what it claims — per-round and per-scenario counters assert 304s, +// overlays, cold refetches, both tombstone channels, child fetches, and +// the presence of expansion-derived, match-transformed, and +// grant-inserted artifacts in the outputs. A trivially-green run is +// treated as a harness bug. +// +// COMPARISON: full semantic diff of both files — resource types, +// resources, entitlements, grants, all fields, annotations normalized by +// sort — read through MULTIPLE paths (primary listings, by-parent +// listings, by-entitlement-resource grant listings) so replay-time index +// synthesis bugs surface, not just primary-row differences. +// +// Also compared, beyond the v2 read surface: the source-cache MANIFEST +// and per-scope INDEX COUNTS of both files. These are invisible to v2 +// reads but poison FUTURE syncs' replays — a warm sync could produce +// perfect v2 data while writing wrong stamps or phantom manifest +// entries, and a purely v2-level comparison would pass until the damage +// detonated rounds later. Row listings are also duplicate-checked (the +// deduping snapshot maps would otherwise hide pagination double-yields). +// +// Round 2's warm sync is additionally CRASHED mid-grants-phase (injected +// connector error after other scopes replayed) and resumed into the same +// file, asserting resume-not-restart — so page re-runs over partially +// replayed state (replay idempotency, checkpoint interplay) are part of +// the compared output, every scenario. +// +// THE DIRTY FAMILY (source_cache_equivalence_dirty_test.go) is this +// harness's DEFAULT-MODE sibling: connectors that emit referentially +// broken data (I7/I8/I9 dangling shapes, mixed exempt-carrier +// populations), run WITHOUT fail-fast so the drop arms execute inside +// the equivalence property, with the warning-silence oracle inverted +// (expected drop warnings must be PRESENT). Every scenario in THIS file +// stays fail-fast and warning-silent; the dirty knobs on the model +// (ghostAdminGrant, ghostTeamEnt, extraTeamResources, entsVersion) are +// zero here and must stay zero. +// +// EXPLICITLY NOT COVERED HERE (do not mistake green for proof of these): +// - error-outcome parity (covered by the dedicated exclusion-group +// parity test in source_cache_sideeffects_test.go); +// - the ask/answer lookup continuation topology (covered by +// source_cache_continuation_test.go / _e2e_test.go); +// - EnqueuePageTokens / type-scoped grants and entitlements +// (spawn_cursors_test.go, type_scoped_grants_test.go, +// type_scoped_entitlements_test.go); the team topology below also +// exercises both type-scoped shapes in the differential chain; +// - compaction in the warm chain (covered separately by +// pkg/synccompactor's TestCompactedFileIsColdReplaySource — an import +// cycle keeps the compactor out of this package; the contract is that +// compacted artifacts carry NO manifest entries and replay cold); +// - concurrent syncs; SQLite (source cache is Pebble-only); +// - delta tombstones combined with external-match annotations (a +// documented contract limitation — the model deliberately never +// generates that combination). +// +// The model DELIBERATELY overlaps MatchID and MatchAll on one +// entitlement (both resolving the same principals): the harness's first +// run caught that colliding transformed grants were resolved by store +// iteration order, which differs between cold syncs (iterating source +// grants) and warm syncs (iterating replayed transformed grants). +// processGrantsWithExternalPrincipals now resolves collisions by rule +// specificity; this topology pins that. + +import ( + "context" + "errors" + "fmt" + "math/rand" + "path/filepath" + "sort" + "strings" + stdsync "sync" + "testing" + + "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" + "github.com/stretchr/testify/require" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/encoding/prototext" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/anypb" + + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + "github.com/conductorone/baton-sdk/pkg/annotations" + "github.com/conductorone/baton-sdk/pkg/dotc1z" + "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" + pebbleengine "github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble" + "github.com/conductorone/baton-sdk/pkg/logging" + "github.com/conductorone/baton-sdk/pkg/sourcecache" + "github.com/conductorone/baton-sdk/pkg/types" + et "github.com/conductorone/baton-sdk/pkg/types/entitlement" + gt "github.com/conductorone/baton-sdk/pkg/types/grant" + rs "github.com/conductorone/baton-sdk/pkg/types/resource" +) + +// --------------------------------------------------------------------- +// Upstream model +// --------------------------------------------------------------------- + +var ( + equivOrgRT = v2.ResourceType_builder{Id: "eq_org", DisplayName: "Org"}.Build() + equivProjectRT = v2.ResourceType_builder{Id: "eq_project", DisplayName: "Project"}.Build() + equivRepoRT = v2.ResourceType_builder{Id: "eq_repo", DisplayName: "Repo"}.Build() + // equivTeamRT is TYPE-SCOPED for both grants and entitlements: no + // per-resource pages; one planner call per phase spawns chunk + // cursors (EnqueuePageTokens), each chunk a scope covering two teams — + // the Entra/Okta planner shape. + equivTeamRT = v2.ResourceType_builder{ + Id: "eq_team", + DisplayName: "Team", + Annotations: annotations.New(&v2.TypeScopedGrants{}, &v2.TypeScopedEntitlements{}), + }.Build() +) + +type equivGroup struct { + id string + members map[string]bool + + // version is the grants scope's validator generation, bumped at most + // ONCE per mutation round (dirty guards it) so a round with several + // membership changes still reads as exactly one generation behind — + // the delta-overlay shape. delta* is the coalesced diff between + // version-1 and version (nil when the group was untouched last + // round). canonicalTombstones selects which delta channel removals + // ride: canonical grant ids on the replay annotation, or bare + // principal ids on the scope annotation. + version int + dirty bool + deltaAdds []string + deltaRemoves []string + canonicalTombstones bool + + // exclusionDefault attaches a (valid, single-member) exclusion-group + // default annotation to the member entitlement, so replayed + // entitlement rows carry exclusion state through validation. + exclusionDefault bool + + // Dirty-connector knobs (see the dirty scenario family in + // source_cache_equivalence_dirty_test.go). All zero for the clean + // model, keeping clean-scenario behavior byte-identical. + // + // ghostAdminGrant makes the group's grants page emit a grant on the + // SHADOW host resource's "admin" entitlement (see + // equivShadowAdminGrant). While the shadow type is disabled the + // grant is an I8-class dangling reference inside a stamped scope + // (an expected configuration gap: the referent's TYPE is not + // synced); enabling the shadow type is the "referent appears" + // transition while the grants scope's validator stays put. + ghostAdminGrant bool + // entsVersion is the group's ENTITLEMENTS scope validator generation + // (the clean model never rotates it). + entsVersion int +} + +type equivOrg struct { + id string + projects map[string]string // project id -> display name + // projectsVersion is the per-org project-listing scope generation. + projectsVersion int + // repoGrants: repo id -> user id holding "admin". The org's grants + // page carries InsertResourceGrants; repoVersion is its generation + // (fresh-on-change, 304 when unchanged — no delta). + repoGrants map[string]string + repoVersion int +} + +type equivModel struct { + users map[string]bool + usersVersion int + + orgs map[string]*equivOrg + orgsVersion int + + groups map[string]*equivGroup + groupsVersion int + + // extGroup's grants page emits ExternalResourceMatchID grants (one + // per entry of extMatchIDs) plus one ExternalResourceMatchAll(USER) + // grant. extVersion is that scope's generation. extUsers is the + // EXTERNAL c1z's content — mutating it WITHOUT bumping extVersion is + // the divergence trap for the match re-arm path. + extGroupID string + extMatchIDs []string + extVersion int + extUsers map[string]bool + extDirty bool // external c1z must be rebuilt before the next sync + + // Expansion: expandableParent's grants scope permanently contains an + // expandable membership of expandableChild, so members of the child + // derive grants on the parent. Churning the child changes expansion + // output while the parent's scope 304s. + expandableParent string + expandableChild string + + // multiPageGroup's grants scope serves multi-page rounds: fresh + // fetches AND delta overlays span two pages, with the validator only + // on the final page and tombstones riding the final page's scope + // annotation (the page-spanning delta contract). + multiPageGroup string + + // teams (type-scoped grants): team id -> members. Teams are fixed; + // only membership churns. teamChunkVersions[i] is chunk i's + // validator generation (chunk = two teams by sorted order); chunks + // are fresh-or-304 scopes, no deltas. + teams map[string]map[string]bool + teamChunkVersions []int + + // Dirty-connector knobs (zero for the clean model; see the dirty + // scenario family in source_cache_equivalence_dirty_test.go). + // + // ghostTeamEnt makes chunk 0's type-scoped ENTITLEMENTS page emit an + // entitlement for the SHADOW resource "shadow-host2" — and chunk 0's + // grants page a grant under it. While the shadow type is disabled + // the entitlement is an I7-class dangling reference (and the grant + // its I8 cascade) — expected configuration gaps, since the referent + // TYPE is not synced — inside stamped type-scoped scopes whose + // validators never rotate. + ghostTeamEnt bool + // shadowEnabled "enables the resource type" eq_shadow: the type is + // listed in resource types and its resources (shadowUsers + the two + // fixed hosts) are listed fresh each sync. The dirty family's + // dirty→clean transition flips this while every dangling scope's + // validator stays put. + shadowEnabled bool + // shadowUsers are the shadow-typed principals dirty groups reference + // as members (ids prefixed "shadow-"). Rows exist only while + // shadowEnabled. + shadowUsers map[string]bool +} + +// teamChunkOf returns the chunk index a team belongs to (two teams per +// chunk, by sorted team id). Teams are never added or removed, so the +// mapping is stable across rounds. +func (m *equivModel) teamChunkOf(teamID string) int { + ids := sortedKeys(m.teams) + for i, id := range ids { + if id == teamID { + return i / 2 + } + } + panic("unknown team " + teamID) +} + +func (m *equivModel) teamsInChunk(chunk int) []string { + ids := sortedKeys(m.teams) + var out []string + for i, id := range ids { + if i/2 == chunk { + out = append(out, id) + } + } + return out +} + +func (m *equivModel) mutateTeam(teamID, uid string, add bool) { + members := m.teams[teamID] + if add == members[uid] { + return + } + if add { + members[uid] = true + } else { + delete(members, uid) + } + m.teamChunkVersions[m.teamChunkOf(teamID)]++ +} + +func newEquivModel() *equivModel { + m := &equivModel{ + users: map[string]bool{}, + orgs: map[string]*equivOrg{}, + groups: map[string]*equivGroup{}, + + extGroupID: "gext", + extMatchIDs: []string{"ext-0", "ext-1"}, + extUsers: map[string]bool{"ext-0": true, "ext-1": true, "ext-2": true}, + extDirty: true, + + expandableParent: "g0", + expandableChild: "g3", + multiPageGroup: "g1", + + teams: map[string]map[string]bool{ + "t0": {"u0": true, "u1": true}, + "t1": {"u2": true}, + "t2": {"u3": true, "u4": true}, + "t3": {"u9": true}, + }, + teamChunkVersions: []int{0, 0}, + } + for i := 0; i < 10; i++ { + m.users[fmt.Sprintf("u%d", i)] = true + } + m.orgs["org0"] = &equivOrg{ + id: "org0", + projects: map[string]string{"p0": "Project 0", "p1": "Project 1"}, + repoGrants: map[string]string{"repo0": "u0"}, + } + m.orgs["org1"] = &equivOrg{ + id: "org1", + projects: map[string]string{"p2": "Project 2", "p3": "Project 3"}, + repoGrants: map[string]string{}, // zero-row scope: entry must still persist + } + m.groups["g0"] = &equivGroup{id: "g0", members: map[string]bool{"u0": true, "u1": true}, canonicalTombstones: true, exclusionDefault: true} + m.groups["g1"] = &equivGroup{id: "g1", members: map[string]bool{"u2": true, "u3": true}, exclusionDefault: true} + m.groups["g2"] = &equivGroup{id: "g2", members: map[string]bool{"u4": true}} + m.groups["g3"] = &equivGroup{id: "g3", members: map[string]bool{"u5": true, "u6": true}} + m.groups[m.extGroupID] = &equivGroup{id: m.extGroupID, members: map[string]bool{}} + return m +} + +// clearDeltas resets last round's diffs; called at the start of each +// mutation round so overlays only ever describe one round's changes. +func (m *equivModel) clearDeltas() { + for _, g := range m.groups { + g.deltaAdds, g.deltaRemoves = nil, nil + g.dirty = false + } + m.extDirty = false +} + +func (g *equivGroup) touch() { + if !g.dirty { + g.dirty = true + g.version++ + } +} + +func (m *equivModel) addMember(gid, uid string) { + g := m.groups[gid] + if g.members[uid] { + return + } + g.members[uid] = true + g.touch() + g.deltaAdds = append(g.deltaAdds, uid) +} + +func (m *equivModel) removeMember(gid, uid string) { + g := m.groups[gid] + if !g.members[uid] { + return + } + delete(g.members, uid) + g.touch() + g.deltaRemoves = append(g.deltaRemoves, uid) +} + +func sortedKeys[V any](in map[string]V) []string { + out := make([]string, 0, len(in)) + for k := range in { + out = append(out, k) + } + sort.Strings(out) + return out +} + +// mutate applies round r's forced schedule plus seeded churn. The forced +// schedule guarantees every replay behavior occurs at least once per +// scenario regardless of seed; see the round comments. +func (m *equivModel) mutate(round int, rng *rand.Rand) { + m.clearDeltas() + switch round { + case 1: + // Canonical-id tombstone (g0), principal-id tombstone on the + // MULTI-PAGE overlay group (g1), overlay add (g2), project + // rename (cold refetch of a resources scope), repo grant add + // (fresh InsertResourceGrants page), new external user + match + // (external file rebuild + fresh page). + m.removeMember("g0", "u1") + m.removeMember("g1", "u3") + m.addMember("g1", "u9") + m.addMember("g2", "u7") + m.orgs["org0"].projects["p0"] = "Project 0 (renamed)" + m.orgs["org0"].projectsVersion++ + m.orgs["org0"].repoGrants["repo1"] = "u2" + m.orgs["org0"].repoVersion++ + m.extUsers["ext-3"] = true + m.extMatchIDs = append(m.extMatchIDs, "ext-3") + m.extVersion++ + m.extDirty = true + // Team churn: chunk 0 refetches, chunk 1 must 304. + m.mutateTeam("t0", "u5", true) + case 2: + // THE DIVERGENCE TRAP: the external c1z changes while the + // internal match scope's validator does NOT rotate (the internal + // source grants are unchanged, so the connector legitimately + // 304s). Cold output reflects the new external set; a warm sync + // that fails to re-run match processing keeps stale transformed + // grants. Also: user-set refetch (multi-page fresh), repo grant + // removal, and expansion input churn under the parent's 304. + // The RUNNER additionally crashes and resumes this round's warm + // sync mid-grants-phase. + delete(m.extUsers, "ext-1") + m.extDirty = true + m.users["u10"] = true + m.usersVersion++ + delete(m.orgs["org0"].repoGrants, "repo1") + m.orgs["org0"].repoVersion++ + m.addMember(m.expandableChild, "u8") + m.removeMember(m.expandableChild, "u5") + case 3: + // Deletions: a project, a WHOLE group, and a user vanish (their + // rows must vanish from the warm output exactly as from a cold + // one — rows only enter a sync via fresh pages or replay, so a + // leftover here means a replay copied outside its scope). A new + // group also appears mid-chain (cold scopes while everything + // else replays), plus seeded churn. + delete(m.orgs["org0"].projects, "p0") + m.orgs["org0"].projectsVersion++ + delete(m.groups, "g2") + m.groupsVersion++ + delete(m.users, "u7") // was only ever a member of g2 + m.usersVersion++ + ng := &equivGroup{id: "g4", members: map[string]bool{"u9": true}, canonicalTombstones: rng.Intn(2) == 0} + m.groups[ng.id] = ng + m.mutateTeam("t0", "u5", false) + m.randomChurn(rng) + default: + // Round 4: deliberate no-op — every scope must 304 and the + // output must STILL match a cold sync (chain-decay probe). + } +} + +func (m *equivModel) randomChurn(rng *rand.Rand) { + users := sortedKeys(m.users) + for _, gid := range sortedKeys(m.groups) { + if gid == m.extGroupID { + continue // never mix delta churn into the match scope + } + g := m.groups[gid] + switch rng.Intn(3) { + case 0: // add an absent user + for _, u := range users { + if !g.members[u] { + m.addMember(gid, u) + break + } + } + case 1: // remove a present member (keep expansion child non-empty) + members := sortedKeys(g.members) + if len(members) > 1 { + m.removeMember(gid, members[rng.Intn(len(members))]) + } + default: // untouched: must 304 + } + } +} + +// --------------------------------------------------------------------- +// Connector +// --------------------------------------------------------------------- + +type equivCounters struct { + pages304 int + pagesOverlay int // final (or only) page of an overlay round + pagesOverlayInterim int // non-final overlay page (no validator yet) + pagesColdStale int // lookup hit but validator stale -> full refetch + pagesFresh int // lookup miss -> full fetch (final page) + pagesFreshInterim int // non-final fresh page (no validator yet) + tombsCanonical int + tombsPrincipal int + childPageCalls int + injectedFailures int + plannerPages int // type-scoped planning calls (EnqueuePageTokens emitters) + chunkPages304 int // spawned chunk cursors that replayed + chunkPagesFresh int // spawned chunk cursors fetched fresh + askPages int // continuation phase-1 responses (lookup asks) +} + +// equivConnector serves the model. forceCold ignores the lookup and +// always emits full fresh pages (still scope-annotated — stamps are +// invisible to the comparator), which is exactly what a from-scratch +// resync sees. +type equivConnector struct { + *mockConnector + + mu stdsync.Mutex + model *equivModel + lookup sourcecache.Lookup + forceCold bool + // continuation switches the lookup topology: instead of the direct + // SetSourceCache lookup, lookups are served from request-delivered + // SourceCacheLookupAnswers and deferred via SourceCacheLookupAsk + // responses — the single-shot (Lambda) transport, played by hand. + // The connector must be wrapped in equivContinuationClient so the + // syncer does not see a SetLookup implementation. + continuation bool + counters equivCounters + // failNext injects one transient failure per key ("grants:"), + // simulating a crash mid-sync; the runner resumes the same file and + // the resumed output is what gets compared. + failNext map[string]bool +} + +// equivContinuationClient hides the embedded SetSourceCache from the +// syncer's sourcecache.SourceCacheSetter type assertion, making the connector a +// single-shot transport: the syncer attaches SourceCacheLookupOffer to +// requests and the connector answers with asks. +type equivContinuationClient struct { + *equivConnector +} + +// SetSourceCache shadows the embedded method with a different signature +// so equivContinuationClient does NOT satisfy sourcecache.SourceCacheSetter. +func (equivContinuationClient) SetSourceCache() {} + +func (c *equivConnector) injectGrantsFailure(resourceID string) { + c.mu.Lock() + defer c.mu.Unlock() + if c.failNext == nil { + c.failNext = map[string]bool{} + } + c.failNext["grants:"+resourceID] = true +} + +func newEquivConnector(model *equivModel, forceCold bool) *equivConnector { + mc := &equivConnector{mockConnector: newMockConnector(), model: model, forceCold: forceCold} + mc.rtDB = []*v2.ResourceType{equivOrgRT, equivProjectRT, equivRepoRT, equivTeamRT, groupResourceType, userResourceType} + return mc +} + +func (c *equivConnector) SetSourceCache(_ context.Context, lookup sourcecache.Lookup) { + c.mu.Lock() + defer c.mu.Unlock() + c.lookup = lookup +} + +// ListResourceTypes includes the dirty family's shadow type only while +// the model has "enabled" it — the store-level type row is what the +// invariants' enabled/disabled attribution probes, so the type must be +// genuinely absent before the transition. Clean scenarios never enable +// it and see exactly rtDB. +func (c *equivConnector) ListResourceTypes(context.Context, *v2.ResourceTypesServiceListResourceTypesRequest, ...grpc.CallOption) (*v2.ResourceTypesServiceListResourceTypesResponse, error) { + c.mu.Lock() + enabled := c.model.shadowEnabled + c.mu.Unlock() + list := c.rtDB + if enabled { + list = append(append([]*v2.ResourceType{}, list...), equivShadowRT) + } + return v2.ResourceTypesServiceListResourceTypesResponse_builder{List: list}.Build(), nil +} + +func (c *equivConnector) Validate(context.Context, *v2.ConnectorServiceValidateRequest, ...grpc.CallOption) (*v2.ConnectorServiceValidateResponse, error) { + return v2.ConnectorServiceValidateResponse_builder{ + Annotations: annotations.New(v2.SourceCacheCapability_builder{ + Mode: v2.SourceCacheCapability_MODE_READ_WRITE, + }.Build()), + }.Build(), nil +} + +func equivEtag(version int) string { return fmt.Sprintf("v%d", version) } + +// revalidate consults the lookup for (kind, scope) and classifies the +// round: fresh (miss), current (304), overlay-eligible (exactly one +// generation behind), or stale (older: full refetch). forceCold always +// reports fresh. +type equivVerdict int + +const ( + equivFresh equivVerdict = iota + equivCurrent + equivOneBehind + equivStale +) + +// requestLookup resolves the lookup for one request. Direct topology: +// the SetSourceCache-installed lookup. Continuation topology: a +// per-request ContinuationLookup built from the request's answers (nil +// when the request carries neither offer nor answers — cold requests +// must not ask). The returned *ContinuationLookup is non-nil only when +// deferred lookups must be converted into an ask response. +func (c *equivConnector) requestLookup(reqAnnos []*anypb.Any) (sourcecache.Lookup, *sourcecache.ContinuationLookup, error) { + if !c.continuation { + c.mu.Lock() + lookup := c.lookup + c.mu.Unlock() + if lookup == nil { + lookup = sourcecache.NoopLookup{} + } + return lookup, nil, nil + } + annos := annotations.Annotations(reqAnnos) + answersMsg := &v2.SourceCacheLookupAnswers{} + hasAnswers, err := annos.Pick(answersMsg) + if err != nil { + return nil, nil, err + } + if !hasAnswers && !annos.Contains(&v2.SourceCacheLookupOffer{}) { + return sourcecache.NoopLookup{}, nil, nil + } + var answers []sourcecache.Answer + if hasAnswers { + answers, err = sourcecache.AnswersFromProto(answersMsg) + if err != nil { + return nil, nil, err + } + } + cl := sourcecache.NewContinuationLookup(answers) + return cl, cl, nil +} + +// askAnnos converts a deferred lookup into the ask response's +// annotations, counting the phase-1 turn. +func (c *equivConnector) askAnnos(cl *sourcecache.ContinuationLookup) annotations.Annotations { + c.count(func(ct *equivCounters) { ct.askPages++ }) + return annotations.New(sourcecache.AskProto(cl.Asked())) +} + +// isDeferred reports whether err is a deferred continuation lookup that +// must become an ask response (only possible when cl is non-nil). +func isDeferred(cl *sourcecache.ContinuationLookup, err error) bool { + return cl != nil && err != nil && errors.Is(err, sourcecache.ErrLookupDeferred) +} + +func (c *equivConnector) revalidate(ctx context.Context, lookup sourcecache.Lookup, kind sourcecache.RowKind, scope string, version int) (equivVerdict, error) { + if c.forceCold { + return equivFresh, nil + } + entry, found, err := lookup.Lookup(ctx, kind, scope) + if err != nil { + return equivFresh, err + } + if !found { + return equivFresh, nil + } + switch entry.CacheValidator { + case equivEtag(version): + return equivCurrent, nil + case equivEtag(version - 1): + return equivOneBehind, nil + default: + return equivStale, nil + } +} + +func (c *equivConnector) count(f func(*equivCounters)) { + c.mu.Lock() + defer c.mu.Unlock() + f(&c.counters) +} + +func (c *equivConnector) snapshotCounters() equivCounters { + c.mu.Lock() + defer c.mu.Unlock() + return c.counters +} + +func equivScopeAnnos(scope string, version int) annotations.Annotations { + return annotations.New(v2.SourceCacheRecord_builder{ + ScopeKey: sourcecache.HashScope(scope), + CacheValidator: equivEtag(version), + }.Build()) +} + +func equivReplayAnnos(scope string, version int) annotations.Annotations { + return annotations.New(v2.SourceCacheReplay_builder{ + ScopeKey: sourcecache.HashScope(scope), + CacheValidator: equivEtag(version), + }.Build()) +} + +// --- resource builders (deterministic; shared by fresh pages) --------- + +func equivOrgResource(id string) *v2.Resource { + r, err := rs.NewResource("Org "+id, equivOrgRT, id, + rs.WithAnnotation(v2.ChildResourceType_builder{ResourceTypeId: equivProjectRT.GetId()}.Build()), + rs.WithAnnotation(&v2.SkipEntitlements{}), + ) + if err != nil { + panic(err) + } + return r +} + +func equivProjectResource(orgID, id, displayName string) *v2.Resource { + r, err := rs.NewResource(displayName, equivProjectRT, id, + rs.WithParentResourceID(v2.ResourceId_builder{ResourceType: equivOrgRT.GetId(), Resource: orgID}.Build()), + rs.WithAnnotation(&v2.SkipEntitlementsAndGrants{}), + ) + if err != nil { + panic(err) + } + return r +} + +func equivGroupResource(id string) *v2.Resource { + r, err := rs.NewGroupResource(id, groupResourceType, id, nil) + if err != nil { + panic(err) + } + return r +} + +func equivUserResource(id string) *v2.Resource { + r, err := rs.NewUserResource(id, userResourceType, id, nil, rs.WithAnnotation(&v2.SkipEntitlementsAndGrants{})) + if err != nil { + panic(err) + } + return r +} + +func equivRepoResource(id string) *v2.Resource { + r, err := rs.NewResource("Repo "+id, equivRepoRT, id, rs.WithAnnotation(&v2.SkipEntitlementsAndGrants{})) + if err != nil { + panic(err) + } + return r +} + +func equivTeamResource(id string) *v2.Resource { + r, err := rs.NewResource("Team "+id, equivTeamRT, id) + if err != nil { + panic(err) + } + return r +} + +func equivTeamGrant(tid, uid string) *v2.Grant { + return gt.NewGrant(equivTeamResource(tid), "member", + v2.ResourceId_builder{ResourceType: userResourceType.GetId(), Resource: uid}.Build()) +} + +func equivMemberEnt(g *v2.Resource, exclusionDefault bool) *v2.Entitlement { + ent := et.NewAssignmentEntitlement(g, "member", et.WithGrantableTo(groupResourceType, userResourceType)) + ent.SetSlug("member") + if exclusionDefault { + annos := annotations.Annotations(ent.GetAnnotations()) + annos.Update(v2.EntitlementExclusionGroup_builder{ + ExclusionGroupId: "eg-" + g.GetId().GetResource(), + IsDefault: true, + }.Build()) + ent.SetAnnotations(annos) + } + return ent +} + +// equivMemberGrant builds a group-membership grant. Principal type +// routes by id prefix for the dirty family: "shadow-" members are the +// toggleable disabled type (eq_shadow), "ghost2-" members a type that +// NEVER gets enabled (steady-dirty shapes). Clean-model member ids use +// neither prefix and stay user-typed. +func equivMemberGrant(gid, uid string) *v2.Grant { + principalType := userResourceType.GetId() + switch { + case strings.HasPrefix(uid, "shadow-"): + principalType = equivShadowRT.GetId() + case strings.HasPrefix(uid, "ghost2-"): + principalType = equivGhost2TypeID + } + return gt.NewGrant(equivGroupResource(gid), "member", + v2.ResourceId_builder{ResourceType: principalType, Resource: uid}.Build()) +} + +// equivGhost2TypeID is the dirty family's never-enabled principal type: +// no resource type row ever exists for it, so references stay expected +// configuration gaps (the drop arm) in every round. +const equivGhost2TypeID = "eq_ghost2" + +// --- dirty-family shadow builders ------------------------------------- +// +// equivShadowRT is the dirty family's toggleable resource type: DISABLED +// (never listed) until the model's shadowEnabled flips. References into +// it are expected configuration gaps — the replay policy's DROP arm — +// and enabling it is the dirty→clean "referent appears" transition. +// Never used by clean scenarios. +var equivShadowRT = v2.ResourceType_builder{Id: "eq_shadow", DisplayName: "Shadow"}.Build() + +const ( + // equivShadowHostID hosts the "admin" entitlement ghostAdminGrant + // references (the I8 dirty shape). + equivShadowHostID = "shadow-host" + // equivShadowHost2ID hosts the entitlement the ghost team-chunk + // emission references (the I7 dirty shape + I8 cascade). + equivShadowHost2ID = "shadow-host2" +) + +func equivShadowResource(id string) *v2.Resource { + r, err := rs.NewResource("Shadow "+id, equivShadowRT, id, rs.WithAnnotation(&v2.SkipEntitlementsAndGrants{})) + if err != nil { + panic(err) + } + return r +} + +// equivShadowHostResource is a shadow host WITHOUT the skip annotation: +// hosts serve entitlement pages once the type is enabled. +func equivShadowHostResource(id string) *v2.Resource { + r, err := rs.NewResource("Shadow "+id, equivShadowRT, id) + if err != nil { + panic(err) + } + return r +} + +// equivShadowAdminEnt / equivShadowAdminGrant: the I8 dirty shape — the +// grant is emitted (inside a group's stamped grants scope) whether or +// not the shadow type is enabled; the entitlement row exists only once +// it is. +func equivShadowAdminEnt() *v2.Entitlement { + ent := et.NewAssignmentEntitlement(equivShadowHostResource(equivShadowHostID), "admin", et.WithGrantableTo(userResourceType)) + ent.SetSlug("admin") + return ent +} + +func equivShadowAdminGrant(uid string) *v2.Grant { + return gt.NewGrant(equivShadowHostResource(equivShadowHostID), "admin", + v2.ResourceId_builder{ResourceType: userResourceType.GetId(), Resource: uid}.Build()) +} + +// equivShadowHost2Ent / equivShadowHost2Grant: the I7 dirty shape and +// its I8 cascade, emitted inside the type-scoped team chunk-0 scopes. +func equivShadowHost2Ent() *v2.Entitlement { + return equivMemberEnt(equivShadowHostResource(equivShadowHost2ID), false) +} + +func equivShadowHost2Grant(uid string) *v2.Grant { + return gt.NewGrant(equivShadowHostResource(equivShadowHost2ID), "member", + v2.ResourceId_builder{ResourceType: userResourceType.GetId(), Resource: uid}.Build()) +} + +// --- list RPCs --------------------------------------------------------- + +func (c *equivConnector) ListResources(ctx context.Context, in *v2.ResourcesServiceListResourcesRequest, opts ...grpc.CallOption) (*v2.ResourcesServiceListResourcesResponse, error) { + lookup, cl, err := c.requestLookup(in.GetAnnotations()) + if err != nil { + return nil, err + } + resp, err := c.listResourcesInner(ctx, lookup, in) + if isDeferred(cl, err) { + return v2.ResourcesServiceListResourcesResponse_builder{ + Annotations: c.askAnnos(cl), + }.Build(), nil + } + return resp, err +} + +func (c *equivConnector) listResourcesInner(ctx context.Context, lookup sourcecache.Lookup, in *v2.ResourcesServiceListResourcesRequest) (*v2.ResourcesServiceListResourcesResponse, error) { + c.mu.Lock() + m := c.model + switch in.GetResourceTypeId() { + case equivOrgRT.GetId(): + version := m.orgsVersion + orgIDs := sortedKeys(m.orgs) + c.mu.Unlock() + return c.resourcesPage(ctx, lookup, "res:org", version, func() []*v2.Resource { + out := make([]*v2.Resource, 0, len(orgIDs)) + for _, id := range orgIDs { + out = append(out, equivOrgResource(id)) + } + return out + }) + case equivProjectRT.GetId(): + if in.GetParentResourceId().GetResource() == "" { + c.mu.Unlock() + return v2.ResourcesServiceListResourcesResponse_builder{}.Build(), nil + } + orgID := in.GetParentResourceId().GetResource() + org := m.orgs[orgID] + if org == nil { + c.mu.Unlock() + return v2.ResourcesServiceListResourcesResponse_builder{}.Build(), nil + } + version := org.projectsVersion + projects := make(map[string]string, len(org.projects)) + for k, v := range org.projects { + projects[k] = v + } + c.mu.Unlock() + c.count(func(ct *equivCounters) { ct.childPageCalls++ }) + return c.resourcesPage(ctx, lookup, "res:project:"+orgID, version, func() []*v2.Resource { + out := make([]*v2.Resource, 0, len(projects)) + for _, id := range sortedKeys(projects) { + out = append(out, equivProjectResource(orgID, id, projects[id])) + } + return out + }) + case groupResourceType.GetId(): + version := m.groupsVersion + groupIDs := sortedKeys(m.groups) + c.mu.Unlock() + return c.resourcesPage(ctx, lookup, "res:group", version, func() []*v2.Resource { + out := make([]*v2.Resource, 0, len(groupIDs)) + for _, id := range groupIDs { + out = append(out, equivGroupResource(id)) + } + return out + }) + case equivTeamRT.GetId(): + // Teams are UNSCOPED: listed fresh every sync (mixes never-scoped + // pages into an otherwise warm sync). + teamIDs := sortedKeys(m.teams) + c.mu.Unlock() + out := make([]*v2.Resource, 0, len(teamIDs)) + for _, id := range teamIDs { + out = append(out, equivTeamResource(id)) + } + return v2.ResourcesServiceListResourcesResponse_builder{List: out}.Build(), nil + case equivShadowRT.GetId(): + // The dirty family's "enable the resource type" switch: listed + // UNSCOPED and fresh (like teams) only while enabled; before + // that, references into the type are expected configuration + // gaps that the invariants drop. + if !m.shadowEnabled { + c.mu.Unlock() + return v2.ResourcesServiceListResourcesResponse_builder{}.Build(), nil + } + ids := sortedKeys(m.shadowUsers) + ids = append(ids, equivShadowHostID, equivShadowHost2ID) + sort.Strings(ids) + c.mu.Unlock() + out := make([]*v2.Resource, 0, len(ids)) + for _, id := range ids { + if id == equivShadowHostID { + // The admin-ent host serves an entitlements page (no + // skip annotation); everything else is skip-annotated + // (host2's entitlement arrives via the team chunk). + out = append(out, equivShadowHostResource(id)) + continue + } + out = append(out, equivShadowResource(id)) + } + return v2.ResourcesServiceListResourcesResponse_builder{List: out}.Build(), nil + case userResourceType.GetId(): + version := m.usersVersion + userIDs := sortedKeys(m.users) + c.mu.Unlock() + return c.usersPageMulti(ctx, lookup, in.GetPageToken(), version, userIDs) + default: // repo: insert-only, never listed + c.mu.Unlock() + return v2.ResourcesServiceListResourcesResponse_builder{}.Build(), nil + } +} + +// usersPageMulti serves the users scope as a MULTI-PAGE scope: fresh +// fetches span two pages, with the scope annotation on both but the +// validator only on the final page (the interim-page contract). A 304 is +// a single page like any other scope. +func (c *equivConnector) usersPageMulti(ctx context.Context, lookup sourcecache.Lookup, pageToken string, version int, userIDs []string) (*v2.ResourcesServiceListResourcesResponse, error) { + const scope = "res:user" + verdict, err := c.revalidate(ctx, lookup, sourcecache.RowKindResources, sourcecache.HashScope(scope), version) + if err != nil { + return nil, err + } + if verdict == equivCurrent { + c.count(func(ct *equivCounters) { ct.pages304++ }) + return v2.ResourcesServiceListResourcesResponse_builder{ + Annotations: equivReplayAnnos(scope, version), + }.Build(), nil + } + half := len(userIDs) / 2 + buildRows := func(ids []string) []*v2.Resource { + out := make([]*v2.Resource, 0, len(ids)) + for _, id := range ids { + out = append(out, equivUserResource(id)) + } + return out + } + if pageToken == "" { + c.count(func(ct *equivCounters) { ct.pagesFreshInterim++ }) + return v2.ResourcesServiceListResourcesResponse_builder{ + List: buildRows(userIDs[:half]), + Annotations: annotations.New(v2.SourceCacheRecord_builder{ + ScopeKey: sourcecache.HashScope(scope), + // No etag: the validator arrives on the final page. + }.Build()), + NextPageToken: "users:2", + }.Build(), nil + } + c.count(func(ct *equivCounters) { + if verdict == equivFresh { + ct.pagesFresh++ + } else { + ct.pagesColdStale++ + } + }) + return v2.ResourcesServiceListResourcesResponse_builder{ + List: buildRows(userIDs[half:]), + Annotations: equivScopeAnnos(scope, version), + }.Build(), nil +} + +// resourcesPage implements the fresh/304/stale protocol for a resources +// scope (no deltas: resources scopes refetch fully on change). +func (c *equivConnector) resourcesPage(ctx context.Context, lookup sourcecache.Lookup, scope string, version int, build func() []*v2.Resource) (*v2.ResourcesServiceListResourcesResponse, error) { + verdict, err := c.revalidate(ctx, lookup, sourcecache.RowKindResources, sourcecache.HashScope(scope), version) + if err != nil { + return nil, err + } + if verdict == equivCurrent { + c.count(func(ct *equivCounters) { ct.pages304++ }) + return v2.ResourcesServiceListResourcesResponse_builder{ + Annotations: equivReplayAnnos(scope, version), + }.Build(), nil + } + c.count(func(ct *equivCounters) { + if verdict == equivFresh { + ct.pagesFresh++ + } else { + ct.pagesColdStale++ + } + }) + return v2.ResourcesServiceListResourcesResponse_builder{ + List: build(), + Annotations: equivScopeAnnos(scope, version), + }.Build(), nil +} + +func (c *equivConnector) ListEntitlements(ctx context.Context, in *v2.EntitlementsServiceListEntitlementsRequest, opts ...grpc.CallOption) (*v2.EntitlementsServiceListEntitlementsResponse, error) { + lookup, cl, err := c.requestLookup(in.GetAnnotations()) + if err != nil { + return nil, err + } + resp, err := c.listEntitlementsInner(ctx, lookup, in) + if isDeferred(cl, err) { + return v2.EntitlementsServiceListEntitlementsResponse_builder{ + Annotations: c.askAnnos(cl), + }.Build(), nil + } + return resp, err +} + +func (c *equivConnector) listEntitlementsInner( + ctx context.Context, + lookup sourcecache.Lookup, + in *v2.EntitlementsServiceListEntitlementsRequest, +) (*v2.EntitlementsServiceListEntitlementsResponse, error) { + rid := in.GetResource().GetId() + if rid.GetResourceType() == equivTeamRT.GetId() { + // Route on the REQUEST marker, not just the resource type: a + // syncer bug that drops the TypeScopedEntitlements annotation + // (e.g. on a continuation bounce or spawned cursor) must fail + // the harness, not silently keep working. + reqAnnos := annotations.Annotations(in.GetAnnotations()) + if !reqAnnos.Contains(&v2.TypeScopedEntitlements{}) { + return nil, fmt.Errorf("team entitlements call missing TypeScopedEntitlements request marker (page token %q)", in.GetPageToken()) + } + return c.teamEntitlementsPage(ctx, lookup, in.GetPageToken()) + } + if rid.GetResourceType() == equivShadowRT.GetId() { + // Shadow HOST entitlements: unscoped, fresh — served only while + // the type is enabled (the dirty family's dirty→clean referent). + c.mu.Lock() + enabled := c.model.shadowEnabled + c.mu.Unlock() + if enabled && rid.GetResource() == equivShadowHostID { + return v2.EntitlementsServiceListEntitlementsResponse_builder{ + List: []*v2.Entitlement{equivShadowAdminEnt()}, + }.Build(), nil + } + return v2.EntitlementsServiceListEntitlementsResponse_builder{}.Build(), nil + } + if rid.GetResourceType() != groupResourceType.GetId() { + return v2.EntitlementsServiceListEntitlementsResponse_builder{}.Build(), nil + } + c.mu.Lock() + g := c.model.groups[rid.GetResource()] + var entsVersion int + if g != nil { + entsVersion = g.entsVersion + } + c.mu.Unlock() + if g == nil { + return v2.EntitlementsServiceListEntitlementsResponse_builder{}.Build(), nil + } + scope := "ents:" + g.id + // Entitlement scopes never rotate after creation (entsVersion stays + // 0): warm rounds are pure 304s, which is exactly what keeps + // exclusion-group state on the replayed path. + verdict, err := c.revalidate(ctx, lookup, sourcecache.RowKindEntitlements, sourcecache.HashScope(scope), entsVersion) + if err != nil { + return nil, err + } + if verdict == equivCurrent { + c.count(func(ct *equivCounters) { ct.pages304++ }) + return v2.EntitlementsServiceListEntitlementsResponse_builder{ + Annotations: equivReplayAnnos(scope, entsVersion), + }.Build(), nil + } + c.count(func(ct *equivCounters) { ct.pagesFresh++ }) + return v2.EntitlementsServiceListEntitlementsResponse_builder{ + List: []*v2.Entitlement{equivMemberEnt(equivGroupResource(g.id), g.exclusionDefault)}, + Annotations: equivScopeAnnos(scope, entsVersion), + }.Build(), nil +} + +func (c *equivConnector) ListGrants(ctx context.Context, in *v2.GrantsServiceListGrantsRequest, opts ...grpc.CallOption) (*v2.GrantsServiceListGrantsResponse, error) { + rid := in.GetResource().GetId() + c.mu.Lock() + if c.failNext["grants:"+rid.GetResource()] { + delete(c.failNext, "grants:"+rid.GetResource()) + c.counters.injectedFailures++ + c.mu.Unlock() + // codes.Internal: NOT in the syncer's retryable set (only + // Unavailable/DeadlineExceeded are), so the sync dies here — + // the process-crash analog the resume path exists for. + return nil, status.Error(codes.Internal, "injected crash: simulated mid-sync failure") + } + c.mu.Unlock() + lookup, cl, err := c.requestLookup(in.GetAnnotations()) + if err != nil { + return nil, err + } + var resp *v2.GrantsServiceListGrantsResponse + switch rid.GetResourceType() { + case groupResourceType.GetId(): + resp, err = c.groupGrantsPage(ctx, lookup, rid.GetResource(), in.GetPageToken()) + case equivOrgRT.GetId(): + resp, err = c.orgGrantsPage(ctx, lookup, rid.GetResource()) + case equivTeamRT.GetId(): + // Same marker enforcement as listEntitlementsInner: dropping the + // TypeScopedGrants request marker must fail the harness. + reqAnnos := annotations.Annotations(in.GetAnnotations()) + if !reqAnnos.Contains(&v2.TypeScopedGrants{}) { + return nil, fmt.Errorf("team grants call missing TypeScopedGrants request marker (page token %q)", in.GetPageToken()) + } + resp, err = c.teamGrantsPage(ctx, lookup, in.GetPageToken()) + default: + return v2.GrantsServiceListGrantsResponse_builder{}.Build(), nil + } + if isDeferred(cl, err) { + return v2.GrantsServiceListGrantsResponse_builder{ + Annotations: c.askAnnos(cl), + }.Build(), nil + } + return resp, err +} + +// teamEntitlementsPage mirrors teamGrantsPage for type-scoped +// entitlements: planning EnqueuePageTokens, then one fresh-or-304 scope per +// chunk of two teams. Entitlement rows are stable (one "member" per +// team); version stays 0 so warm rounds are pure 304s after the cold +// fetch — exercising type-scoped entitlement replay in the differential +// chain without coupling to membership churn. +func (c *equivConnector) teamEntitlementsPage(ctx context.Context, lookup sourcecache.Lookup, pageToken string) (*v2.EntitlementsServiceListEntitlementsResponse, error) { + if pageToken == "" { + c.mu.Lock() + chunks := len(c.model.teamChunkVersions) + c.mu.Unlock() + c.count(func(ct *equivCounters) { ct.plannerPages++ }) + tokens := make([]string, 0, chunks) + for i := 0; i < chunks; i++ { + tokens = append(tokens, fmt.Sprintf("ent-chunk:%d", i)) + } + return v2.EntitlementsServiceListEntitlementsResponse_builder{ + Annotations: annotations.New(v2.EnqueuePageTokens_builder{PageTokens: tokens}.Build()), + }.Build(), nil + } + var chunk int + if _, err := fmt.Sscanf(pageToken, "ent-chunk:%d", &chunk); err != nil { + return nil, fmt.Errorf("teamEntitlementsPage: bad page token %q: %w", pageToken, err) + } + c.mu.Lock() + teamIDs := c.model.teamsInChunk(chunk) + ghost := c.model.ghostTeamEnt && chunk == 0 + c.mu.Unlock() + + scope := fmt.Sprintf("ents:team-chunk:%d", chunk) + verdict, err := c.revalidate(ctx, lookup, sourcecache.RowKindEntitlements, sourcecache.HashScope(scope), 0) + if err != nil { + return nil, err + } + if verdict == equivCurrent { + c.count(func(ct *equivCounters) { ct.chunkPages304++ }) + return v2.EntitlementsServiceListEntitlementsResponse_builder{ + Annotations: equivReplayAnnos(scope, 0), + }.Build(), nil + } + c.count(func(ct *equivCounters) { ct.chunkPagesFresh++ }) + rows := make([]*v2.Entitlement, 0, len(teamIDs)) + for _, tid := range teamIDs { + rows = append(rows, equivMemberEnt(equivTeamResource(tid), false)) + } + if ghost { + // The I7-class dangling shape: an entitlement for the shadow + // host2 resource, which is only listed once the shadow type is + // enabled. The scope's validator never rotates. + rows = append(rows, equivShadowHost2Ent()) + } + return v2.EntitlementsServiceListEntitlementsResponse_builder{ + List: rows, + Annotations: equivScopeAnnos(scope, 0), + }.Build(), nil +} + +// teamGrantsPage is the type-scoped planner shape: the planning call +// (empty page token) emits EnqueuePageTokens — one token per chunk of two +// teams — and each spawned cursor serves its chunk as a fresh-or-304 +// scope. +func (c *equivConnector) teamGrantsPage(ctx context.Context, lookup sourcecache.Lookup, pageToken string) (*v2.GrantsServiceListGrantsResponse, error) { + if pageToken == "" { + c.mu.Lock() + chunks := len(c.model.teamChunkVersions) + c.mu.Unlock() + c.count(func(ct *equivCounters) { ct.plannerPages++ }) + tokens := make([]string, 0, chunks) + for i := 0; i < chunks; i++ { + tokens = append(tokens, fmt.Sprintf("chunk:%d", i)) + } + return v2.GrantsServiceListGrantsResponse_builder{ + Annotations: annotations.New(v2.EnqueuePageTokens_builder{PageTokens: tokens}.Build()), + }.Build(), nil + } + var chunk int + if _, err := fmt.Sscanf(pageToken, "chunk:%d", &chunk); err != nil { + return nil, fmt.Errorf("teamGrantsPage: bad page token %q: %w", pageToken, err) + } + c.mu.Lock() + m := c.model + version := m.teamChunkVersions[chunk] + teamIDs := m.teamsInChunk(chunk) + ghost := m.ghostTeamEnt && chunk == 0 + membersByTeam := make(map[string][]string, len(teamIDs)) + for _, tid := range teamIDs { + membersByTeam[tid] = sortedKeys(m.teams[tid]) + } + c.mu.Unlock() + + scope := fmt.Sprintf("grants:team-chunk:%d", chunk) + verdict, err := c.revalidate(ctx, lookup, sourcecache.RowKindGrants, sourcecache.HashScope(scope), version) + if err != nil { + return nil, err + } + if verdict == equivCurrent { + c.count(func(ct *equivCounters) { ct.chunkPages304++ }) + return v2.GrantsServiceListGrantsResponse_builder{ + Annotations: equivReplayAnnos(scope, version), + }.Build(), nil + } + c.count(func(ct *equivCounters) { ct.chunkPagesFresh++ }) + var rows []*v2.Grant + for _, tid := range teamIDs { + for _, uid := range membersByTeam[tid] { + rows = append(rows, equivTeamGrant(tid, uid)) + } + } + if ghost { + // The shadow host2 entitlement's grant: dropped as I7's I8 + // cascade while the shadow type is disabled, kept once enabled. + rows = append(rows, equivShadowHost2Grant("u0")) + } + return v2.GrantsServiceListGrantsResponse_builder{ + List: rows, + Annotations: equivScopeAnnos(scope, version), + }.Build(), nil +} + +// groupGrantsPage: delta-scope protocol for group membership (and the +// external-match group, which is fresh-or-304 only — the model never +// mixes tombstones into it). The multiPageGroup serves its overlay +// rounds across TWO pages: page 1 carries the replay annotation +// (overlay, no validator) plus the first add; page 2 carries the scope +// annotation with the remaining adds, the tombstones, and the NEW +// validator — the page-spanning delta contract. +func (c *equivConnector) groupGrantsPage(ctx context.Context, lookup sourcecache.Lookup, gid string, pageToken string) (*v2.GrantsServiceListGrantsResponse, error) { + c.mu.Lock() + m := c.model + g := m.groups[gid] + if g == nil { + c.mu.Unlock() + return v2.GrantsServiceListGrantsResponse_builder{}.Build(), nil + } + isExt := gid == m.extGroupID + isExpandableParent := gid == m.expandableParent + isMultiPage := gid == m.multiPageGroup + expandableChild := m.expandableChild + version := g.version + ghostAdmin := g.ghostAdminGrant + members := sortedKeys(g.members) + deltaAdds := append([]string{}, g.deltaAdds...) + deltaRemoves := append([]string{}, g.deltaRemoves...) + canonical := g.canonicalTombstones + extMatchIDs := append([]string{}, m.extMatchIDs...) + if isExt { + version = m.extVersion + } + c.mu.Unlock() + + scope := "grants:" + gid + fullRows := func() []*v2.Grant { + var out []*v2.Grant + if isExt { + groupRes := equivGroupResource(gid) + for _, extID := range extMatchIDs { + out = append(out, gt.NewGrant(groupRes, "member", + v2.ResourceId_builder{ResourceType: userResourceType.GetId(), Resource: "placeholder-" + extID}.Build(), + gt.WithAnnotation(v2.ExternalResourceMatchID_builder{Id: extID}.Build()), + )) + } + out = append(out, gt.NewGrant(groupRes, "member", + v2.ResourceId_builder{ResourceType: userResourceType.GetId(), Resource: "placeholder-all"}.Build(), + gt.WithAnnotation(v2.ExternalResourceMatchAll_builder{ResourceType: v2.ResourceType_TRAIT_USER}.Build()), + )) + return out + } + for _, uid := range members { + out = append(out, equivMemberGrant(gid, uid)) + } + if ghostAdmin { + // Emitted every fresh fetch; an I8-class dangling (expected + // disabled-type gap) until the shadow type is enabled and + // its host's admin entitlement row exists. + out = append(out, equivShadowAdminGrant("u1")) + } + if isExpandableParent { + childRes := equivGroupResource(expandableChild) + childEnt := equivMemberEnt(childRes, false) + out = append(out, gt.NewGrant(equivGroupResource(gid), "member", childRes, + gt.WithAnnotation(v2.GrantExpandable_builder{EntitlementIds: []string{childEnt.GetId()}}.Build()), + )) + } + return out + } + + verdict, err := c.revalidate(ctx, lookup, sourcecache.RowKindGrants, sourcecache.HashScope(scope), version) + if err != nil { + return nil, err + } + switch verdict { + case equivCurrent: + c.count(func(ct *equivCounters) { ct.pages304++ }) + return v2.GrantsServiceListGrantsResponse_builder{ + Annotations: equivReplayAnnos(scope, version), + }.Build(), nil + case equivOneBehind: + if isExt || (len(deltaAdds) == 0 && len(deltaRemoves) == 0) { + // No delta journal for this transition: full refetch. + break + } + adds := make([]*v2.Grant, 0, len(deltaAdds)) + for _, uid := range deltaAdds { + adds = append(adds, equivMemberGrant(gid, uid)) + } + if isMultiPage { + if pageToken == "" { + // Overlay page 1: replay annotation (overlay, NO + // validator yet) plus the first add. + c.count(func(ct *equivCounters) { ct.pagesOverlayInterim++ }) + firstAdds := adds + if len(firstAdds) > 1 { + firstAdds = adds[:1] + } + return v2.GrantsServiceListGrantsResponse_builder{ + List: firstAdds, + Annotations: annotations.New(v2.SourceCacheReplay_builder{ + ScopeKey: sourcecache.HashScope(scope), + Overlay: true, + }.Build()), + NextPageToken: "ov:2", + }.Build(), nil + } + // Overlay final page: remaining adds + tombstones + the NEW + // validator, all on the scope annotation. + c.count(func(ct *equivCounters) { ct.pagesOverlay++ }) + var restAdds []*v2.Grant + if len(adds) > 1 { + restAdds = adds[1:] + } + finalAnno := v2.SourceCacheRecord_builder{ + ScopeKey: sourcecache.HashScope(scope), + CacheValidator: equivEtag(version), + }.Build() + if canonical { + ids := make([]string, 0, len(deltaRemoves)) + for _, uid := range deltaRemoves { + ids = append(ids, equivMemberGrant(gid, uid).GetId()) + } + finalAnno.SetDeletedIds(ids) + c.count(func(ct *equivCounters) { ct.tombsCanonical += len(ids) }) + } else { + finalAnno.SetDeletedPrincipalIds(deltaRemoves) + c.count(func(ct *equivCounters) { ct.tombsPrincipal += len(deltaRemoves) }) + } + return v2.GrantsServiceListGrantsResponse_builder{ + List: restAdds, + Annotations: annotations.New(finalAnno), + }.Build(), nil + } + c.count(func(ct *equivCounters) { ct.pagesOverlay++ }) + replay := v2.SourceCacheReplay_builder{ + ScopeKey: sourcecache.HashScope(scope), + CacheValidator: equivEtag(version), + Overlay: true, + }.Build() + scopeAnno := v2.SourceCacheRecord_builder{ + ScopeKey: sourcecache.HashScope(scope), + }.Build() + if canonical { + ids := make([]string, 0, len(deltaRemoves)) + for _, uid := range deltaRemoves { + ids = append(ids, equivMemberGrant(gid, uid).GetId()) + } + replay.SetDeletedIds(ids) + c.count(func(ct *equivCounters) { ct.tombsCanonical += len(ids) }) + } else { + scopeAnno.SetDeletedPrincipalIds(deltaRemoves) + c.count(func(ct *equivCounters) { ct.tombsPrincipal += len(deltaRemoves) }) + } + return v2.GrantsServiceListGrantsResponse_builder{ + List: adds, + Annotations: annotations.New(replay, scopeAnno), + }.Build(), nil + default: + } + c.count(func(ct *equivCounters) { + if verdict == equivFresh { + ct.pagesFresh++ + } else { + ct.pagesColdStale++ + } + }) + return v2.GrantsServiceListGrantsResponse_builder{ + List: fullRows(), + Annotations: equivScopeAnnos(scope, version), + }.Build(), nil +} + +// orgGrantsPage: the InsertResourceGrants scope (fresh-or-304). +func (c *equivConnector) orgGrantsPage(ctx context.Context, lookup sourcecache.Lookup, orgID string) (*v2.GrantsServiceListGrantsResponse, error) { + c.mu.Lock() + org := c.model.orgs[orgID] + if org == nil { + c.mu.Unlock() + return v2.GrantsServiceListGrantsResponse_builder{}.Build(), nil + } + version := org.repoVersion + repoGrants := make(map[string]string, len(org.repoGrants)) + for k, v := range org.repoGrants { + repoGrants[k] = v + } + c.mu.Unlock() + + scope := "grants:org:" + orgID + verdict, err := c.revalidate(ctx, lookup, sourcecache.RowKindGrants, sourcecache.HashScope(scope), version) + if err != nil { + return nil, err + } + if verdict == equivCurrent { + c.count(func(ct *equivCounters) { ct.pages304++ }) + return v2.GrantsServiceListGrantsResponse_builder{ + Annotations: equivReplayAnnos(scope, version), + }.Build(), nil + } + c.count(func(ct *equivCounters) { + if verdict == equivFresh { + ct.pagesFresh++ + } else { + ct.pagesColdStale++ + } + }) + rows := make([]*v2.Grant, 0, len(repoGrants)) + for _, repoID := range sortedKeys(repoGrants) { + rows = append(rows, gt.NewGrant(equivRepoResource(repoID), "admin", + v2.ResourceId_builder{ResourceType: userResourceType.GetId(), Resource: repoGrants[repoID]}.Build())) + } + annos := equivScopeAnnos(scope, version) + annos.Update(&v2.InsertResourceGrants{}) + return v2.GrantsServiceListGrantsResponse_builder{ + List: rows, + Annotations: annos, + }.Build(), nil +} + +// --------------------------------------------------------------------- +// External c1z +// --------------------------------------------------------------------- + +// buildExternalC1z materializes the model's external users into a fresh +// c1z (rebuilt whenever extUsers changes; both warm and cold syncs of a +// round read the same file). +func buildExternalC1z(ctx context.Context, t *testing.T, m *equivModel, path, tmpDir string) { + t.Helper() + mc := newMockConnector() + mc.rtDB = append(mc.rtDB, userResourceType, groupResourceType) + for _, extID := range sortedKeys(m.extUsers) { + _, err := mc.AddUserProfile(ctx, extID, map[string]any{}) + require.NoError(t, err) + } + syncer, err := NewSyncer(ctx, mc, WithC1ZPath(path), WithTmpDir(tmpDir)) + require.NoError(t, err) + require.NoError(t, syncer.Sync(ctx)) + require.NoError(t, syncer.Close(ctx)) +} + +// --------------------------------------------------------------------- +// Snapshot + comparison +// --------------------------------------------------------------------- + +type c1zSnapshot struct { + resourceTypes map[string]*v2.ResourceType + resources map[string]*v2.Resource // "rt/id" + entitlements map[string]*v2.Entitlement + grants map[string]*v2.Grant + // byParent: parent "rt/id" -> child resource keys via the by_parent + // index read path (cross-index consistency). + byParent map[string][]string + // grantsByEntResource: entitlement-resource "rt/id" -> grant ids via + // the by_entitlement_resource index read path. + grantsByEntResource map[string][]string + // manifest ("row_kind\x00scope_key" -> typed entry: validator + + // invalidation verdict) and scopeIndexCounts (row kind -> scope key + // -> index entries): the source-cache state invisible to v2 reads + // but load-bearing for the NEXT sync's replay. + manifest map[string]pebbleengine.SourceCacheManifestEntry + scopeIndexCounts map[string]map[string]int + // syncRuns is the number of sync runs in the file — the + // crash/resume round asserts exactly one (resume, not restart). + syncRuns int +} + +func resourceKey(id *v2.ResourceId) string { return id.GetResourceType() + "/" + id.GetResource() } + +func snapshotC1z(ctx context.Context, t *testing.T, path string) *c1zSnapshot { + t.Helper() + reopen, err := dotc1z.NewStore(ctx, path, + dotc1z.WithEngine(c1zstore.EnginePebble), + dotc1z.WithReadOnly(true), + ) + require.NoError(t, err) + defer func() { _ = reopen.Close(ctx) }() + + syncRun, err := reopen.SyncMeta().LatestFullSync(ctx) + require.NoError(t, err) + require.NotNil(t, syncRun) + require.NoError(t, reopen.SetCurrentSync(ctx, syncRun.ID)) + + snap := &c1zSnapshot{ + resourceTypes: map[string]*v2.ResourceType{}, + resources: map[string]*v2.Resource{}, + entitlements: map[string]*v2.Entitlement{}, + grants: map[string]*v2.Grant{}, + byParent: map[string][]string{}, + grantsByEntResource: map[string][]string{}, + } + + // Raw row counts per listing: the deduping maps below would silently + // hide duplicate rows (a pagination double-yield or a replay writing + // two rows under one identity), so every listing asserts raw == map. + pageToken := "" + rawRows := 0 + for { + resp, err := reopen.ListResourceTypes(ctx, v2.ResourceTypesServiceListResourceTypesRequest_builder{PageToken: pageToken}.Build()) + require.NoError(t, err) + for _, rt := range resp.GetList() { + snap.resourceTypes[rt.GetId()] = rt + rawRows++ + } + if pageToken = resp.GetNextPageToken(); pageToken == "" { + break + } + } + require.Equal(t, len(snap.resourceTypes), rawRows, "duplicate resource-type rows in %s", path) + rawRows = 0 + for { + resp, err := reopen.ListResources(ctx, v2.ResourcesServiceListResourcesRequest_builder{PageToken: pageToken}.Build()) + require.NoError(t, err) + for _, r := range resp.GetList() { + snap.resources[resourceKey(r.GetId())] = r + rawRows++ + } + if pageToken = resp.GetNextPageToken(); pageToken == "" { + break + } + } + require.Equal(t, len(snap.resources), rawRows, "duplicate resource rows in %s", path) + rawRows = 0 + for { + resp, err := reopen.ListEntitlements(ctx, v2.EntitlementsServiceListEntitlementsRequest_builder{PageToken: pageToken}.Build()) + require.NoError(t, err) + for _, e := range resp.GetList() { + snap.entitlements[e.GetId()] = e + rawRows++ + } + if pageToken = resp.GetNextPageToken(); pageToken == "" { + break + } + } + require.Equal(t, len(snap.entitlements), rawRows, "duplicate entitlement rows in %s", path) + rawRows = 0 + for { + resp, err := reopen.ListGrants(ctx, v2.GrantsServiceListGrantsRequest_builder{PageToken: pageToken}.Build()) + require.NoError(t, err) + for _, g := range resp.GetList() { + snap.grants[g.GetId()] = g + rawRows++ + } + if pageToken = resp.GetNextPageToken(); pageToken == "" { + break + } + } + require.Equal(t, len(snap.grants), rawRows, "duplicate grant rows in %s", path) + + // Source-cache state below the v2 read surface, plus the sync-run + // count (crash/resume must resume, never restart). + engineHolder, ok := any(reopen).(interface{ PebbleEngine() *pebbleengine.Engine }) + require.True(t, ok, "store must expose its pebble engine for source-cache inspection") + snap.manifest, err = engineHolder.PebbleEngine().SourceCacheManifestSnapshot(ctx) + require.NoError(t, err) + snap.scopeIndexCounts, err = engineHolder.PebbleEngine().SourceScopeIndexSnapshot(ctx) + require.NoError(t, err) + runLister, ok := any(reopen).(interface { + ListSyncRuns(ctx context.Context, pageToken string, pageSize uint32) ([]*c1zstore.SyncRun, string, error) + }) + require.True(t, ok, "store must list sync runs") + runs, _, err := runLister.ListSyncRuns(ctx, "", 100) + require.NoError(t, err) + snap.syncRuns = len(runs) + + // Secondary read paths: by-parent resource listings for every parent + // that has children per the primary rows, and by-entitlement-resource + // grant listings for every entitlement resource seen in grants. + parents := map[string]*v2.ResourceId{} + for _, r := range snap.resources { + if p := r.GetParentResourceId(); p.GetResource() != "" { + parents[resourceKey(p)] = p + } + } + for pk, pid := range parents { + for { + resp, err := reopen.ListResources(ctx, v2.ResourcesServiceListResourcesRequest_builder{ + ResourceTypeId: "", + ParentResourceId: pid, + PageToken: pageToken, + }.Build()) + require.NoError(t, err) + for _, r := range resp.GetList() { + snap.byParent[pk] = append(snap.byParent[pk], resourceKey(r.GetId())) + } + if pageToken = resp.GetNextPageToken(); pageToken == "" { + break + } + } + sort.Strings(snap.byParent[pk]) + } + entResources := map[string]*v2.Resource{} + for _, g := range snap.grants { + res := g.GetEntitlement().GetResource() + if res.GetId() != nil { + entResources[resourceKey(res.GetId())] = res + } + } + for rk, res := range entResources { + for { + resp, err := reopen.ListGrants(ctx, v2.GrantsServiceListGrantsRequest_builder{ + Resource: res, + PageToken: pageToken, + }.Build()) + require.NoError(t, err) + for _, g := range resp.GetList() { + snap.grantsByEntResource[rk] = append(snap.grantsByEntResource[rk], g.GetId()) + } + if pageToken = resp.GetNextPageToken(); pageToken == "" { + break + } + } + sort.Strings(snap.grantsByEntResource[rk]) + } + return snap +} + +// normalizeMessage returns a clone with annotations sorted so proto +// equality ignores annotation ordering (the only legitimate ordering +// difference between cold and warm outputs). +func normalizeMessage[T proto.Message](msg T, get func(T) []*anypb.Any, set func(T, []*anypb.Any)) T { + clone, _ := proto.Clone(msg).(T) + annos := get(clone) + if len(annos) > 1 { + sorted := append([]*anypb.Any{}, annos...) + sort.Slice(sorted, func(i, j int) bool { + if sorted[i].GetTypeUrl() != sorted[j].GetTypeUrl() { + return sorted[i].GetTypeUrl() < sorted[j].GetTypeUrl() + } + return string(sorted[i].GetValue()) < string(sorted[j].GetValue()) + }) + set(clone, sorted) + } + return clone +} + +func diffMaps[T proto.Message](t *testing.T, label string, cold, warm map[string]T, normalize func(T) T) { + t.Helper() + for _, k := range sortedKeys(cold) { + w, ok := warm[k] + if !ok { + t.Errorf("replay equivalence: %s %q present in COLD output but MISSING from warm output", label, k) + continue + } + nc, nw := normalize(cold[k]), normalize(w) + if !proto.Equal(nc, nw) { + t.Errorf("replay equivalence: %s %q differs\n--- cold ---\n%s\n--- warm ---\n%s", + label, k, prototext.Format(nc), prototext.Format(nw)) + } + } + for _, k := range sortedKeys(warm) { + if _, ok := cold[k]; !ok { + t.Errorf("replay equivalence: %s %q present in WARM output but missing from cold output (stale row survived?)", label, k) + } + } +} + +func requireSnapshotsEqual(t *testing.T, round int, cold, warm *c1zSnapshot) { + t.Helper() + t.Logf("round %d: comparing cold (%d res, %d ents, %d grants) vs warm (%d res, %d ents, %d grants)", + round, len(cold.resources), len(cold.entitlements), len(cold.grants), + len(warm.resources), len(warm.entitlements), len(warm.grants)) + + diffMaps(t, "resource type", cold.resourceTypes, warm.resourceTypes, + func(m *v2.ResourceType) *v2.ResourceType { + return normalizeMessage(m, (*v2.ResourceType).GetAnnotations, (*v2.ResourceType).SetAnnotations) + }) + diffMaps(t, "resource", cold.resources, warm.resources, + func(m *v2.Resource) *v2.Resource { + return normalizeMessage(m, (*v2.Resource).GetAnnotations, (*v2.Resource).SetAnnotations) + }) + diffMaps(t, "entitlement", cold.entitlements, warm.entitlements, + func(m *v2.Entitlement) *v2.Entitlement { + return normalizeMessage(m, (*v2.Entitlement).GetAnnotations, (*v2.Entitlement).SetAnnotations) + }) + diffMaps(t, "grant", cold.grants, warm.grants, + func(m *v2.Grant) *v2.Grant { + return normalizeMessage(m, (*v2.Grant).GetAnnotations, (*v2.Grant).SetAnnotations) + }) + + require.Equal(t, cold.byParent, warm.byParent, + "round %d: by-parent index read path diverged (replay index synthesis?)", round) + require.Equal(t, cold.grantsByEntResource, warm.grantsByEntResource, + "round %d: by-entitlement-resource index read path diverged (replay index synthesis?)", round) + + // Below the v2 read surface: both connectors emitted identical scope + // annotations and validators, so the manifests and scope-index shapes + // must match. Divergence here is invisible to every check above but + // poisons the NEXT sync's replay from the warm file. + require.Equal(t, cold.manifest, warm.manifest, + "round %d: source-cache manifest diverged (phantom or missing entries poison future replays)", round) + require.Equal(t, cold.scopeIndexCounts, warm.scopeIndexCounts, + "round %d: source-cache scope-index shape diverged (stamp divergence poisons future replays)", round) + + if t.Failed() { + t.Fatalf("round %d: warm sync output is not equivalent to a cold resync of the same upstream state", round) + } +} + +// assertNoStaleResources is the reverse-containment check: every resource +// row of a modeled type must correspond to CURRENT model state (or an +// external principal). The forward diff proves warm == cold; this proves +// neither side is carrying rows the upstream no longer has — the +// deletion-round oracle. +func assertNoStaleResources(t *testing.T, round int, m *equivModel, label string, snap *c1zSnapshot) { + t.Helper() + for key := range snap.resources { + rt, id, _ := strings.Cut(key, "/") + switch rt { + case equivOrgRT.GetId(): + require.Contains(t, m.orgs, id, "round %d: %s output holds deleted org %s", round, label, id) + case equivProjectRT.GetId(): + found := false + for _, org := range m.orgs { + if _, ok := org.projects[id]; ok { + found = true + } + } + require.True(t, found, "round %d: %s output holds deleted project %s", round, label, id) + case equivRepoRT.GetId(): + found := false + for _, org := range m.orgs { + if _, ok := org.repoGrants[id]; ok { + found = true + } + } + require.True(t, found, "round %d: %s output holds deleted repo %s", round, label, id) + case groupResourceType.GetId(): + require.Contains(t, m.groups, id, "round %d: %s output holds deleted group %s", round, label, id) + case equivTeamRT.GetId(): + require.Contains(t, m.teams, id, "round %d: %s output holds unknown team %s", round, label, id) + case userResourceType.GetId(): + require.True(t, m.users[id] || m.extUsers[id], + "round %d: %s output holds deleted user %s", round, label, id) + } + } +} + +// expectedResourceKeys derives the COMPLETE resource set ("rt/id" keys) +// the model implies: users (local + matched external principals), groups, +// orgs, per-org child projects, InsertResourceGrants repos, and teams. +// Exact set equality against the cold output removes the differential +// test's structural blind spot for the resources phase (see +// expectedGrantIDs). +func expectedResourceKeys(m *equivModel) []string { + set := map[string]struct{}{} + for uid := range m.users { + set[userResourceType.GetId()+"/"+uid] = struct{}{} + } + // Every external user is matched by the MatchAll(USER) grant, so + // every one is inserted as a principal resource. + for extID := range m.extUsers { + set[userResourceType.GetId()+"/"+extID] = struct{}{} + } + for gid := range m.groups { + set[groupResourceType.GetId()+"/"+gid] = struct{}{} + } + for orgID, org := range m.orgs { + set[equivOrgRT.GetId()+"/"+orgID] = struct{}{} + for projID := range org.projects { + set[equivProjectRT.GetId()+"/"+projID] = struct{}{} + } + for repoID := range org.repoGrants { + set[equivRepoRT.GetId()+"/"+repoID] = struct{}{} + } + } + for tid := range m.teams { + set[equivTeamRT.GetId()+"/"+tid] = struct{}{} + } + return sortedKeys(set) +} + +// expectedEntitlementIDs derives the COMPLETE entitlement set the model +// implies: one member entitlement per group (including the external-match +// group — its entitlement anchors the transformed grants) and one per +// type-scoped team. Users, projects, and repos are skip-annotated or +// never enumerated; orgs serve no entitlements; expansion and external +// processing create no entitlement rows. +func expectedEntitlementIDs(m *equivModel) []string { + set := map[string]struct{}{} + for gid, g := range m.groups { + set[equivMemberEnt(equivGroupResource(gid), g.exclusionDefault).GetId()] = struct{}{} + } + for tid := range m.teams { + set[equivMemberEnt(equivTeamResource(tid), false).GetId()] = struct{}{} + } + return sortedKeys(set) +} + +// expectedGrantIDs derives the COMPLETE grant set the model implies: +// group memberships, the expandable membership plus its expansion-derived +// grants, match-transformed external grants, repo (InsertResourceGrants) +// grants, and type-scoped team memberships. Comparing the cold output +// against this — exact set equality, not spot checks — removes the +// differential test's structural blind spot: cold and warm share the +// syncer, so a bug corrupting both identically would otherwise pass. +func expectedGrantIDs(m *equivModel) []string { + set := map[string]struct{}{} + for gid, g := range m.groups { + if gid == m.extGroupID { + continue + } + for uid := range g.members { + set[equivMemberGrant(gid, uid).GetId()] = struct{}{} + } + } + // The expandable group-to-group membership and its derived grants. + parentRes := equivGroupResource(m.expandableParent) + parentEnt := equivMemberEnt(parentRes, false) + set[gt.NewGrantID(equivGroupResource(m.expandableChild), parentEnt)] = struct{}{} + for uid := range m.groups[m.expandableChild].members { + set[gt.NewGrantID( + v2.ResourceId_builder{ResourceType: userResourceType.GetId(), Resource: uid}.Build(), + parentEnt)] = struct{}{} + } + // Match-transformed grants: MatchAll covers every external user (the + // MatchID targets are a subset with identical resulting ids). + extEnt := equivMemberEnt(equivGroupResource(m.extGroupID), false) + for extID := range m.extUsers { + set[gt.NewGrantID( + v2.ResourceId_builder{ResourceType: userResourceType.GetId(), Resource: extID}.Build(), + extEnt)] = struct{}{} + } + for _, org := range m.orgs { + for repoID, uid := range org.repoGrants { + set[gt.NewGrant(equivRepoResource(repoID), "admin", + v2.ResourceId_builder{ResourceType: userResourceType.GetId(), Resource: uid}.Build()).GetId()] = struct{}{} + } + } + for tid, members := range m.teams { + for uid := range members { + set[equivTeamGrant(tid, uid).GetId()] = struct{}{} + } + } + return sortedKeys(set) +} + +// --------------------------------------------------------------------- +// Coverage guards (anti-vacuity) +// --------------------------------------------------------------------- + +// assertColdArtifacts fails unless the cold output actually contains the +// artifacts of every feature the harness claims to cover — a comparison +// against an output that never exercised a feature proves nothing about +// that feature. +func assertColdArtifacts(t *testing.T, round int, m *equivModel, snap *c1zSnapshot) { + t.Helper() + for _, orgID := range sortedKeys(m.orgs) { + for projID := range m.orgs[orgID].projects { + require.Contains(t, snap.resources, equivProjectRT.GetId()+"/"+projID, + "round %d: coverage: child project %s must be present", round, projID) + } + for repoID := range m.orgs[orgID].repoGrants { + require.Contains(t, snap.resources, equivRepoRT.GetId()+"/"+repoID, + "round %d: coverage: InsertResourceGrants repo %s must be present", round, repoID) + } + } + // Expansion-derived grants: every member of the expandable child must + // hold a derived grant on the parent. + parentRes := equivGroupResource(m.expandableParent) + for uid := range m.groups[m.expandableChild].members { + derived := gt.NewGrantID( + v2.ResourceId_builder{ResourceType: userResourceType.GetId(), Resource: uid}.Build(), + equivMemberEnt(parentRes, false)) + require.Contains(t, snap.grants, derived, + "round %d: coverage: expansion-derived grant for %s must be present", round, uid) + } + // Match-transformed grants: MatchAll must cover every external user; + // MatchID entries must cover exactly the still-existing targets. + extGroupRes := equivGroupResource(m.extGroupID) + extEnt := equivMemberEnt(extGroupRes, false) + for extID := range m.extUsers { + transformed := gt.NewGrantID( + v2.ResourceId_builder{ResourceType: userResourceType.GetId(), Resource: extID}.Build(), extEnt) + require.Contains(t, snap.grants, transformed, + "round %d: coverage: MatchAll-transformed grant for %s must be present", round, extID) + } + for _, extID := range m.extMatchIDs { + transformed := gt.NewGrantID( + v2.ResourceId_builder{ResourceType: userResourceType.GetId(), Resource: extID}.Build(), extEnt) + if m.extUsers[extID] { + require.Contains(t, snap.grants, transformed, + "round %d: coverage: MatchID-transformed grant for %s must be present", round, extID) + } else { + require.NotContains(t, snap.grants, transformed, + "round %d: coverage: MatchID grant for REMOVED external user %s must be absent", round, extID) + } + } + // No placeholder principals may survive the transformation. + for id := range snap.grants { + require.NotContains(t, id, "placeholder-", + "round %d: coverage: untransformed placeholder grant leaked into output", round) + } + // The independent oracle: the cold output's resource, entitlement, + // and grant sets must EXACTLY equal the model-derived expectations — + // the checks that don't trust the syncer code both outputs share. + require.Equal(t, expectedResourceKeys(m), sortedKeys(snap.resources), + "round %d: cold output's resource set must equal the model-derived expectation", round) + require.Equal(t, expectedEntitlementIDs(m), sortedKeys(snap.entitlements), + "round %d: cold output's entitlement set must equal the model-derived expectation", round) + require.Equal(t, expectedGrantIDs(m), sortedKeys(snap.grants), + "round %d: cold output's grant set must equal the model-derived expectation", round) +} + +// --------------------------------------------------------------------- +// The harness +// --------------------------------------------------------------------- + +const equivRounds = 4 + +// equivCrashRound is the round whose WARM sync is crashed mid-grants-phase +// (injected connector failure) and resumed into the same file. +const equivCrashRound = 2 + +// runEquivSync runs one sync into path (creating or RESUMING the file's +// unfinished sync), returning Sync's error. Path-based store creation — +// not WithConnectorStore — so a crashed sync's file can be reopened and +// resumed exactly like production resume. +// +// Clean scenarios run fail-fast: check-only ingestion invariants +// hard-fail, so a violation names itself before the differential +// comparison would catch its downstream divergence. The dirty family +// (source_cache_equivalence_dirty_test.go) uses runEquivSyncMode with +// failFast=false — its PROPERTY is default-warm ≡ default-cold, which +// requires the drop arms to actually execute. +func runEquivSync(ctx context.Context, t *testing.T, cc types.ConnectorClient, path, prevPath, tmpDir string, extraOpts ...SyncOpt) error { + t.Helper() + return runEquivSyncMode(ctx, t, cc, path, prevPath, tmpDir, true, extraOpts...) +} + +func runEquivSyncMode(ctx context.Context, t *testing.T, cc types.ConnectorClient, path, prevPath, tmpDir string, failFast bool, extraOpts ...SyncOpt) error { + t.Helper() + opts := []SyncOpt{ + WithC1ZPath(path), + WithStorageEngine(c1zstore.EnginePebble), + WithTmpDir(tmpDir), + } + if failFast { + opts = append(opts, WithFailFastInvariants()) + } + if prevPath != "" { + opts = append(opts, WithPreviousSyncC1ZPath(prevPath)) + } + opts = append(opts, extraOpts...) + syncer, err := NewSyncer(ctx, cc, opts...) + require.NoError(t, err) + syncErr := syncer.Sync(ctx) + if syncErr != nil { + // Close under a canceled context so the unfinished sync's state + // survives for resumption (same pattern as the resume tests). + cctx, cancel := context.WithCancel(ctx) + cancel() + _ = syncer.Close(cctx) + return syncErr + } + require.NoError(t, syncer.Close(ctx)) + return nil +} + +func TestSourceCache_ReplayEquivalence(t *testing.T) { + scenarios := []struct { + workers int + seed int64 + continuation bool + }{ + {workers: 0, seed: 1}, + {workers: 0, seed: 2}, + {workers: 4, seed: 3}, + // The single-shot (Lambda) topology: lookups ride the ask/answer + // continuation instead of a direct SetSourceCache install. + {workers: 0, seed: 4, continuation: true}, + {workers: 4, seed: 5, continuation: true}, + } + for _, sc := range scenarios { + name := fmt.Sprintf("workers=%d/seed=%d", sc.workers, sc.seed) + if sc.continuation { + name += "/continuation" + } + t.Run(name, func(t *testing.T) { + runReplayEquivalenceScenario(t, sc.workers, sc.seed, sc.continuation) + }) + } +} + +// invariantWarningRecorder captures ingestion-invariant warnings emitted +// during a scenario. Warnings are the invariants' DEFAULT-mode verdicts, +// and nothing in a test suite fails on a log line — which is exactly how +// a false-positive warning (the I6-on-compaction class) can fire on every +// run of a green suite without anyone noticing. The harness therefore +// asserts warning-SILENCE for its clean scenarios: an invariant warning +// here is either an invariant false positive or an undetected data +// problem in the harness fixtures, and both must be named. +type invariantWarningRecorder struct { + mu stdsync.Mutex + msgs []string +} + +func (r *invariantWarningRecorder) record(msg string) { + r.mu.Lock() + defer r.mu.Unlock() + r.msgs = append(r.msgs, msg) +} + +func (r *invariantWarningRecorder) captured() []string { + r.mu.Lock() + defer r.mu.Unlock() + return append([]string(nil), r.msgs...) +} + +// invariantWarningCore is a zapcore.Core that records Warn+ entries whose +// message names an ingestion invariant. Tee'd with the base core so +// human-readable output is unchanged. +type invariantWarningCore struct { + zapcore.LevelEnabler + rec *invariantWarningRecorder +} + +func (c invariantWarningCore) With([]zapcore.Field) zapcore.Core { return c } + +func (c invariantWarningCore) Check(e zapcore.Entry, ce *zapcore.CheckedEntry) *zapcore.CheckedEntry { + if c.Enabled(e.Level) { + return ce.AddCore(e, c) + } + return ce +} + +func (c invariantWarningCore) Write(e zapcore.Entry, _ []zapcore.Field) error { + if strings.Contains(e.Message, "ingest invariant") { + c.rec.record(e.Level.String() + ": " + e.Message) + } + return nil +} + +func (c invariantWarningCore) Sync() error { return nil } + +func withInvariantWarningOracle(ctx context.Context) (context.Context, *invariantWarningRecorder) { + rec := &invariantWarningRecorder{} + base := ctxzap.Extract(ctx) + logger := zap.New(zapcore.NewTee(base.Core(), invariantWarningCore{LevelEnabler: zapcore.WarnLevel, rec: rec})) + return ctxzap.ToContext(ctx, logger), rec +} + +func runReplayEquivalenceScenario(t *testing.T, workers int, seed int64, continuation bool) { + ctx, err := logging.Init(t.Context()) + require.NoError(t, err) + ctx, invariantWarnings := withInvariantWarningOracle(ctx) + tmpDir := t.TempDir() + rng := rand.New(rand.NewSource(seed)) //nolint:gosec // deterministic test randomness + t.Logf("replay-equivalence scenario: workers=%d seed=%d rounds=%d continuation=%v", workers, seed, equivRounds, continuation) + + model := newEquivModel() + warmConn := newEquivConnector(model, false) + coldConn := newEquivConnector(model, true) + warmConn.continuation = continuation + coldConn.continuation = continuation + var warmClient types.ConnectorClient = warmConn + var coldClient types.ConnectorClient = coldConn + if continuation { + warmClient = equivContinuationClient{warmConn} + coldClient = equivContinuationClient{coldConn} + } + + extPath := filepath.Join(tmpDir, "external-r0.c1z") + buildExternalC1z(ctx, t, model, extPath, tmpDir) + model.extDirty = false + + syncOpts := func() []SyncOpt { + opts := []SyncOpt{WithExternalResourceC1ZPath(extPath)} + if workers > 0 { + opts = append(opts, WithWorkerCount(workers)) + } + return opts + } + + prevWarm := "" + for round := 0; round <= equivRounds; round++ { + if round > 0 { + model.mutate(round, rng) + if model.extDirty { + extPath = filepath.Join(tmpDir, fmt.Sprintf("external-r%d.c1z", round)) + buildExternalC1z(ctx, t, model, extPath, tmpDir) + } + } + warmPath := filepath.Join(tmpDir, fmt.Sprintf("warm-r%d.c1z", round)) + coldPath := filepath.Join(tmpDir, fmt.Sprintf("cold-r%d.c1z", round)) + + before := warmConn.snapshotCounters() + if round == equivCrashRound { + // Crash the warm sync mid-grants-phase (after other scopes + // replayed), then resume the SAME file. The resumed output — + // pages re-run over partially replayed state — is what gets + // compared. + warmConn.injectGrantsFailure(model.expandableChild) + crashErr := runEquivSync(ctx, t, warmClient, warmPath, prevWarm, tmpDir, syncOpts()...) + require.Error(t, crashErr, "round %d: the injected failure must crash the warm sync", round) + require.Contains(t, crashErr.Error(), "injected crash") + require.Equal(t, 1, warmConn.snapshotCounters().injectedFailures-before.injectedFailures, + "round %d: exactly one injected failure expected", round) + } + require.NoError(t, runEquivSync(ctx, t, warmClient, warmPath, prevWarm, tmpDir, syncOpts()...)) + after := warmConn.snapshotCounters() + if round > 0 { + require.Positive(t, (after.pages304+after.pagesOverlay)-(before.pages304+before.pagesOverlay), + "round %d: VACUOUS RUN — the warm sync replayed nothing, so equivalence proves nothing", round) + } + require.NoError(t, runEquivSync(ctx, t, coldClient, coldPath, "", tmpDir, syncOpts()...)) + + coldSnap := snapshotC1z(ctx, t, coldPath) + warmSnap := snapshotC1z(ctx, t, warmPath) + if round == equivCrashRound { + require.Equal(t, 1, warmSnap.syncRuns, + "round %d: the crashed warm sync must RESUME (one sync run), not restart", round) + } + assertColdArtifacts(t, round, model, coldSnap) + assertNoStaleResources(t, round, model, "cold", coldSnap) + assertNoStaleResources(t, round, model, "warm", warmSnap) + requireSnapshotsEqual(t, round, coldSnap, warmSnap) + + prevWarm = warmPath + } + + // Scenario-level coverage: every replay behavior must have actually + // occurred, or this scenario silently tested less than it claims. + final := warmConn.snapshotCounters() + require.Positive(t, final.pages304, "coverage: no 304 replay round ever happened") + require.Positive(t, final.pagesOverlay, "coverage: no delta-overlay round ever happened") + require.Positive(t, final.pagesOverlayInterim, "coverage: no multi-page overlay round ever happened") + require.Positive(t, final.pagesColdStale, "coverage: no validator rotation (cold refetch) ever happened") + require.Positive(t, final.pagesFresh, "coverage: no cold-miss page ever happened") + require.Positive(t, final.pagesFreshInterim, "coverage: no multi-page fresh fetch ever happened") + require.Positive(t, final.tombsCanonical, "coverage: canonical-id tombstones never exercised") + require.Positive(t, final.tombsPrincipal, "coverage: principal-id tombstones never exercised") + require.Positive(t, final.childPageCalls, "coverage: child resource pages never fetched") + require.Positive(t, final.injectedFailures, "coverage: the crash/resume round never crashed") + require.Positive(t, final.plannerPages, "coverage: the type-scoped planner never ran") + require.Positive(t, final.chunkPages304, "coverage: no spawned chunk cursor ever replayed") + require.Positive(t, final.chunkPagesFresh, "coverage: no spawned chunk cursor ever fetched fresh") + if continuation { + require.Positive(t, final.askPages, "coverage: continuation topology never asked") + } else { + require.Zero(t, final.askPages, "harness bug: direct topology must never ask") + } + coldFinal := coldConn.snapshotCounters() + require.Zero(t, coldFinal.pages304+coldFinal.pagesOverlay, + "harness bug: the cold connector must never replay") + require.Zero(t, coldFinal.askPages, "harness bug: the cold connector must never ask (no offer without a warm lookup)") + + // Warning-silence oracle: clean scenarios must emit ZERO ingestion- + // invariant warnings across every warm/cold/resume leg. A warning is + // an invariant verdict that no test asserts on — the escape route the + // I6-on-compaction false positive took (warned on every run of a + // green suite). Either the invariant is falsely firing or the harness + // fixtures have an undetected data problem; both must fail here. + require.Empty(t, invariantWarnings.captured(), + "ingestion-invariant warnings during a clean scenario") +} diff --git a/pkg/sync/source_cache_halt_test.go b/pkg/sync/source_cache_halt_test.go new file mode 100644 index 000000000..fab3c4730 --- /dev/null +++ b/pkg/sync/source_cache_halt_test.go @@ -0,0 +1,115 @@ +package sync //nolint:revive,nolintlint // we can't change the package name for backwards compatibility + +// Halt-point sweep: crash the warm sync at EVERY ordering-sensitive seam +// of the scope lifecycle (via the syncer's testSourceCacheHaltHook), +// resume the same file, and require the resumed output to be semantically +// identical to a cold sync — the FoundationDB-style "halt at interesting +// points" check, scoped to the seams whose ordering bugs would produce +// silent bad data (e.g. a manifest entry committed ahead of its rows +// would poison the NEXT sync's replay while this sync reads clean). +// +// Seams swept (see beginSourceCachePage / finishSourceCachePage): +// - replay-copied: after the engine replay copy commits, before +// overlay rows / tombstones / manifest. +// - rows-committed: after the page's rows committed, before +// tombstones. +// - tombstones-applied: after delta tombstones applied, before the +// manifest entry write. +// - manifest-written: after the manifest entry committed — the +// double-execution seam (the resumed action +// re-runs a page whose scope already sealed). + +import ( + "fmt" + "math/rand" + "path/filepath" + stdsync "sync" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/conductorone/baton-sdk/pkg/logging" + "github.com/conductorone/baton-sdk/pkg/types" +) + +// haltOnce arms a one-shot failure at the named stage; every later hook +// call (including after resume, where the hook is still installed) passes. +type haltOnce struct { + mu stdsync.Mutex + stage string + fired bool + scope string +} + +func (h *haltOnce) hook(stage, scopeKey string) error { + h.mu.Lock() + defer h.mu.Unlock() + if h.fired || stage != h.stage { + return nil + } + h.fired = true + h.scope = scopeKey + return fmt.Errorf("injected halt at %s (scope %q)", stage, scopeKey) +} + +func TestSourceCache_HaltPointSweep(t *testing.T) { + stages := []string{"replay-copied", "rows-committed", "tombstones-applied", "manifest-written"} + for _, stage := range stages { + t.Run(stage, func(t *testing.T) { + runHaltPointScenario(t, stage) + }) + } +} + +func runHaltPointScenario(t *testing.T, stage string) { + ctx, err := logging.Init(t.Context()) + require.NoError(t, err) + tmpDir := t.TempDir() + rng := rand.New(rand.NewSource(42)) //nolint:gosec // deterministic test randomness + + model := newEquivModel() + warmConn := newEquivConnector(model, false) + coldConn := newEquivConnector(model, true) + var warmClient types.ConnectorClient = warmConn + var coldClient types.ConnectorClient = coldConn + + extPath := filepath.Join(tmpDir, "external-r0.c1z") + buildExternalC1z(ctx, t, model, extPath, tmpDir) + model.extDirty = false + syncOpts := []SyncOpt{WithExternalResourceC1ZPath(extPath)} + + // Round 0: initial warm-chain sync (a cold fetch — no previous file). + basePath := filepath.Join(tmpDir, "warm-r0.c1z") + require.NoError(t, runEquivSync(ctx, t, warmClient, basePath, "", tmpDir, syncOpts...)) + + // Round 1 mutations: tombstones, overlays, cold refetches, external + // churn — the full mixed shape, so every swept seam actually fires. + model.mutate(1, rng) + if model.extDirty { + extPath = filepath.Join(tmpDir, "external-r1.c1z") + buildExternalC1z(ctx, t, model, extPath, tmpDir) + syncOpts = []SyncOpt{WithExternalResourceC1ZPath(extPath)} + } + + // Warm sync, halted once at the stage under test, then resumed. + halt := &haltOnce{stage: stage} + warmPath := filepath.Join(tmpDir, "warm-halt.c1z") + hookOpt := func(s *syncer) { s.testSourceCacheHaltHook = halt.hook } + haltErr := runEquivSync(ctx, t, warmClient, warmPath, basePath, tmpDir, append(syncOpts, hookOpt)...) + require.Error(t, haltErr, "the armed halt must crash the warm sync") + require.Contains(t, haltErr.Error(), "injected halt at "+stage) + require.NoError(t, runEquivSync(ctx, t, warmClient, warmPath, basePath, tmpDir, append(syncOpts, hookOpt)...), + "resume after halt at %s (scope %q) must complete", stage, halt.scope) + + // Cold reference for the same model state. + coldPath := filepath.Join(tmpDir, "cold-r1.c1z") + require.NoError(t, runEquivSync(ctx, t, coldClient, coldPath, "", tmpDir, syncOpts...)) + + coldSnap := snapshotC1z(ctx, t, coldPath) + warmSnap := snapshotC1z(ctx, t, warmPath) + require.Equal(t, 1, warmSnap.syncRuns, + "halt at %s: the crashed warm sync must RESUME (one sync run), not restart", stage) + assertColdArtifacts(t, 1, model, coldSnap) + assertNoStaleResources(t, 1, model, "warm", warmSnap) + requireSnapshotsEqual(t, 1, coldSnap, warmSnap) +} diff --git a/pkg/sync/source_cache_mixed_version_test.go b/pkg/sync/source_cache_mixed_version_test.go new file mode 100644 index 000000000..518784c45 --- /dev/null +++ b/pkg/sync/source_cache_mixed_version_test.go @@ -0,0 +1,90 @@ +package sync //nolint:revive,nolintlint // we can't change the package name for backwards compatibility + +// Mixed-version compaction artifacts and replay eligibility. +// +// A fold compactor built from an SDK that predates the run record's +// compacted flag produces an artifact that LOOKS replay-eligible to the +// metadata gate: the record carries a full type and no compacted flag +// (the old proto has no such field), and the old fold never cleared the +// base copy's source-cache manifest (the old engine has no source-cache +// code), so the manifest belt passes too. The one signal every compactor +// version has always written is the sync TOKEN's compaction provenance +// section (BuildCompactedToken) — so the syncer's replay gate must read +// it and refuse the artifact. + +import ( + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + "github.com/conductorone/baton-sdk/pkg/dotc1z" + "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" + pebbleengine "github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble" + "github.com/conductorone/baton-sdk/pkg/logging" + gt "github.com/conductorone/baton-sdk/pkg/types/grant" +) + +// TestSourceCache_PreCompactedFlagFoldArtifactDegradesToCold pins the +// token-provenance gate: a previous sync whose run record says +// full/non-compacted but whose sync token carries a compaction section +// (the shape an old-binary fold leaves on a new-format base) must never +// serve replay, even with its manifest fully intact. +func TestSourceCache_PreCompactedFlagFoldArtifactDegradesToCold(t *testing.T) { + ctx, err := logging.Init(t.Context()) + require.NoError(t, err) + tmpDir := t.TempDir() + + group, ent, alice, _ := sourceCacheTestFixtures(t) + g1 := gt.NewGrant(group, "member", alice) + + mc := newSourceCacheMockConnector() + mc.AddResource(ctx, group) + mc.AddResource(ctx, alice) + mc.entDB[group.GetId().GetResource()] = []*v2.Entitlement{ent} + mc.grantDB[group.GetId().GetResource()] = []*v2.Grant{g1} + mc.etagByResource[group.GetId().GetResource()] = `W/"etag-v1"` + + sync1 := filepath.Join(tmpDir, "sync1.c1z") + runSourceCacheSync(ctx, t, mc, sync1, "", tmpDir) + + // Rewrite sync 1's run record into the old-fold shape: keep the full + // type, leave compacted UNSET (the old proto has no field to set), + // and give it the compaction-provenance token every fold — old or + // new — writes. The source-cache manifest stays intact, exactly as + // an old fold (no ClearSourceCacheEntries) would leave it. + marker, err := dotc1z.NewStore(ctx, sync1, + dotc1z.WithEngine(c1zstore.EnginePebble), + dotc1z.WithTmpDir(tmpDir), + ) + require.NoError(t, err) + run, err := marker.SyncMeta().LatestFullSync(ctx) + require.NoError(t, err) + require.NotNil(t, run) + engineHolder, ok := any(marker).(interface{ PebbleEngine() *pebbleengine.Engine }) + require.True(t, ok) + rec, err := engineHolder.PebbleEngine().GetSyncRunRecord(ctx, run.ID) + require.NoError(t, err) + require.False(t, rec.GetCompacted(), "sanity: the record must NOT carry the compacted flag for this test to prove the token gate") + compactedToken, err := BuildCompactedToken(rec.GetSyncToken(), CompactionTokenInput{ + Mode: "fold", + BaseSyncID: run.ID, + PartialSyncIDs: []string{"old-partial-1"}, + }) + require.NoError(t, err) + rec.SetSyncToken(compactedToken) + require.NoError(t, engineHolder.PebbleEngine().PutSyncRunRecord(ctx, rec)) + require.True(t, pebbleengine.MarkStoreDirty(marker), "the token write must persist through Close") + require.NoError(t, marker.Close(ctx)) + + // Warm sync against the doctored file: every lookup must MISS + // despite the intact manifest and the clean-looking run record. + sync2 := filepath.Join(tmpDir, "sync2.c1z") + missesBefore := mc.lookupMisses + runSourceCacheSync(ctx, t, mc, sync2, sync1, tmpDir) + require.Zero(t, mc.lookupHits, + "a previous sync whose token carries compaction provenance must never produce a source-cache lookup hit, even without the compacted flag") + require.Greater(t, mc.lookupMisses, missesBefore, + "sanity: the connector must have looked up and missed (no-op lookup installed)") +} diff --git a/pkg/sync/source_cache_sideeffects_test.go b/pkg/sync/source_cache_sideeffects_test.go new file mode 100644 index 000000000..5e9f365fd --- /dev/null +++ b/pkg/sync/source_cache_sideeffects_test.go @@ -0,0 +1,1070 @@ +package sync //nolint:revive,nolintlint // we can't change the package name for backwards compatibility + +// Replay side-effect regression tests: response-row-driven side effects +// (child scheduling, external-resource-match processing, related-resource +// insertion) must survive a warm replay round exactly as a cold sync +// would produce them. Each test runs sync 1 cold, sync 2 as a 304-style +// replay, and asserts the side effect's output is present in sync 2. + +import ( + "context" + "fmt" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + "github.com/conductorone/baton-sdk/pkg/annotations" + "github.com/conductorone/baton-sdk/pkg/connectorstore" + "github.com/conductorone/baton-sdk/pkg/dotc1z" + "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" + pebbleengine "github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble" + "github.com/conductorone/baton-sdk/pkg/logging" + "github.com/conductorone/baton-sdk/pkg/sourcecache" + "github.com/conductorone/baton-sdk/pkg/types" + gt "github.com/conductorone/baton-sdk/pkg/types/grant" + rs "github.com/conductorone/baton-sdk/pkg/types/resource" +) + +// runSourceCacheSyncClient is runSourceCacheSync for wrapper connectors +// that embed sourceCacheMockConnector but override list methods: the +// OUTER value must be handed to the syncer or the overrides never run. +func runSourceCacheSyncClient(ctx context.Context, t *testing.T, cc types.ConnectorClient, path, prevPath, tmpDir string, extraOpts ...SyncOpt) { + t.Helper() + store, err := dotc1z.NewStore(ctx, path, + dotc1z.WithEngine(c1zstore.EnginePebble), + dotc1z.WithTmpDir(tmpDir), + ) + require.NoError(t, err) + opts := []SyncOpt{WithConnectorStore(store), WithTmpDir(tmpDir), WithFailFastInvariants()} + if prevPath != "" { + opts = append(opts, WithPreviousSyncC1ZPath(prevPath)) + } + opts = append(opts, extraOpts...) + syncer, err := NewSyncer(ctx, cc, opts...) + require.NoError(t, err) + require.NoError(t, syncer.Sync(ctx)) + require.NoError(t, syncer.Close(ctx)) +} + +// listResourcesInFile reopens a finished Pebble c1z and returns its +// resources of one type, keyed by resource ID. +func listResourcesInFile(ctx context.Context, t *testing.T, path, resourceTypeID string) map[string]*v2.Resource { + t.Helper() + reopen, err := dotc1z.NewStore(ctx, path, + dotc1z.WithEngine(c1zstore.EnginePebble), + dotc1z.WithReadOnly(true), + ) + require.NoError(t, err) + defer func() { _ = reopen.Close(ctx) }() + + sync, err := reopen.SyncMeta().LatestFullSync(ctx) + require.NoError(t, err) + require.NotNil(t, sync) + require.NoError(t, reopen.SetCurrentSync(ctx, sync.ID)) + + out := map[string]*v2.Resource{} + pageToken := "" + for { + resp, err := reopen.ListResources(ctx, v2.ResourcesServiceListResourcesRequest_builder{ + ResourceTypeId: resourceTypeID, + PageToken: pageToken, + }.Build()) + require.NoError(t, err) + for _, r := range resp.GetList() { + out[r.GetId().GetResource()] = r + } + pageToken = resp.GetNextPageToken() + if pageToken == "" { + break + } + } + return out +} + +// --------------------------------------------------------------------- +// 1. Replayed parent resources must still schedule child resource syncs. +// --------------------------------------------------------------------- + +// childReplayConnector serves a parent resource type as a source-cache +// scope (fresh on miss, 304 replay on hit) whose resources carry +// ChildResourceType annotations. Child resources are only returned when +// the request names a parent — the standard parent/child connector shape. +type childReplayConnector struct { + *sourceCacheMockConnector + + parentType *v2.ResourceType + childType *v2.ResourceType + parents []*v2.Resource + children map[string][]*v2.Resource // parent resource id -> children + + resourceEtag string + resourceLookupHits int +} + +func (c *childReplayConnector) ListResources(ctx context.Context, in *v2.ResourcesServiceListResourcesRequest, opts ...grpc.CallOption) (*v2.ResourcesServiceListResourcesResponse, error) { + switch in.GetResourceTypeId() { + case c.parentType.GetId(): + scope := sourcecache.HashScope("mock://resources/" + c.parentType.GetId()) + c.mu.Lock() + lookup := c.lookup + etag := c.resourceEtag + c.mu.Unlock() + if lookup == nil { + lookup = sourcecache.NoopLookup{} + } + entry, found, err := lookup.Lookup(ctx, sourcecache.RowKindResources, scope) + if err != nil { + return nil, err + } + if found && entry.CacheValidator == etag { + c.mu.Lock() + c.resourceLookupHits++ + c.mu.Unlock() + return v2.ResourcesServiceListResourcesResponse_builder{ + Annotations: annotations.New(v2.SourceCacheReplay_builder{ + ScopeKey: scope, + CacheValidator: etag, + }.Build()), + }.Build(), nil + } + return v2.ResourcesServiceListResourcesResponse_builder{ + List: c.parents, + Annotations: annotations.New(v2.SourceCacheRecord_builder{ + ScopeKey: scope, + CacheValidator: etag, + }.Build()), + }.Build(), nil + case c.childType.GetId(): + if in.GetParentResourceId() == nil { + // Children are only reachable through a parent. + return v2.ResourcesServiceListResourcesResponse_builder{}.Build(), nil + } + return v2.ResourcesServiceListResourcesResponse_builder{ + List: c.children[in.GetParentResourceId().GetResource()], + }.Build(), nil + } + return c.sourceCacheMockConnector.ListResources(ctx, in, opts...) +} + +// TestSourceCache_ReplaySchedulesChildResources pins the child-scheduling +// contract under replay: a 304 replay of a parent resources page copies +// the parents engine-side and returns no rows, so the response-row loop +// that normally pushes child SyncResources actions never sees them. The +// replay path must schedule those children itself, or a warm sync +// silently drops whole subtrees. +func TestSourceCache_ReplaySchedulesChildResources(t *testing.T) { + for _, workers := range workerCounts { + t.Run(fmt.Sprintf("workers=%d", workers), func(t *testing.T) { + runReplaySchedulesChildResourcesScenario(t, workerSyncOpts(workers)) + }) + } +} + +func runReplaySchedulesChildResourcesScenario(t *testing.T, workerOpts []SyncOpt) { + ctx, err := logging.Init(t.Context()) + require.NoError(t, err) + tmpDir := t.TempDir() + + parentRT := v2.ResourceType_builder{Id: "sc_parent", DisplayName: "Parent"}.Build() + childRT := v2.ResourceType_builder{Id: "sc_child", DisplayName: "Child"}.Build() + + parent1, err := rs.NewResource("Parent 1", parentRT, "parent_1", + rs.WithAnnotation(v2.ChildResourceType_builder{ResourceTypeId: childRT.GetId()}.Build()), + rs.WithAnnotation(&v2.SkipEntitlementsAndGrants{}), + ) + require.NoError(t, err) + child1, err := rs.NewResource("Child 1", childRT, "child_1", + rs.WithParentResourceID(parent1.GetId()), + rs.WithAnnotation(&v2.SkipEntitlementsAndGrants{}), + ) + require.NoError(t, err) + child2, err := rs.NewResource("Child 2", childRT, "child_2", + rs.WithParentResourceID(parent1.GetId()), + rs.WithAnnotation(&v2.SkipEntitlementsAndGrants{}), + ) + require.NoError(t, err) + + inner := newSourceCacheMockConnector() + inner.rtDB = []*v2.ResourceType{parentRT, childRT} + mc := &childReplayConnector{ + sourceCacheMockConnector: inner, + parentType: parentRT, + childType: childRT, + parents: []*v2.Resource{parent1}, + children: map[string][]*v2.Resource{ + parent1.GetId().GetResource(): {child1, child2}, + }, + resourceEtag: `W/"resources-v1"`, + } + + sync1 := filepath.Join(tmpDir, "sync1.c1z") + sync2 := filepath.Join(tmpDir, "sync2.c1z") + + runSourceCacheSyncClient(ctx, t, mc, sync1, "", tmpDir, workerOpts...) + require.Zero(t, mc.resourceLookupHits, "sync 1 has no previous sync to hit") + require.Len(t, listResourcesInFile(ctx, t, sync1, childRT.GetId()), 2, + "sanity: cold sync must fetch children through the parent") + + // Upstream unchanged: the parent page replays. The children must + // still be scheduled and fetched. + runSourceCacheSyncClient(ctx, t, mc, sync2, sync1, tmpDir, workerOpts...) + require.NotZero(t, mc.resourceLookupHits, "sanity: sync 2 must have replayed the parent page") + + parents2 := listResourcesInFile(ctx, t, sync2, parentRT.GetId()) + require.Contains(t, parents2, parent1.GetId().GetResource(), "replayed parent must be present") + + children2 := listResourcesInFile(ctx, t, sync2, childRT.GetId()) + require.Contains(t, children2, child1.GetId().GetResource(), + "child resources of a replayed parent must be scheduled and synced") + require.Contains(t, children2, child2.GetId().GetResource(), + "child resources of a replayed parent must be scheduled and synced") +} + +// --------------------------------------------------------------------- +// 2. External-resource-match grants must survive a warm replay. +// --------------------------------------------------------------------- + +// TestSourceCache_ReplayPreservesExternalMatchGrants pins the +// external-match contract under replay. Cold sync 1 emits a +// scope-stamped grant carrying ExternalResourceMatchID; the +// external-resources step transforms it into a grant on the matched +// external principal and deletes the source grant. Sync 2's 304 replay +// must carry the transformed grant forward — and the replay path must +// re-arm HasExternalResourcesGrants so match processing still runs on +// replayed rows. +func TestSourceCache_ReplayPreservesExternalMatchGrants(t *testing.T) { + for _, workers := range workerCounts { + t.Run(fmt.Sprintf("workers=%d", workers), func(t *testing.T) { + runReplayPreservesExternalMatchGrantsScenario(t, workerSyncOpts(workers)) + }) + } +} + +func runReplayPreservesExternalMatchGrantsScenario(t *testing.T, workerOpts []SyncOpt) { + ctx, err := logging.Init(t.Context()) + require.NoError(t, err) + tmpDir := t.TempDir() + + // External c1z: one user the match annotation will resolve to. + externalMc := newMockConnector() + externalMc.rtDB = append(externalMc.rtDB, userResourceType, groupResourceType) + externalUser, err := externalMc.AddUserProfile(ctx, "ext_user_1", map[string]any{}) + require.NoError(t, err) + + externalC1z := filepath.Join(tmpDir, "external.c1z") + externalSyncer, err := NewSyncer(ctx, externalMc, WithC1ZPath(externalC1z), WithTmpDir(tmpDir)) + require.NoError(t, err) + require.NoError(t, externalSyncer.Sync(ctx)) + require.NoError(t, externalSyncer.Close(ctx)) + + // Internal connector: one group whose grants page is a source-cache + // scope. The single grant references a placeholder principal and + // carries ExternalResourceMatchID pointing at the external user. + mc := newSourceCacheMockConnector() + internalGroup, ent, err := mc.AddGroup(ctx, "internal_group") + require.NoError(t, err) + + placeholder := v2.ResourceId_builder{ResourceType: "user", Resource: "placeholder_1"}.Build() + sourceGrant := gt.NewGrant(internalGroup, "member", placeholder, + gt.WithAnnotation(v2.ExternalResourceMatchID_builder{ + Id: externalUser.GetId().GetResource(), + }.Build()), + ) + mc.grantDB[internalGroup.GetId().GetResource()] = []*v2.Grant{sourceGrant} + mc.etagByResource[internalGroup.GetId().GetResource()] = `W/"grants-v1"` + + transformedGrantID := gt.NewGrantID(externalUser, sourceGrant.GetEntitlement()) + _ = ent + + extOpts := append([]SyncOpt{WithExternalResourceC1ZPath(externalC1z)}, workerOpts...) + + sync1 := filepath.Join(tmpDir, "sync1.c1z") + sync2 := filepath.Join(tmpDir, "sync2.c1z") + sync3 := filepath.Join(tmpDir, "sync3.c1z") + + runSourceCacheSync(ctx, t, mc, sync1, "", tmpDir, extOpts...) + grants1 := listGrantsInFile(ctx, t, sync1) + require.Contains(t, grants1, transformedGrantID, "sanity: cold sync must produce the transformed grant") + require.NotContains(t, grants1, sourceGrant.GetId(), "sanity: the placeholder source grant must be deleted") + + // Upstream unchanged: the grants scope replays. The transformed + // grant must survive. + runSourceCacheSync(ctx, t, mc, sync2, sync1, tmpDir, extOpts...) + require.NotZero(t, mc.lookupHits, "sanity: sync 2 must have replayed") + + grants2 := listGrantsInFile(ctx, t, sync2) + require.Contains(t, grants2, transformedGrantID, + "transformed external-match grant must survive a 304 replay of its source scope") + require.NotContains(t, grants2, sourceGrant.GetId()) + + // And the chain must not decay: sync 2's file must still work as the + // next replay source. + runSourceCacheSync(ctx, t, mc, sync3, sync2, tmpDir, extOpts...) + grants3 := listGrantsInFile(ctx, t, sync3) + require.Contains(t, grants3, transformedGrantID, + "transformed grant must survive a second consecutive replay round") +} + +// --------------------------------------------------------------------- +// 3. InsertResourceGrants resources must survive a warm replay. +// --------------------------------------------------------------------- + +// insertResourceGrantsConnector serves one grants page (for target) as a +// source-cache scope whose response carries InsertResourceGrants: the +// grants reference a resource no ListResources call ever returns, so the +// only way the resource reaches the store is grant-driven insertion. +type insertResourceGrantsConnector struct { + *sourceCacheMockConnector + + target string + insertGrants []*v2.Grant +} + +func (c *insertResourceGrantsConnector) ListGrants(ctx context.Context, in *v2.GrantsServiceListGrantsRequest, _ ...grpc.CallOption) (*v2.GrantsServiceListGrantsResponse, error) { + resourceID := in.GetResource().GetId().GetResource() + if resourceID != c.target { + return v2.GrantsServiceListGrantsResponse_builder{}.Build(), nil + } + scope := sourcecache.HashScope("mock://grants/insert/" + resourceID) + + c.mu.Lock() + lookup := c.lookup + currentEtag := c.etagByResource[resourceID] + c.mu.Unlock() + if lookup == nil { + lookup = sourcecache.NoopLookup{} + } + entry, found, err := lookup.Lookup(ctx, sourcecache.RowKindGrants, scope) + if err != nil { + return nil, err + } + if found && entry.CacheValidator == currentEtag { + c.mu.Lock() + c.lookupHits++ + c.mu.Unlock() + return v2.GrantsServiceListGrantsResponse_builder{ + Annotations: annotations.New(v2.SourceCacheReplay_builder{ + ScopeKey: scope, + CacheValidator: currentEtag, + }.Build()), + }.Build(), nil + } + c.mu.Lock() + c.lookupMisses++ + c.mu.Unlock() + return v2.GrantsServiceListGrantsResponse_builder{ + List: c.insertGrants, + Annotations: annotations.New( + v2.SourceCacheRecord_builder{ScopeKey: scope, CacheValidator: currentEtag}.Build(), + &v2.InsertResourceGrants{}, + ), + }.Build(), nil +} + +// TestSourceCache_ReplayPreservesInsertResourceGrants pins related- +// resource insertion under replay: a grants page annotated with +// InsertResourceGrants writes the grants' entitlement resources into the +// resources table. On a 304 replay the grant rows are copied engine-side +// but the response has no rows, so the insertion loop never runs — the +// replay must recreate those resource rows or sync 2 holds grants whose +// resources do not exist. +func TestSourceCache_ReplayPreservesInsertResourceGrants(t *testing.T) { + for _, workers := range workerCounts { + t.Run(fmt.Sprintf("workers=%d", workers), func(t *testing.T) { + runReplayPreservesInsertResourceGrantsScenario(t, workerSyncOpts(workers)) + }) + } +} + +func runReplayPreservesInsertResourceGrantsScenario(t *testing.T, workerOpts []SyncOpt) { + ctx, err := logging.Init(t.Context()) + require.NoError(t, err) + tmpDir := t.TempDir() + + projectRT := v2.ResourceType_builder{Id: "project", DisplayName: "Project"}.Build() + + inner := newSourceCacheMockConnector() + inner.rtDB = append(inner.rtDB, projectRT) + internalGroup, _, err := inner.AddGroup(ctx, "g1") + require.NoError(t, err) + alice, err := inner.AddUser(ctx, "alice") + require.NoError(t, err) + + // The project resource is NEVER returned by ListResources — it only + // exists as the embedded resource of the grant below. + projectRes, err := rs.NewResource("Project 1", projectRT, "proj_1", + rs.WithAnnotation(&v2.SkipEntitlementsAndGrants{}), + ) + require.NoError(t, err) + projectGrant := gt.NewGrant(projectRes, "admin", alice) + + inner.etagByResource[internalGroup.GetId().GetResource()] = `W/"insert-v1"` + mc := &insertResourceGrantsConnector{ + sourceCacheMockConnector: inner, + target: internalGroup.GetId().GetResource(), + insertGrants: []*v2.Grant{projectGrant}, + } + + sync1 := filepath.Join(tmpDir, "sync1.c1z") + sync2 := filepath.Join(tmpDir, "sync2.c1z") + + runSourceCacheSyncClient(ctx, t, mc, sync1, "", tmpDir, workerOpts...) + projects1 := listResourcesInFile(ctx, t, sync1, projectRT.GetId()) + require.Contains(t, projects1, "proj_1", "sanity: cold sync must insert the grant's resource") + require.Equal(t, "Project 1", projects1["proj_1"].GetDisplayName()) + require.Contains(t, listGrantsInFile(ctx, t, sync1), projectGrant.GetId()) + + // Upstream unchanged: the grants scope replays. The inserted + // resource must survive alongside its grants. + runSourceCacheSyncClient(ctx, t, mc, sync2, sync1, tmpDir, workerOpts...) + require.NotZero(t, mc.lookupHits, "sanity: sync 2 must have replayed") + + grants2 := listGrantsInFile(ctx, t, sync2) + require.Contains(t, grants2, projectGrant.GetId(), "replayed grant must be present") + + projects2 := listResourcesInFile(ctx, t, sync2, projectRT.GetId()) + require.Contains(t, projects2, "proj_1", + "InsertResourceGrants resource must be recreated when its grants page replays") + require.Equal(t, "Project 1", projects2["proj_1"].GetDisplayName(), + "recreated resource must be a faithful copy of the previous sync's row") +} + +// --------------------------------------------------------------------- +// 4. Replayed entitlements must participate in exclusion-group checks. +// --------------------------------------------------------------------- + +// exclusionGroupReplayConnector serves entitlements pages as source-cache +// scopes for resources with a registered validator (fresh on miss, 304 +// replay on hit); resources without one list entitlements plain. +type exclusionGroupReplayConnector struct { + *sourceCacheMockConnector + + entEtags map[string]string // resource id -> upstream validator + entLookupHits int + entLookupMiss int +} + +func (c *exclusionGroupReplayConnector) ListEntitlements( + ctx context.Context, + in *v2.EntitlementsServiceListEntitlementsRequest, + opts ...grpc.CallOption, +) (*v2.EntitlementsServiceListEntitlementsResponse, error) { + resourceID := in.GetResource().GetId().GetResource() + c.mu.Lock() + lookup := c.lookup + etag := c.entEtags[resourceID] + c.mu.Unlock() + if etag == "" { + return c.sourceCacheMockConnector.ListEntitlements(ctx, in, opts...) + } + scope := sourcecache.HashScope("mock://entitlements/" + resourceID) + if lookup == nil { + lookup = sourcecache.NoopLookup{} + } + entry, found, err := lookup.Lookup(ctx, sourcecache.RowKindEntitlements, scope) + if err != nil { + return nil, err + } + if found && entry.CacheValidator == etag { + c.mu.Lock() + c.entLookupHits++ + c.mu.Unlock() + return v2.EntitlementsServiceListEntitlementsResponse_builder{ + Annotations: annotations.New(v2.SourceCacheReplay_builder{ + ScopeKey: scope, + CacheValidator: etag, + }.Build()), + }.Build(), nil + } + c.mu.Lock() + c.entLookupMiss++ + c.mu.Unlock() + return v2.EntitlementsServiceListEntitlementsResponse_builder{ + List: c.entDB[resourceID], + Annotations: annotations.New(v2.SourceCacheRecord_builder{ + ScopeKey: scope, + CacheValidator: etag, + }.Build()), + }.Build(), nil +} + +// TestSourceCache_ReplayedEntitlementsKeepExclusionGroupValidation pins +// the exclusion-group invariants under replay: validation normally runs +// on response rows only, so a replayed entitlement's exclusion-group +// membership would silently stop participating — a duplicate default +// split across a replayed scope and a fresh scope would pass a warm sync +// that a full resync hard-fails. +func TestSourceCache_ReplayedEntitlementsKeepExclusionGroupValidation(t *testing.T) { + for _, workers := range workerCounts { + t.Run(fmt.Sprintf("workers=%d", workers), func(t *testing.T) { + runReplayedEntitlementExclusionGroupScenario(t, workerSyncOpts(workers)) + }) + } +} + +func runReplayedEntitlementExclusionGroupScenario(t *testing.T, workerOpts []SyncOpt) { + ctx, err := logging.Init(t.Context()) + require.NoError(t, err) + tmpDir := t.TempDir() + + inner := newSourceCacheMockConnector() + mc := &exclusionGroupReplayConnector{ + sourceCacheMockConnector: inner, + entEtags: map[string]string{}, + } + + defaultExclusionGroup := v2.EntitlementExclusionGroup_builder{ + ExclusionGroupId: "eg1", + IsDefault: true, + }.Build() + + // Group A: its entitlements page is a scope, carrying the exclusion + // group's default entitlement. + groupA, entA, err := inner.AddGroup(ctx, "group_a") + require.NoError(t, err) + entAAnnos := annotations.Annotations(entA.GetAnnotations()) + entAAnnos.Update(defaultExclusionGroup) + entA.SetAnnotations(entAAnnos) + mc.entEtags[groupA.GetId().GetResource()] = `W/"ents-a-v1"` + + sync1 := filepath.Join(tmpDir, "sync1.c1z") + runSourceCacheSyncClient(ctx, t, mc, sync1, "", tmpDir, workerOpts...) + require.NotZero(t, mc.entLookupMiss, "sanity: sync 1 must fetch group A's entitlements fresh") + + // Group B appears with a FRESH entitlement claiming the same + // exclusion group's default. A full resync would fail; the warm sync + // (group A's page replays) must fail identically. + groupB, entB, err := inner.AddGroup(ctx, "group_b") + require.NoError(t, err) + entBAnnos := annotations.Annotations(entB.GetAnnotations()) + entBAnnos.Update(defaultExclusionGroup) + entB.SetAnnotations(entBAnnos) + _ = groupB + + sync2 := filepath.Join(tmpDir, "sync2.c1z") + store, err := dotc1z.NewStore(ctx, sync2, + dotc1z.WithEngine(c1zstore.EnginePebble), + dotc1z.WithTmpDir(tmpDir), + ) + require.NoError(t, err) + opts := append([]SyncOpt{WithConnectorStore(store), WithTmpDir(tmpDir), WithPreviousSyncC1ZPath(sync1)}, workerOpts...) + syncer, err := NewSyncer(ctx, mc, opts...) + require.NoError(t, err) + err = syncer.Sync(ctx) + require.Error(t, err, + "a duplicate exclusion-group default split across a replayed and a fresh scope must fail the warm sync like a cold sync would") + require.Contains(t, err.Error(), "multiple default entitlements") + require.NoError(t, syncer.Close(ctx)) + require.NotZero(t, mc.entLookupHits, "sanity: sync 2 must have replayed group A's entitlements page") +} + +// --------------------------------------------------------------------- +// Partial syncs must not replay. +// --------------------------------------------------------------------- + +// TestSourceCache_PartialSyncDegradesToCold pins the full-sync-only gate: +// a partial (targeted) sync with a warm previous file must install the +// no-op lookup — every scope misses, the connector fetches fresh — rather +// than replaying whole scopes into a targeted subset (which could +// resurrect out-of-target rows, and whose related-resource fetch only +// runs over response rows). +func TestSourceCache_PartialSyncDegradesToCold(t *testing.T) { + ctx, err := logging.Init(t.Context()) + require.NoError(t, err) + tmpDir := t.TempDir() + + group, ent, alice, _ := sourceCacheTestFixtures(t) + g1 := gt.NewGrant(group, "member", alice) + + mc := newSourceCacheMockConnector() + mc.AddResource(ctx, group) + mc.AddResource(ctx, alice) + mc.entDB[group.GetId().GetResource()] = []*v2.Entitlement{ent} + mc.grantDB[group.GetId().GetResource()] = []*v2.Grant{g1} + mc.etagByResource[group.GetId().GetResource()] = `W/"etag-v1"` + + sync1 := filepath.Join(tmpDir, "sync1.c1z") + runSourceCacheSync(ctx, t, mc, sync1, "", tmpDir) + require.Zero(t, mc.lookupHits) + + // Partial sync targeting the group, with the warm file supplied: the + // connector's lookups must all MISS despite the warm previous sync. + sync2 := filepath.Join(tmpDir, "sync2.c1z") + store, err := dotc1z.NewStore(ctx, sync2, + dotc1z.WithEngine(c1zstore.EnginePebble), + dotc1z.WithTmpDir(tmpDir), + ) + require.NoError(t, err) + missesBefore := mc.lookupMisses + syncer, err := NewSyncer(ctx, mc, + WithConnectorStore(store), + WithTmpDir(tmpDir), + WithPreviousSyncC1ZPath(sync1), + WithTargetedSyncResources([]*v2.Resource{group}), + ) + require.NoError(t, err) + require.NoError(t, syncer.Sync(ctx)) + require.NoError(t, syncer.Close(ctx)) + + require.Zero(t, mc.lookupHits, "a partial sync must never get a source-cache lookup hit") + require.Greater(t, mc.lookupMisses, missesBefore, + "sanity: the connector must have looked up and missed (no-op lookup installed)") +} + +// --------------------------------------------------------------------- +// Non-full previous syncs must not replay. +// --------------------------------------------------------------------- + +// TestSourceCache_PartialPreviousSyncDegradesToCold pins the type half of +// the replay-source predicate (UsableAsReplaySource): a PARTIAL-typed +// previous sync must never serve replay, even when its file carries a +// manifest entry over stamped rows. The fixture plants exactly that — +// a partial sync with a valid-looking manifest — so a pass proves the +// run-record TYPE gate causes the degradation, not an incidentally empty +// manifest. Partial syncs are subsets: their rows do not cover the +// scopes any validator vouches for. +func TestSourceCache_PartialPreviousSyncDegradesToCold(t *testing.T) { + ctx, err := logging.Init(t.Context()) + require.NoError(t, err) + tmpDir := t.TempDir() + + group, ent, alice, _ := sourceCacheTestFixtures(t) + g1 := gt.NewGrant(group, "member", alice) + grantsScope := grantScopeForResource(group.GetId().GetResource()) + + // Hand-build a PARTIAL sync whose grants are stamped with the scope + // the connector will look up, manifest entry included. + sync1 := filepath.Join(tmpDir, "partial-prev.c1z") + prevStore, err := dotc1z.NewStore(ctx, sync1, + dotc1z.WithEngine(c1zstore.EnginePebble), + dotc1z.WithTmpDir(tmpDir), + ) + require.NoError(t, err) + _, isNew, err := prevStore.StartOrResumeSync(ctx, connectorstore.SyncTypePartial, "") + require.NoError(t, err) + require.True(t, isNew) + require.NoError(t, prevStore.PutResourceTypes(ctx, groupResourceType, userResourceType)) + require.NoError(t, prevStore.PutResources(ctx, group, alice)) + require.NoError(t, prevStore.PutEntitlements(ctx, ent)) + require.NoError(t, prevStore.PutGrants(sourcecache.WithScope(ctx, grantsScope), g1)) + scPrev, ok := any(prevStore).(dotc1z.SourceCacheStore) + require.True(t, ok) + require.NoError(t, scPrev.PutSourceCacheEntry(ctx, sourcecache.RowKindGrants, grantsScope, `W/"etag-v1"`)) + require.NoError(t, prevStore.EndSync(ctx)) + require.NoError(t, prevStore.Close(ctx)) + + mc := newSourceCacheMockConnector() + mc.AddResource(ctx, group) + mc.AddResource(ctx, alice) + mc.entDB[group.GetId().GetResource()] = []*v2.Entitlement{ent} + mc.grantDB[group.GetId().GetResource()] = []*v2.Grant{g1} + mc.etagByResource[group.GetId().GetResource()] = `W/"etag-v1"` + + sync2 := filepath.Join(tmpDir, "sync2.c1z") + missesBefore := mc.lookupMisses + runSourceCacheSync(ctx, t, mc, sync2, sync1, tmpDir) + require.Zero(t, mc.lookupHits, + "a partial-typed previous sync must never produce a source-cache lookup hit, even with a planted manifest entry") + require.Greater(t, mc.lookupMisses, missesBefore, + "sanity: the connector must have looked up and missed (no-op lookup installed)") +} + +// --------------------------------------------------------------------- +// Replay-compatibility key: exact match or cold. +// --------------------------------------------------------------------- + +// TestSourceCache_CompatKeyGate pins the replay-compatibility key: replay +// is permitted ONLY when the previous artifact's recorded key matches the +// current run's key byte-exactly on every component. Each subtest changes +// exactly one component between two otherwise-identical syncs and +// requires zero lookup hits; the control run (identical key) must hit. +// Compatibility is never inferred from versions — the gate compares +// explicit declarations. +func TestSourceCache_CompatKeyGate(t *testing.T) { + newFixtureConnector := func(ctx context.Context, t *testing.T) *sourceCacheMockConnector { + t.Helper() + group, ent, alice, _ := sourceCacheTestFixtures(t) + mc := newSourceCacheMockConnector() + mc.AddResource(ctx, group) + mc.AddResource(ctx, alice) + mc.entDB[group.GetId().GetResource()] = []*v2.Entitlement{ent} + mc.grantDB[group.GetId().GetResource()] = []*v2.Grant{gt.NewGrant(group, "member", alice)} + mc.etagByResource[group.GetId().GetResource()] = `W/"etag-v1"` + return mc + } + + t.Run("control: identical key replays", func(t *testing.T) { + ctx, err := logging.Init(t.Context()) + require.NoError(t, err) + tmpDir := t.TempDir() + mc := newFixtureConnector(ctx, t) + mc.cacheGeneration = "gen-1" + mc.configFingerprint = "cfg-abc" + + sync1 := filepath.Join(tmpDir, "sync1.c1z") + runSourceCacheSync(ctx, t, mc, sync1, "", tmpDir) + runSourceCacheSync(ctx, t, mc, filepath.Join(tmpDir, "sync2.c1z"), sync1, tmpDir) + require.Positive(t, mc.lookupHits, "an exactly-matching compatibility key must permit replay") + }) + + t.Run("connector cache-generation change degrades to cold", func(t *testing.T) { + ctx, err := logging.Init(t.Context()) + require.NoError(t, err) + tmpDir := t.TempDir() + mc := newFixtureConnector(ctx, t) + mc.cacheGeneration = "gen-1" + + sync1 := filepath.Join(tmpDir, "sync1.c1z") + runSourceCacheSync(ctx, t, mc, sync1, "", tmpDir) + mc.cacheGeneration = "gen-2" // connector changed its cache semantics + missesBefore := mc.lookupMisses + runSourceCacheSync(ctx, t, mc, filepath.Join(tmpDir, "sync2.c1z"), sync1, tmpDir) + require.Zero(t, mc.lookupHits, "a cache-generation bump must refuse the prior artifact") + require.Greater(t, mc.lookupMisses, missesBefore, "sanity: lookups must have missed, not been skipped") + }) + + t.Run("connector config fingerprint change degrades to cold", func(t *testing.T) { + ctx, err := logging.Init(t.Context()) + require.NoError(t, err) + tmpDir := t.TempDir() + mc := newFixtureConnector(ctx, t) + mc.configFingerprint = "cfg-abc" + + sync1 := filepath.Join(tmpDir, "sync1.c1z") + runSourceCacheSync(ctx, t, mc, sync1, "", tmpDir) + mc.configFingerprint = "cfg-def" // permissions/config changed what upstream is visible + missesBefore := mc.lookupMisses + runSourceCacheSync(ctx, t, mc, filepath.Join(tmpDir, "sync2.c1z"), sync1, tmpDir) + require.Zero(t, mc.lookupHits, "a config-fingerprint change must refuse the prior artifact") + require.Greater(t, mc.lookupMisses, missesBefore) + }) + + t.Run("sync-selection change degrades to cold", func(t *testing.T) { + ctx, err := logging.Init(t.Context()) + require.NoError(t, err) + tmpDir := t.TempDir() + mc := newFixtureConnector(ctx, t) + + sync1 := filepath.Join(tmpDir, "sync1.c1z") + runSourceCacheSync(ctx, t, mc, sync1, "", tmpDir) + // Same connector, same config — but the SDK-side selection now + // names the types explicitly, changing the selection fingerprint. + missesBefore := mc.lookupMisses + runSourceCacheSync(ctx, t, mc, filepath.Join(tmpDir, "sync2.c1z"), sync1, tmpDir, + WithSyncResourceTypes([]string{groupResourceType.GetId(), userResourceType.GetId()})) + require.Zero(t, mc.lookupHits, "a sync-selection change must refuse the prior artifact") + require.Greater(t, mc.lookupMisses, missesBefore) + }) + + t.Run("pre-key artifact degrades to cold", func(t *testing.T) { + ctx, err := logging.Init(t.Context()) + require.NoError(t, err) + tmpDir := t.TempDir() + + // Hand-build a FULL previous sync with a valid-looking manifest + // over stamped rows but NO compatibility record — the shape every + // artifact written by a pre-key SDK has. + group, ent, alice, _ := sourceCacheTestFixtures(t) + g1 := gt.NewGrant(group, "member", alice) + grantsScope := grantScopeForResource(group.GetId().GetResource()) + sync1 := filepath.Join(tmpDir, "prekey-prev.c1z") + prevStore, err := dotc1z.NewStore(ctx, sync1, + dotc1z.WithEngine(c1zstore.EnginePebble), + dotc1z.WithTmpDir(tmpDir), + ) + require.NoError(t, err) + _, isNew, err := prevStore.StartOrResumeSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + require.True(t, isNew) + require.NoError(t, prevStore.PutResourceTypes(ctx, groupResourceType, userResourceType)) + require.NoError(t, prevStore.PutResources(ctx, group, alice)) + require.NoError(t, prevStore.PutEntitlements(ctx, ent)) + require.NoError(t, prevStore.PutGrants(sourcecache.WithScope(ctx, grantsScope), g1)) + scPrev, ok := any(prevStore).(dotc1z.SourceCacheStore) + require.True(t, ok) + require.NoError(t, scPrev.PutSourceCacheEntry(ctx, sourcecache.RowKindGrants, grantsScope, `W/"etag-v1"`)) + _, found, err := scPrev.GetSourceCacheCompat(ctx) + require.NoError(t, err) + require.False(t, found, "fixture: the hand-built artifact must carry NO compat record, or this test proves nothing") + require.NoError(t, prevStore.EndSync(ctx)) + require.NoError(t, prevStore.Close(ctx)) + + mc := newFixtureConnector(ctx, t) + missesBefore := mc.lookupMisses + runSourceCacheSync(ctx, t, mc, filepath.Join(tmpDir, "sync2.c1z"), sync1, tmpDir) + require.Zero(t, mc.lookupHits, "an artifact without a compatibility key must never serve replay") + require.Greater(t, mc.lookupMisses, missesBefore) + }) +} + +// --------------------------------------------------------------------- +// SQLite artifacts must not replay (engine non-goal). +// --------------------------------------------------------------------- + +// TestSourceCache_SQLitePreviousSyncDegradesToCold pins the READ-side +// engine gate: SQLite replay is an explicit non-goal, so a SQLite +// previous-sync c1z — even a complete, finished full sync — must degrade +// to a cold sync (every lookup misses; the file opens fine but is not a +// dotc1z.SourceCacheStore). The current sync's OUTPUT store is Pebble, +// so the write side stays enabled: this isolates exactly the +// "previous-sync store engine does not support source cache" arm of +// configureSourceCache. +func TestSourceCache_SQLitePreviousSyncDegradesToCold(t *testing.T) { + ctx, err := logging.Init(t.Context()) + require.NoError(t, err) + tmpDir := t.TempDir() + + group, ent, alice, _ := sourceCacheTestFixtures(t) + g1 := gt.NewGrant(group, "member", alice) + + // A complete SQLITE previous sync (default engine). + sync1 := filepath.Join(tmpDir, "sqlite-prev.c1z") + prevStore, err := dotc1z.NewStore(ctx, sync1, + dotc1z.WithEngine(c1zstore.EngineSQLite), + dotc1z.WithTmpDir(tmpDir), + ) + require.NoError(t, err) + _, isNew, err := prevStore.StartOrResumeSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + require.True(t, isNew) + require.NoError(t, prevStore.PutResourceTypes(ctx, groupResourceType, userResourceType)) + require.NoError(t, prevStore.PutResources(ctx, group, alice)) + require.NoError(t, prevStore.PutEntitlements(ctx, ent)) + require.NoError(t, prevStore.PutGrants(ctx, g1)) + _, isSourceCacheStore := any(prevStore).(dotc1z.SourceCacheStore) + require.False(t, isSourceCacheStore, + "fixture: the SQLite store must NOT implement SourceCacheStore, or this test proves nothing") + require.NoError(t, prevStore.EndSync(ctx)) + require.NoError(t, prevStore.Close(ctx)) + + mc := newSourceCacheMockConnector() + mc.AddResource(ctx, group) + mc.AddResource(ctx, alice) + mc.entDB[group.GetId().GetResource()] = []*v2.Entitlement{ent} + mc.grantDB[group.GetId().GetResource()] = []*v2.Grant{g1} + mc.etagByResource[group.GetId().GetResource()] = `W/"etag-v1"` + + sync2 := filepath.Join(tmpDir, "sync2.c1z") + missesBefore := mc.lookupMisses + runSourceCacheSync(ctx, t, mc, sync2, sync1, tmpDir) + require.Zero(t, mc.lookupHits, + "a SQLite previous sync must never produce a source-cache lookup hit") + require.Greater(t, mc.lookupMisses, missesBefore, + "sanity: the connector must have looked up and missed (no-op lookup installed)") +} + +// TestSourceCache_SQLiteOutputStoreDegradesToCold pins the WRITE-side +// engine gate: when the CURRENT sync's output store is SQLite, source +// cache disables entirely — the connector's scope annotations are +// ignored (not an error), replay would be a hard error, and the sync +// completes cold. This is the exact shape of the default-engine CLI +// footgun (--previous-sync-c1z without --storage-engine pebble), which +// must degrade loudly-but-safely rather than fail or silently corrupt. +func TestSourceCache_SQLiteOutputStoreDegradesToCold(t *testing.T) { + ctx, err := logging.Init(t.Context()) + require.NoError(t, err) + tmpDir := t.TempDir() + + group, ent, alice, _ := sourceCacheTestFixtures(t) + g1 := gt.NewGrant(group, "member", alice) + + // A valid PEBBLE previous sync: eligible on its own, but the write + // side's engine gate must disable the whole feature first. + mc := newSourceCacheMockConnector() + mc.AddResource(ctx, group) + mc.AddResource(ctx, alice) + mc.entDB[group.GetId().GetResource()] = []*v2.Entitlement{ent} + mc.grantDB[group.GetId().GetResource()] = []*v2.Grant{g1} + mc.etagByResource[group.GetId().GetResource()] = `W/"etag-v1"` + sync1 := filepath.Join(tmpDir, "pebble-prev.c1z") + runSourceCacheSync(ctx, t, mc, sync1, "", tmpDir) + + // Warm attempt into a SQLITE output store: the connector still emits + // SourceCacheRecord annotations on its fresh pages (it declared the + // capability and fetched fresh after every miss) — they must be + // ignored without error. + sqliteOut, err := dotc1z.NewStore(ctx, filepath.Join(tmpDir, "sqlite-out.c1z"), + dotc1z.WithEngine(c1zstore.EngineSQLite), + dotc1z.WithTmpDir(tmpDir), + ) + require.NoError(t, err) + missesBefore := mc.lookupMisses + syncer, err := NewSyncer(ctx, mc, + WithConnectorStore(sqliteOut), + WithTmpDir(tmpDir), + WithPreviousSyncC1ZPath(sync1), + ) + require.NoError(t, err) + require.NoError(t, syncer.Sync(ctx), + "a SQLite output store must degrade source cache to cold, never fail the sync") + require.NoError(t, syncer.Close(ctx)) + require.Zero(t, mc.lookupHits, + "with source cache disabled the connector must never see a lookup hit") + require.Greater(t, mc.lookupMisses, missesBefore, + "sanity: the connector must have looked up and missed (no-op lookup installed)") +} + +// --------------------------------------------------------------------- +// Compacted previous syncs must not replay. +// --------------------------------------------------------------------- + +// TestSourceCache_CompactedPreviousSyncDegradesToCold pins the METADATA +// gate: a previous sync whose run record carries the compacted flag must +// never serve replay — independent of its manifest contents. The test +// leaves the manifest fully intact and only flips the flag, so a pass +// proves the gate itself (not the compactor's manifest clearing) causes +// the degradation. Compaction invalidates validators categorically: a +// keep-newer merge is not a row set any input's etag describes. +func TestSourceCache_CompactedPreviousSyncDegradesToCold(t *testing.T) { + ctx, err := logging.Init(t.Context()) + require.NoError(t, err) + tmpDir := t.TempDir() + + group, ent, alice, _ := sourceCacheTestFixtures(t) + g1 := gt.NewGrant(group, "member", alice) + + mc := newSourceCacheMockConnector() + mc.AddResource(ctx, group) + mc.AddResource(ctx, alice) + mc.entDB[group.GetId().GetResource()] = []*v2.Entitlement{ent} + mc.grantDB[group.GetId().GetResource()] = []*v2.Grant{g1} + mc.etagByResource[group.GetId().GetResource()] = `W/"etag-v1"` + + sync1 := filepath.Join(tmpDir, "sync1.c1z") + runSourceCacheSync(ctx, t, mc, sync1, "", tmpDir) + + // Flip ONLY the compacted flag on sync 1's run record; its + // source-cache manifest stays intact and would otherwise replay. + marker, err := dotc1z.NewStore(ctx, sync1, + dotc1z.WithEngine(c1zstore.EnginePebble), + dotc1z.WithTmpDir(tmpDir), + ) + require.NoError(t, err) + run, err := marker.SyncMeta().LatestFullSync(ctx) + require.NoError(t, err) + require.NotNil(t, run) + engineHolder, ok := any(marker).(interface{ PebbleEngine() *pebbleengine.Engine }) + require.True(t, ok) + rec, err := engineHolder.PebbleEngine().GetSyncRunRecord(ctx, run.ID) + require.NoError(t, err) + rec.SetCompacted(true) + require.NoError(t, engineHolder.PebbleEngine().PutSyncRunRecord(ctx, rec)) + require.True(t, pebbleengine.MarkStoreDirty(marker), "the flag write must persist through Close") + require.NoError(t, marker.Close(ctx)) + + // Warm sync against the compacted-marked file: every lookup must + // MISS despite the intact manifest. + sync2 := filepath.Join(tmpDir, "sync2.c1z") + missesBefore := mc.lookupMisses + runSourceCacheSync(ctx, t, mc, sync2, sync1, tmpDir) + require.Zero(t, mc.lookupHits, + "a compacted previous sync must never produce a source-cache lookup hit") + require.Greater(t, mc.lookupMisses, missesBefore, + "sanity: the connector must have looked up and missed (no-op lookup installed)") +} + +// --------------------------------------------------------------------- +// 5. Continuation answers accumulated across bounces must stay +// acceptable to the connector-side answer parser. +// --------------------------------------------------------------------- + +// TestSourceCacheContinuation_AnswerAccumulationBeyondOneAsk pins the +// cap coherence between the two sides of the ask/answer continuation: +// answers accumulate across bounces (every re-invoke carries the union), +// so a maximal first ask (4096 queries, the ask cap) followed by ONE +// late scope — explicitly legal per the annotation contract — produces +// 4097 answers on the next re-invoke. The connector-side parser must +// accept that union; capping it at one ask's size hard-fails a legal +// exchange. +func TestSourceCacheContinuation_AnswerAccumulationBeyondOneAsk(t *testing.T) { + ctx, err := logging.Init(t.Context()) + require.NoError(t, err) + + store := &stubSourceCacheStore{} + s := &syncer{ + state: newState(), + sourceCache: syncerSourceCache{ + enabled: true, + current: store, + prev: store, + lookup: staticLookup{etag: "e"}, + replayedScopes: &replayedScopeSet{}, + }, + } + + bigAsk := make([]sourcecache.Query, 0, 4096) + for i := 0; i < 4096; i++ { + bigAsk = append(bigAsk, sourcecache.Query{ + RowKind: sourcecache.RowKindGrants, + ScopeKey: fmt.Sprintf("scope-%04d", i), + }) + } + lateAsk := []sourcecache.Query{{RowKind: sourcecache.RowKindGrants, ScopeKey: "late-scope"}} + + err = s.withSourceCacheContinuation(ctx, "test-op", func(extra annotations.Annotations, attempt int) (listAttempt, error) { + answersMsg := &v2.SourceCacheLookupAnswers{} + hasAnswers, err := extra.Pick(answersMsg) + if err != nil { + return listAttempt{}, err + } + if hasAnswers { + // The connector-side boundary every re-invoke passes through + // (connectorbuilder's continuationOpAttrs): the syncer's + // accumulated union must never exceed what it accepts. + if _, err := sourcecache.AnswersFromProto(answersMsg); err != nil { + return listAttempt{}, err + } + } + switch attempt { + case 0: + require.False(t, hasAnswers) + return listAttempt{annos: annotations.New(sourcecache.AskProto(bigAsk))}, nil + case 1: + require.Len(t, answersMsg.GetAnswers(), 4096) + // Phase 2 legally queries one scope it did not ask before. + return listAttempt{annos: annotations.New(sourcecache.AskProto(lateAsk))}, nil + default: + require.Len(t, answersMsg.GetAnswers(), 4097) + return listAttempt{}, nil + } + }) + require.NoError(t, err, + "a maximal first ask plus one late scope is a legal exchange; the accumulated answer union must be accepted end to end") +} + +// TestSourceCacheContinuation_OversizedAskFailsLoudly pins the intended +// hard boundary on ONE ask: more than 4096 queries in a single ask is a +// connector bug (shard via EnqueuePageTokens instead) and must fail the sync +// with a descriptive error, not silently truncate. +func TestSourceCacheContinuation_OversizedAskFailsLoudly(t *testing.T) { + ctx, err := logging.Init(t.Context()) + require.NoError(t, err) + + store := &stubSourceCacheStore{} + s := &syncer{ + state: newState(), + sourceCache: syncerSourceCache{ + enabled: true, + current: store, + prev: store, + lookup: staticLookup{etag: "e"}, + replayedScopes: &replayedScopeSet{}, + }, + } + + oversized := make([]sourcecache.Query, 0, 4097) + for i := 0; i < 4097; i++ { + oversized = append(oversized, sourcecache.Query{ + RowKind: sourcecache.RowKindGrants, + ScopeKey: fmt.Sprintf("scope-%04d", i), + }) + } + err = s.withSourceCacheContinuation(ctx, "test-op", func(extra annotations.Annotations, attempt int) (listAttempt, error) { + return listAttempt{annos: annotations.New(sourcecache.AskProto(oversized))}, nil + }) + require.Error(t, err) + require.Contains(t, err.Error(), "max 4096") +} diff --git a/pkg/sync/source_cache_skew_fixture_test.go b/pkg/sync/source_cache_skew_fixture_test.go new file mode 100644 index 000000000..3fc32c9e6 --- /dev/null +++ b/pkg/sync/source_cache_skew_fixture_test.go @@ -0,0 +1,83 @@ +package sync //nolint:revive,nolintlint // we can't change the package name for backwards compatibility + +// Frozen-artifact version-skew test. Unlike the synthesized +// mixed-version test (source_cache_mixed_version_test.go), this one +// reads REAL bytes produced by the merge-base SDK's fold compactor — +// see testdata/README-old-fold.md — so it also verifies the parts of +// old-binary behavior we would otherwise have to guess (measured: the +// old fold carries the base's source-cache manifest through INTACT, +// validators and all). + +import ( + "io" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + "github.com/conductorone/baton-sdk/pkg/dotc1z" + "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" + "github.com/conductorone/baton-sdk/pkg/logging" + gt "github.com/conductorone/baton-sdk/pkg/types/grant" +) + +func TestSourceCache_FrozenOldFoldArtifactRefusedForReplay(t *testing.T) { + ctx, err := logging.Init(t.Context()) + require.NoError(t, err) + tmpDir := t.TempDir() + + // Copy the frozen artifact out of testdata (syncs open the previous + // file next to their own tmp state). + prev := filepath.Join(tmpDir, "old-fold.c1z") + src, err := os.Open(filepath.Join("testdata", "old-fold-mixed-version.c1z")) + require.NoError(t, err) + dst, err := os.Create(prev) + require.NoError(t, err) + _, err = io.Copy(dst, src) + require.NoError(t, err) + require.NoError(t, src.Close()) + require.NoError(t, dst.Close()) + + // Sanity: this fixture is the dangerous shape — every record-level + // gate passes and the manifest is intact, so ONLY the token + // provenance can refuse it. If any of these fail, the fixture no + // longer proves what it claims (see testdata/README-old-fold.md). + store, err := dotc1z.NewStore(ctx, prev, + dotc1z.WithEngine(c1zstore.EnginePebble), + dotc1z.WithReadOnly(true), + dotc1z.WithTmpDir(tmpDir), + ) + require.NoError(t, err) + run, err := store.SyncMeta().LatestFullSync(ctx) + require.NoError(t, err) + require.NotNil(t, run) + require.False(t, run.Compacted, "fixture: old binary cannot set the compacted flag") + require.True(t, run.UsableAsReplaySource(), "fixture: the record-level gate must PASS, or this test proves nothing") + comp, err := CompactionStatsFromToken(run.SyncToken) + require.NoError(t, err) + require.NotNil(t, comp, "fixture: the token must carry compaction provenance — the one honest signal") + inspector, ok := store.(dotc1z.SourceCacheInspector) + require.True(t, ok) + manifest, err := inspector.SourceCacheManifestSnapshot(ctx) + require.NoError(t, err) + require.NotEmpty(t, manifest, "fixture: the old fold carries the base manifest intact — the manifest belt must PASS too") + require.NoError(t, store.Close(ctx)) + + // The trap: a connector whose current validator MATCHES the carried + // manifest entry. A broken gate would 304 and replay merged rows. + group, ent, alice, _ := sourceCacheTestFixtures(t) + mc := newSourceCacheMockConnector() + mc.AddResource(ctx, group) + mc.AddResource(ctx, alice) + mc.entDB[group.GetId().GetResource()] = []*v2.Entitlement{ent} + mc.grantDB[group.GetId().GetResource()] = []*v2.Grant{gt.NewGrant(group, "member", alice)} + mc.etagByResource[group.GetId().GetResource()] = `W/"skew-v1"` // matches the carried validator + + out := filepath.Join(tmpDir, "out.c1z") + runSourceCacheSync(ctx, t, mc, out, prev, tmpDir) + require.Zero(t, mc.lookupHits, + "an old-binary fold artifact must never serve replay: its rows are a merge no upstream validator describes") + require.Positive(t, mc.lookupMisses, "sanity: lookups must have been offered and missed") +} diff --git a/pkg/sync/source_cache_stats.go b/pkg/sync/source_cache_stats.go new file mode 100644 index 000000000..b360de043 --- /dev/null +++ b/pkg/sync/source_cache_stats.go @@ -0,0 +1,153 @@ +package sync //nolint:revive,nolintlint // we can't change the package name for backwards compatibility + +// Source-cache counters, following the sync-stats model (see the stats +// fields on state and syncSummaryFields): recorded through the state so +// they persist in the checkpointed sync token — a warm sync that suspends +// and resumes keeps its counts — and reported once in the sync summary. + +import ( + "context" + + "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" + "go.uber.org/zap" + + "github.com/conductorone/baton-sdk/pkg/sourcecache" +) + +// SourceCacheStats accumulates per-sync source-cache counters: replay +// effectiveness, delta tombstones, spawn fan-out, and the lookup +// continuation (ask/answer) protocol. All fields are cumulative for the +// sync; the zero value means "feature unused". +type SourceCacheStats struct { + // ScopesReplayed counts pages answered with SourceCacheReplay. With + // ScopesStamped it gives the sync's replay hit ratio — the headline + // warm-sync health metric. + ScopesReplayed int64 `json:"scopes_replayed,omitempty"` + // ScopesStamped counts manifest entries written for NON-replayed + // (cold/fresh) pages. Interim pages with no validator count in + // neither bucket. + // + // APPROXIMATE UNDER RESUME: the suppression that keeps a multi-page + // overlay round's final (validator-carrying) page out of this count + // rides an in-memory set (syncerSourceCache.replayedScopes); a + // resume landing mid-round books that round as both a hit and a + // miss. Rows are unaffected — but do NOT gate behavior on the + // replay ratio without first persisting that set (or deriving + // suppression from durable manifest state). + ScopesStamped int64 `json:"scopes_stamped,omitempty"` + // RowsReplayed counts rows bulk-copied from the previous sync, keyed + // by row kind (resources / entitlements / grants). + RowsReplayed map[sourcecache.RowKind]int64 `json:"rows_replayed,omitempty"` + // OverlayRows counts connector-emitted rows upserted on top of a + // replayed base (delta adds/changes). + OverlayRows int64 `json:"overlay_rows,omitempty"` + + // TombstoneIDs / TombstonePrincipals count tombstones received on + // replay/scope annotations; RowsDeleted counts rows they removed + // (where the delete path reports it). A delta connector applying zero + // tombstones for weeks is a signal worth charting. + TombstoneIDs int64 `json:"tombstone_ids,omitempty"` + TombstonePrincipals int64 `json:"tombstone_principals,omitempty"` + RowsDeleted int64 `json:"rows_deleted,omitempty"` + + // EnqueuedPageTokens counts sibling actions enqueued via EnqueuePageTokens + // (pairs with the per-response cap). + EnqueuedPageTokens int64 `json:"enqueued_page_tokens,omitempty"` + + // Lookup continuation (ask/answer) counters. BouncesByOp splits + // bounces by list-op kind, distinguishing planner asks (expected: one + // per planning page) from per-row asks (a batching opportunity). + // AnswersTruncated counts found answers degraded to not-found (cold + // fetch) because their etags did not fit the per-request answer size + // budget; nonzero means a connector is batching more fat-token scopes + // per ask than the transport can carry and should shard via + // EnqueuePageTokens. + LookupBounces int64 `json:"lookup_bounces,omitempty"` + LookupRequestsBounced int64 `json:"lookup_requests_bounced,omitempty"` + LookupScopesAsked int64 `json:"lookup_scopes_asked,omitempty"` + LookupAnsweredFound int64 `json:"lookup_answered_found,omitempty"` + LookupAnsweredNotFound int64 `json:"lookup_answered_not_found,omitempty"` + LookupAnswersTruncated int64 `json:"lookup_answers_truncated,omitempty"` + LookupCapFailures int64 `json:"lookup_cap_failures,omitempty"` + LookupBouncesByOp map[string]int64 `json:"lookup_bounces_by_op,omitempty"` +} + +func (s *SourceCacheStats) merge(delta SourceCacheStats) { + s.ScopesReplayed += delta.ScopesReplayed + s.ScopesStamped += delta.ScopesStamped + for kind, n := range delta.RowsReplayed { + if s.RowsReplayed == nil { + s.RowsReplayed = make(map[sourcecache.RowKind]int64, len(delta.RowsReplayed)) + } + s.RowsReplayed[kind] += n + } + s.OverlayRows += delta.OverlayRows + s.TombstoneIDs += delta.TombstoneIDs + s.TombstonePrincipals += delta.TombstonePrincipals + s.RowsDeleted += delta.RowsDeleted + s.EnqueuedPageTokens += delta.EnqueuedPageTokens + s.LookupBounces += delta.LookupBounces + s.LookupRequestsBounced += delta.LookupRequestsBounced + s.LookupScopesAsked += delta.LookupScopesAsked + s.LookupAnsweredFound += delta.LookupAnsweredFound + s.LookupAnsweredNotFound += delta.LookupAnsweredNotFound + s.LookupAnswersTruncated += delta.LookupAnswersTruncated + s.LookupCapFailures += delta.LookupCapFailures + for op, n := range delta.LookupBouncesByOp { + if s.LookupBouncesByOp == nil { + s.LookupBouncesByOp = make(map[string]int64, len(delta.LookupBouncesByOp)) + } + s.LookupBouncesByOp[op] += n + } +} + +// copy returns a deep copy (maps included). +func (s *SourceCacheStats) copy() *SourceCacheStats { + out := *s + if s.RowsReplayed != nil { + out.RowsReplayed = make(map[sourcecache.RowKind]int64, len(s.RowsReplayed)) + for k, v := range s.RowsReplayed { + out.RowsReplayed[k] = v + } + } + if s.LookupBouncesByOp != nil { + out.LookupBouncesByOp = make(map[string]int64, len(s.LookupBouncesByOp)) + for k, v := range s.LookupBouncesByOp { + out.LookupBouncesByOp[k] = v + } + } + return &out +} + +func (st *state) AddSourceCacheStats(delta SourceCacheStats) { + st.mtx.Lock() + defer st.mtx.Unlock() + if st.sourceCacheStats == nil { + st.sourceCacheStats = &SourceCacheStats{} + } + st.sourceCacheStats.merge(delta) +} + +func (st *state) SourceCacheStatsSnapshot() *SourceCacheStats { + st.mtx.RLock() + defer st.mtx.RUnlock() + if st.sourceCacheStats == nil { + return nil + } + return st.sourceCacheStats.copy() +} + +// logSourceCacheStats emits the source-cache summary line when the +// feature saw any use. Deliberately NOT gated on recordStats: replay is +// opt-in per connector and these counters are its rollout telemetry (the +// replay hit ratio, and the bounce counts the consumer briefs chart). +func (s *syncer) logSourceCacheStats(ctx context.Context) { + if s.state == nil { + return + } + stats := s.state.SourceCacheStatsSnapshot() + if stats == nil { + return + } + ctxzap.Extract(ctx).Info("source cache stats", zap.Any("source_cache_stats", stats)) +} diff --git a/pkg/sync/source_cache_stats_test.go b/pkg/sync/source_cache_stats_test.go new file mode 100644 index 000000000..610cfe967 --- /dev/null +++ b/pkg/sync/source_cache_stats_test.go @@ -0,0 +1,117 @@ +package sync //nolint:revive,nolintlint // we can't change the package name for backwards compatibility + +import ( + "context" + "path/filepath" + "testing" + + "github.com/conductorone/baton-sdk/pkg/sourcecache" + + "github.com/stretchr/testify/require" + + "github.com/conductorone/baton-sdk/pkg/dotc1z" + "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" + "github.com/conductorone/baton-sdk/pkg/logging" +) + +// TestSourceCacheStats_TokenRoundTrip pins the resume property: the +// counters live in the checkpointed sync token, so a suspended warm sync +// keeps its replay/bounce counts across Marshal → Unmarshal. +func TestSourceCacheStats_TokenRoundTrip(t *testing.T) { + st := newState() + require.NoError(t, st.Unmarshal("")) + + st.AddSourceCacheStats(SourceCacheStats{ + ScopesReplayed: 3, + ScopesStamped: 2, + RowsReplayed: map[sourcecache.RowKind]int64{"grants": 40, "resources": 5}, + OverlayRows: 4, + TombstoneIDs: 2, + LookupBounces: 6, + LookupBouncesByOp: map[string]int64{ + "sync-grants-for-resource": 6, + }, + }) + // Second merge accumulates. + st.AddSourceCacheStats(SourceCacheStats{ + ScopesReplayed: 1, + RowsReplayed: map[sourcecache.RowKind]int64{"grants": 10}, + LookupBouncesByOp: map[string]int64{"sync-resources": 1}, + }) + + token, err := st.Marshal() + require.NoError(t, err) + + resumed := newState() + require.NoError(t, resumed.Unmarshal(token)) + got := resumed.SourceCacheStatsSnapshot() + require.NotNil(t, got, "stats must survive the token round trip") + require.Equal(t, int64(4), got.ScopesReplayed) + require.Equal(t, int64(2), got.ScopesStamped) + require.Equal(t, int64(50), got.RowsReplayed["grants"]) + require.Equal(t, int64(5), got.RowsReplayed["resources"]) + require.Equal(t, int64(4), got.OverlayRows) + require.Equal(t, int64(2), got.TombstoneIDs) + require.Equal(t, int64(6), got.LookupBounces) + require.Equal(t, int64(6), got.LookupBouncesByOp["sync-grants-for-resource"]) + require.Equal(t, int64(1), got.LookupBouncesByOp["sync-resources"]) + + // Resumed state keeps accumulating on top of the restored counters. + resumed.AddSourceCacheStats(SourceCacheStats{ScopesReplayed: 1}) + require.Equal(t, int64(5), resumed.SourceCacheStatsSnapshot().ScopesReplayed) + + // A fresh (empty-input) state reports nothing. + fresh := newState() + require.NoError(t, fresh.Unmarshal("")) + require.Nil(t, fresh.SourceCacheStatsSnapshot()) +} + +// TestSourceCacheStats_EndToEnd runs a cold sync then a warm +// continuation sync and asserts the recorded counters match the known +// shape: 5 groups → 5 stamped scopes cold; 5 replayed scopes, 5 bounces +// (one ask per group page), 10 rows replayed, zero cold pages warm. +func TestSourceCacheStats_EndToEnd(t *testing.T) { + ctx, err := logging.Init(context.Background()) + require.NoError(t, err) + tmpDir := t.TempDir() + mc := newContinuationMockConnector(t, 5) + + runWithStats := func(path, prevPath string) *SourceCacheStats { + store, err := dotc1z.NewStore(ctx, path, + dotc1z.WithEngine(c1zstore.EnginePebble), + dotc1z.WithTmpDir(tmpDir), + ) + require.NoError(t, err) + opts := []SyncOpt{WithConnectorStore(store), WithTmpDir(tmpDir)} + if prevPath != "" { + opts = append(opts, WithPreviousSyncC1ZPath(prevPath)) + } + sc, err := NewSyncer(ctx, mc, opts...) + require.NoError(t, err) + require.NoError(t, sc.Sync(ctx)) + impl, ok := sc.(*syncer) + require.True(t, ok) + stats := impl.state.SourceCacheStatsSnapshot() + require.NoError(t, sc.Close(ctx)) + return stats + } + + sync1 := filepath.Join(tmpDir, "sync1.c1z") + cold := runWithStats(sync1, "") + require.NotNil(t, cold) + require.Equal(t, int64(5), cold.ScopesStamped, "cold: every group page stamps a scope") + require.Zero(t, cold.ScopesReplayed) + require.Zero(t, cold.LookupBounces, "no offer on a cold sync, so no bounces") + + warm := runWithStats(filepath.Join(tmpDir, "sync2.c1z"), sync1) + require.NotNil(t, warm) + require.Equal(t, int64(5), warm.ScopesReplayed, "warm: every group page replays") + require.Zero(t, warm.ScopesStamped, "no cold pages on the warm sync") + require.Equal(t, int64(10), warm.RowsReplayed["grants"], "5 groups × 2 member grants replayed") + require.Equal(t, int64(5), warm.LookupBounces, "one ask bounce per group page") + require.Equal(t, int64(5), warm.LookupRequestsBounced) + require.Equal(t, int64(5), warm.LookupScopesAsked) + require.Equal(t, int64(5), warm.LookupAnsweredFound) + require.Equal(t, int64(5), warm.LookupBouncesByOp["sync-grants-for-resource"]) + require.Zero(t, warm.LookupCapFailures) +} diff --git a/pkg/sync/source_cache_test.go b/pkg/sync/source_cache_test.go new file mode 100644 index 000000000..6139b8bbe --- /dev/null +++ b/pkg/sync/source_cache_test.go @@ -0,0 +1,578 @@ +package sync //nolint:revive,nolintlint // we can't change the package name for backwards compatibility + +import ( + "context" + "fmt" + "path/filepath" + "sync" + "testing" + + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + "github.com/conductorone/baton-sdk/pkg/annotations" + "github.com/conductorone/baton-sdk/pkg/connectorstore" + "github.com/conductorone/baton-sdk/pkg/dotc1z" + "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" + "github.com/conductorone/baton-sdk/pkg/logging" + "github.com/conductorone/baton-sdk/pkg/sourcecache" + et "github.com/conductorone/baton-sdk/pkg/types/entitlement" + gt "github.com/conductorone/baton-sdk/pkg/types/grant" + rs "github.com/conductorone/baton-sdk/pkg/types/resource" +) + +// sourceCacheMockConnector emulates a connector against an upstream with +// cheap change detection. Every grants page for a resource has a stable +// scope; the connector looks up the previous validator via the lookup the +// syncer installs (SetSourceCache) and either: +// +// - miss, or validator != current upstream etag: return fresh rows + +// SourceCacheRecord (a 200), +// - hit with matching etag: return no rows + SourceCacheReplay (a 304), +// - hit in delta mode: return only changed rows + overlay replay with +// tombstones and a rotated token (a Graph-style delta round). +type sourceCacheMockConnector struct { + *mockConnector + + mu sync.Mutex + lookup sourcecache.Lookup + + // etagByResource is the upstream's CURRENT validator per resource. + etagByResource map[string]string + + // delta mode: overlay rows + deletions returned on a lookup hit. + deltaMode bool + deltaAdds map[string][]*v2.Grant + deltaDeletedIDs map[string][]string + // deltaDeletedPrincipals are bare principal-id tombstones, emitted on + // the SourceCacheRecord annotation (the per-page channel) rather than + // the replay annotation — exercising both proposal A and B paths. + deltaDeletedPrincipals map[string][]string + + lookupHits int + lookupMisses int + + // Replay-compatibility key components declared on Validate + // (SourceCacheCapability.cache_generation / config_fingerprint). + // Empty by default — a constant, matching generation. + cacheGeneration string + configFingerprint string +} + +func newSourceCacheMockConnector() *sourceCacheMockConnector { + mc := &sourceCacheMockConnector{ + mockConnector: newMockConnector(), + etagByResource: map[string]string{}, + deltaAdds: map[string][]*v2.Grant{}, + deltaDeletedIDs: map[string][]string{}, + deltaDeletedPrincipals: map[string][]string{}, + } + mc.rtDB = append(mc.rtDB, groupResourceType, userResourceType) + return mc +} + +func (mc *sourceCacheMockConnector) SetSourceCache(_ context.Context, lookup sourcecache.Lookup) { + mc.mu.Lock() + defer mc.mu.Unlock() + if lookup == nil { + lookup = sourcecache.NoopLookup{} + } + mc.lookup = lookup +} + +func (mc *sourceCacheMockConnector) Validate(context.Context, *v2.ConnectorServiceValidateRequest, ...grpc.CallOption) (*v2.ConnectorServiceValidateResponse, error) { + mc.mu.Lock() + defer mc.mu.Unlock() + return v2.ConnectorServiceValidateResponse_builder{ + Annotations: annotations.New(v2.SourceCacheCapability_builder{ + Mode: v2.SourceCacheCapability_MODE_READ_WRITE, + CacheGeneration: mc.cacheGeneration, + ConfigFingerprint: mc.configFingerprint, + }.Build()), + }.Build(), nil +} + +func grantScopeForResource(resourceID string) string { + return sourcecache.HashScope("mock://grants/" + resourceID) +} + +func (mc *sourceCacheMockConnector) ListGrants(ctx context.Context, in *v2.GrantsServiceListGrantsRequest, _ ...grpc.CallOption) (*v2.GrantsServiceListGrantsResponse, error) { + resourceID := in.GetResource().GetId().GetResource() + scope := grantScopeForResource(resourceID) + + mc.mu.Lock() + lookup := mc.lookup + currentEtag := mc.etagByResource[resourceID] + deltaMode := mc.deltaMode + adds := mc.deltaAdds[resourceID] + deleted := mc.deltaDeletedIDs[resourceID] + deletedPrincipals := mc.deltaDeletedPrincipals[resourceID] + mc.mu.Unlock() + + if lookup == nil { + lookup = sourcecache.NoopLookup{} + } + entry, found, err := lookup.Lookup(ctx, sourcecache.RowKindGrants, scope) + if err != nil { + return nil, err + } + + if found && deltaMode { + // Delta round: replay the base, overlay changes, tombstone + // removals, rotate the token. Canonical-id tombstones ride the + // replay annotation; bare principal-id tombstones ride the scope + // annotation (the per-page channel every page of a round can use). + mc.mu.Lock() + mc.lookupHits++ + mc.mu.Unlock() + return v2.GrantsServiceListGrantsResponse_builder{ + List: adds, + Annotations: annotations.New( + v2.SourceCacheReplay_builder{ + ScopeKey: scope, + CacheValidator: currentEtag, // new token for this round + Overlay: true, + DeletedIds: deleted, + }.Build(), + v2.SourceCacheRecord_builder{ + ScopeKey: scope, + DeletedPrincipalIds: deletedPrincipals, + }.Build(), + ), + }.Build(), nil + } + + if found && entry.CacheValidator == currentEtag { + // Conditional-request 304: nothing changed upstream. + mc.mu.Lock() + mc.lookupHits++ + mc.mu.Unlock() + return v2.GrantsServiceListGrantsResponse_builder{ + List: []*v2.Grant{}, + Annotations: annotations.New(v2.SourceCacheReplay_builder{ + ScopeKey: scope, + CacheValidator: currentEtag, + }.Build()), + }.Build(), nil + } + + // Fresh fetch (miss or changed). + mc.mu.Lock() + mc.lookupMisses++ + mc.mu.Unlock() + return v2.GrantsServiceListGrantsResponse_builder{ + List: mc.grantDB[resourceID], + Annotations: annotations.New(v2.SourceCacheRecord_builder{ + ScopeKey: scope, + CacheValidator: currentEtag, + }.Build()), + }.Build(), nil +} + +// runSourceCacheSync runs one full sync into a fresh Pebble c1z at path, +// optionally replaying from prevPath. +func runSourceCacheSync(ctx context.Context, t *testing.T, mc *sourceCacheMockConnector, path, prevPath, tmpDir string, extraOpts ...SyncOpt) { + t.Helper() + store, err := dotc1z.NewStore(ctx, path, + dotc1z.WithEngine(c1zstore.EnginePebble), + dotc1z.WithTmpDir(tmpDir), + ) + require.NoError(t, err) + opts := []SyncOpt{WithConnectorStore(store), WithTmpDir(tmpDir), WithFailFastInvariants()} + if prevPath != "" { + opts = append(opts, WithPreviousSyncC1ZPath(prevPath)) + } + opts = append(opts, extraOpts...) + syncer, err := NewSyncer(ctx, mc, opts...) + require.NoError(t, err) + require.NoError(t, syncer.Sync(ctx)) + require.NoError(t, syncer.Close(ctx)) +} + +// workerCounts are the worker configurations every source-cache scenario +// runs under: 0 (sequential) and 4 (parallel). Parallel sync + replay +// shares one engine across workers, so the scenarios double as races on +// concurrent Replay*/PutSourceCacheEntry batches and expansion arming. +var workerCounts = []int{0, 4} + +func workerSyncOpts(workers int) []SyncOpt { + if workers > 0 { + return []SyncOpt{WithWorkerCount(workers)} + } + return nil +} + +// listGrantsInFile reopens a finished Pebble c1z and returns its grants +// keyed by grant ID. +func listGrantsInFile(ctx context.Context, t *testing.T, path string) map[string]*v2.Grant { + t.Helper() + reopen, err := dotc1z.NewStore(ctx, path, + dotc1z.WithEngine(c1zstore.EnginePebble), + dotc1z.WithReadOnly(true), + ) + require.NoError(t, err) + defer func() { _ = reopen.Close(ctx) }() + + sync, err := reopen.SyncMeta().LatestFullSync(ctx) + require.NoError(t, err) + require.NotNil(t, sync) + require.NoError(t, reopen.SetCurrentSync(ctx, sync.ID)) + + out := map[string]*v2.Grant{} + pageToken := "" + for { + resp, err := reopen.ListGrants(ctx, v2.GrantsServiceListGrantsRequest_builder{PageToken: pageToken}.Build()) + require.NoError(t, err) + for _, g := range resp.GetList() { + out[g.GetId()] = g + } + pageToken = resp.GetNextPageToken() + if pageToken == "" { + break + } + } + return out +} + +func sourceCacheTestFixtures(t *testing.T) (*v2.Resource, *v2.Entitlement, *v2.Resource, *v2.Resource) { + t.Helper() + group, err := rs.NewGroupResource("g1", groupResourceType, "g1", nil) + require.NoError(t, err) + ent := et.NewAssignmentEntitlement(group, "member", et.WithGrantableTo(groupResourceType, userResourceType)) + ent.SetSlug("member") + alice, err := rs.NewUserResource("alice", userResourceType, "alice", nil, rs.WithAnnotation(&v2.SkipEntitlementsAndGrants{})) + require.NoError(t, err) + bob, err := rs.NewUserResource("bob", userResourceType, "bob", nil, rs.WithAnnotation(&v2.SkipEntitlementsAndGrants{})) + require.NoError(t, err) + return group, ent, alice, bob +} + +// TestSourceCache_ConditionalRequestReplay is the GitHub-shape end-to-end +// test: sync 1 stamps rows and records the scope's etag; sync 2's lookup +// hits, the connector answers with a 304-style replay, and the grants are +// carried into the new c1z without the connector emitting any rows. +func TestSourceCache_ConditionalRequestReplay(t *testing.T) { + for _, workers := range workerCounts { + t.Run(fmt.Sprintf("workers=%d", workers), func(t *testing.T) { + runConditionalRequestReplayScenario(t, workerSyncOpts(workers)) + }) + } +} + +func runConditionalRequestReplayScenario(t *testing.T, workerOpts []SyncOpt) { + ctx, err := logging.Init(t.Context()) + require.NoError(t, err) + tmpDir := t.TempDir() + + group, ent, alice, bob := sourceCacheTestFixtures(t) + g1 := gt.NewGrant(group, "member", alice) + g2 := gt.NewGrant(group, "member", bob) + + mc := newSourceCacheMockConnector() + mc.AddResource(ctx, group) + mc.AddResource(ctx, alice) + mc.AddResource(ctx, bob) + mc.entDB[group.GetId().GetResource()] = []*v2.Entitlement{ent} + mc.grantDB[group.GetId().GetResource()] = []*v2.Grant{g1, g2} + mc.etagByResource[group.GetId().GetResource()] = `W/"etag-v1"` + + sync1 := filepath.Join(tmpDir, "sync1.c1z") + sync2 := filepath.Join(tmpDir, "sync2.c1z") + + runSourceCacheSync(ctx, t, mc, sync1, "", tmpDir, workerOpts...) + require.Zero(t, mc.lookupHits, "sync 1 has no previous sync to hit") + + // Upstream unchanged: sync 2 must replay. + runSourceCacheSync(ctx, t, mc, sync2, sync1, tmpDir, workerOpts...) + require.NotZero(t, mc.lookupHits, "sync 2's lookup must hit the etag persisted by sync 1") + + grants := listGrantsInFile(ctx, t, sync2) + require.Contains(t, grants, g1.GetId()) + require.Contains(t, grants, g2.GetId()) + + // And sync 2's file is usable as the NEXT previous sync: manifest + // entry + stamped rows were carried forward (a third sync replays too). + sync3 := filepath.Join(tmpDir, "sync3.c1z") + hitsBefore := mc.lookupHits + runSourceCacheSync(ctx, t, mc, sync3, sync2, tmpDir, workerOpts...) + require.Greater(t, mc.lookupHits, hitsBefore, "a replay-only sync must remain usable as the next replay source") + grants3 := listGrantsInFile(ctx, t, sync3) + require.Contains(t, grants3, g1.GetId()) + require.Contains(t, grants3, g2.GetId()) +} + +// TestSourceCache_DeltaOverlayAndDeletes is the Microsoft Graph-shape +// test: the second sync replays the base, upserts changed rows on top, +// applies @removed tombstones, and rotates the token. +func TestSourceCache_DeltaOverlayAndDeletes(t *testing.T) { + for _, workers := range workerCounts { + t.Run(fmt.Sprintf("workers=%d", workers), func(t *testing.T) { + runDeltaOverlayAndDeletesScenario(t, workerSyncOpts(workers)) + }) + } +} + +func runDeltaOverlayAndDeletesScenario(t *testing.T, workerOpts []SyncOpt) { + ctx, err := logging.Init(t.Context()) + require.NoError(t, err) + tmpDir := t.TempDir() + + group, ent, alice, bob := sourceCacheTestFixtures(t) + carol, err := rs.NewUserResource("carol", userResourceType, "carol", nil, rs.WithAnnotation(&v2.SkipEntitlementsAndGrants{})) + require.NoError(t, err) + + g1 := gt.NewGrant(group, "member", alice) + g2 := gt.NewGrant(group, "member", bob) + g3 := gt.NewGrant(group, "member", carol) + + mc := newSourceCacheMockConnector() + mc.AddResource(ctx, group) + mc.AddResource(ctx, alice) + mc.AddResource(ctx, bob) + mc.AddResource(ctx, carol) + mc.entDB[group.GetId().GetResource()] = []*v2.Entitlement{ent} + mc.grantDB[group.GetId().GetResource()] = []*v2.Grant{g1, g2} + mc.etagByResource[group.GetId().GetResource()] = "delta-token-1" + + sync1 := filepath.Join(tmpDir, "sync1.c1z") + sync2 := filepath.Join(tmpDir, "sync2.c1z") + + runSourceCacheSync(ctx, t, mc, sync1, "", tmpDir, workerOpts...) + + // Delta round: bob removed (canonical-id tombstone on the replay + // annotation), carol added, token rotates. + mc.mu.Lock() + mc.deltaMode = true + mc.etagByResource[group.GetId().GetResource()] = "delta-token-2" + mc.deltaAdds[group.GetId().GetResource()] = []*v2.Grant{g3} + mc.deltaDeletedIDs[group.GetId().GetResource()] = []string{g2.GetId()} + mc.mu.Unlock() + + runSourceCacheSync(ctx, t, mc, sync2, sync1, tmpDir, workerOpts...) + require.NotZero(t, mc.lookupHits) + + grants := listGrantsInFile(ctx, t, sync2) + require.Contains(t, grants, g1.GetId(), "replayed base row must survive") + require.Contains(t, grants, g3.GetId(), "overlay row must be upserted") + require.NotContains(t, grants, g2.GetId(), "tombstoned row must be deleted") + + // Second delta round: carol removed by BARE PRINCIPAL ID via the + // scope annotation — no principal type, no canonical-id + // reconstruction (the Entra @removed shape). + sync3 := filepath.Join(tmpDir, "sync3.c1z") + mc.mu.Lock() + mc.etagByResource[group.GetId().GetResource()] = "delta-token-3" + mc.deltaAdds[group.GetId().GetResource()] = nil + mc.deltaDeletedIDs[group.GetId().GetResource()] = nil + mc.deltaDeletedPrincipals[group.GetId().GetResource()] = []string{"carol"} + mc.mu.Unlock() + + runSourceCacheSync(ctx, t, mc, sync3, sync2, tmpDir, workerOpts...) + + grants3 := listGrantsInFile(ctx, t, sync3) + require.Contains(t, grants3, g1.GetId(), "alice's replayed row must survive the principal tombstone") + require.NotContains(t, grants3, g3.GetId(), "carol's row must die from a bare principal-id tombstone") + require.NotContains(t, grants3, g2.GetId()) +} + +// TestSourceCache_ReplayArmsExpansion pins the expansion-arming contract: +// a sync in which EVERY grants page replays must still run grant +// expansion, because replayed rows carry needs_expansion but never pass +// GrantExpandable-annotated rows through the syncer's connector-response +// path (the only other thing that arms the expansion phase). +func TestSourceCache_ReplayArmsExpansion(t *testing.T) { + for _, workers := range workerCounts { + t.Run(fmt.Sprintf("workers=%d", workers), func(t *testing.T) { + runReplayArmsExpansionScenario(t, workerSyncOpts(workers)) + }) + } +} + +func runReplayArmsExpansionScenario(t *testing.T, workerOpts []SyncOpt) { + ctx, err := logging.Init(t.Context()) + require.NoError(t, err) + tmpDir := t.TempDir() + + mc := newSourceCacheMockConnector() + mc.rtDB = []*v2.ResourceType{groupResourceType, userResourceType} + + // Standard expansion topology: alice is a member of group2, and + // group2 is an expandable member of group1 — expansion derives + // alice's grant on group1's member entitlement. + group1, group1Ent, err := mc.AddGroup(ctx, "group_1") + require.NoError(t, err) + group2, group2Ent, err := mc.AddGroup(ctx, "group_2") + require.NoError(t, err) + alice, err := mc.AddUser(ctx, "alice") + require.NoError(t, err) + _ = mc.AddGroupMember(ctx, group2, alice) + _ = mc.AddGroupMember(ctx, group1, group2, group2Ent) + + mc.etagByResource[group1.GetId().GetResource()] = "etag-g1" + mc.etagByResource[group2.GetId().GetResource()] = "etag-g2" + + derivedGrantID := gt.NewGrant(group1, "member", alice).GetId() + + sync1 := filepath.Join(tmpDir, "sync1.c1z") + sync2 := filepath.Join(tmpDir, "sync2.c1z") + + runSourceCacheSync(ctx, t, mc, sync1, "", tmpDir, workerOpts...) + require.Contains(t, listGrantsInFile(ctx, t, sync1), derivedGrantID, + "sanity: sync 1's expansion must derive alice's group1 grant") + + // Sync 2 replays every grants page (upstream unchanged). Derived + // grants are never replayed (they carry no scope stamp), so if replay + // fails to arm expansion, SyncGrantExpansion is skipped and the + // derived grant vanishes from sync 2. + runSourceCacheSync(ctx, t, mc, sync2, sync1, tmpDir, workerOpts...) + require.NotZero(t, mc.lookupHits, "sanity: sync 2 must have replayed") + + grants2 := listGrantsInFile(ctx, t, sync2) + baseGrantID := gt.NewGrant(group1, "member", group2).GetId() + require.Contains(t, grants2, baseGrantID, "replayed expandable base grant missing") + require.Contains(t, grants2, derivedGrantID, + "sync 2 must run grant expansion over replayed rows: the derived grant is missing, "+ + "meaning SetNeedsExpansion was never armed by the replay path") + _ = group1Ent +} + +// TestSourceCache_ReplayWhileDegradedFails pins the fail-loud half of the +// error contract: a replay annotation arriving with no usable previous +// sync is a hard sync error, never a silent fallback (the connector has +// already skipped row generation). +func TestSourceCache_ReplayWhileDegradedFails(t *testing.T) { + ctx, err := logging.Init(t.Context()) + require.NoError(t, err) + tmpDir := t.TempDir() + + group, ent, alice, _ := sourceCacheTestFixtures(t) + g1 := gt.NewGrant(group, "member", alice) + + mc := newSourceCacheMockConnector() + mc.AddResource(ctx, group) + mc.AddResource(ctx, alice) + mc.entDB[group.GetId().GetResource()] = []*v2.Entitlement{ent} + mc.grantDB[group.GetId().GetResource()] = []*v2.Grant{g1} + mc.etagByResource[group.GetId().GetResource()] = "etag-v1" + + // Misbehaving connector: fabricate a lookup hit so it emits a replay + // annotation even though the syncer never installed a lookup with a + // previous source. + mc.SetSourceCache(ctx, staticLookup{etag: "etag-v1"}) + + store, err := dotc1z.NewStore(ctx, filepath.Join(tmpDir, "out.c1z"), + dotc1z.WithEngine(c1zstore.EnginePebble), + dotc1z.WithTmpDir(tmpDir), + ) + require.NoError(t, err) + syncer, err := NewSyncer(ctx, °radedLookupConnector{mc}, WithConnectorStore(store), WithTmpDir(tmpDir)) + require.NoError(t, err) + err = syncer.Sync(ctx) + require.Error(t, err, "replay with no previous sync must fail the sync loudly") + require.Contains(t, err.Error(), "source cache") + require.NoError(t, syncer.Close(ctx)) +} + +// stubSourceCacheStore fakes the store surface begin/finish touch, so the +// stats bookkeeping can be pinned without a real engine. +type stubSourceCacheStore struct { + dotc1z.SourceCacheStore + replayRows int64 + stamped []string +} + +func (s *stubSourceCacheStore) LookupSourceCacheEntry(context.Context, sourcecache.RowKind, string) (sourcecache.Entry, bool, error) { + return sourcecache.Entry{CacheValidator: "prev-token"}, true, nil +} + +func (s *stubSourceCacheStore) ReplaySourceCache(context.Context, connectorstore.Reader, sourcecache.RowKind, string) (dotc1z.SourceCacheReplayResult, error) { + return dotc1z.SourceCacheReplayResult{Rows: s.replayRows}, nil +} + +func (s *stubSourceCacheStore) PutSourceCacheEntry(_ context.Context, _ sourcecache.RowKind, scopeKey, _ string) error { + s.stamped = append(s.stamped, scopeKey) + return nil +} + +// TestSourceCacheStats_OverlayRoundBooksOneHit pins two stats contracts: +// +// 1. Replay counters are recorded at FINISH, not begin — a page re-run +// after failing between replay and finish must not double-count. +// 2. A multi-page overlay round, whose rotated validator legally arrives +// on the FINAL page's scope annotation (no replay annotation there), +// books exactly one ScopesReplayed and zero ScopesStamped — not one +// of each, which would read a 100%-warm delta sync as 50%. +func TestSourceCacheStats_OverlayRoundBooksOneHit(t *testing.T) { + ctx, err := logging.Init(context.Background()) + require.NoError(t, err) + + store := &stubSourceCacheStore{replayRows: 2} + s := &syncer{ + state: newState(), + sourceCache: syncerSourceCache{ + enabled: true, + current: store, + prev: store, + replayedScopes: &replayedScopeSet{}, + }, + } + scope := sourcecache.HashScope("delta://groups") + + // Page 1: overlay replay, no etag yet (the delta token rotates on the + // final page). One overlay row rides along. + _, page1, err := s.beginSourceCachePage(ctx, sourcecache.RowKindGrants, + annotations.New(v2.SourceCacheReplay_builder{ScopeKey: scope, Overlay: true}.Build()), 1) + require.NoError(t, err) + require.True(t, page1.replayed) + + if stats := s.state.SourceCacheStatsSnapshot(); stats != nil { + require.Zero(t, stats.ScopesReplayed, "replay counters must not be recorded at begin (re-run double-count)") + } + + require.NoError(t, s.finishSourceCachePage(ctx, page1)) + + // Final page of the round: scope annotation with the NEW token, no + // replay annotation — this is a stamp write, but of the round's own + // scope, not a cold miss. + _, page2, err := s.beginSourceCachePage(ctx, sourcecache.RowKindGrants, + annotations.New(v2.SourceCacheRecord_builder{ScopeKey: scope, CacheValidator: "delta-token-2"}.Build()), 0) + require.NoError(t, err) + require.False(t, page2.replayed) + require.NoError(t, s.finishSourceCachePage(ctx, page2)) + require.Equal(t, []string{scope}, store.stamped, "the rotated validator must still be persisted") + + // A genuinely cold scope still counts as stamped. + coldScope := sourcecache.HashScope("delta://users") + _, page3, err := s.beginSourceCachePage(ctx, sourcecache.RowKindGrants, + annotations.New(v2.SourceCacheRecord_builder{ScopeKey: coldScope, CacheValidator: "etag-1"}.Build()), 3) + require.NoError(t, err) + require.NoError(t, s.finishSourceCachePage(ctx, page3)) + + stats := s.state.SourceCacheStatsSnapshot() + require.NotNil(t, stats) + require.Equal(t, int64(1), stats.ScopesReplayed, "one warm round, one hit") + require.Equal(t, int64(1), stats.ScopesStamped, "only the cold scope counts as stamped") + require.Equal(t, int64(2), stats.RowsReplayed[sourcecache.RowKindGrants]) + require.Equal(t, int64(1), stats.OverlayRows) +} + +// staticLookup always hits with a fixed etag — used to simulate a +// connector that violates the only-replay-what-you-looked-up invariant. +type staticLookup struct{ etag string } + +func (s staticLookup) Lookup(context.Context, sourcecache.RowKind, string) (sourcecache.Entry, bool, error) { + return sourcecache.Entry{CacheValidator: s.etag}, true, nil +} + +// degradedLookupConnector shadows SetSourceCache so the syncer cannot +// overwrite the fabricated lookup: the connector keeps "hitting" while the +// syncer's source-cache read side is degraded. +type degradedLookupConnector struct { + *sourceCacheMockConnector +} + +func (d *degradedLookupConnector) SetSourceCache(context.Context, sourcecache.Lookup) { + // Deliberately ignore the syncer's install; the mock keeps its + // fabricated staticLookup. +} diff --git a/pkg/sync/state.go b/pkg/sync/state.go index f1e3cc08c..c118fc30d 100644 --- a/pkg/sync/state.go +++ b/pkg/sync/state.go @@ -6,6 +6,7 @@ import ( "fmt" "slices" "sync" + "sync/atomic" "time" "github.com/conductorone/baton-sdk/pkg/sync/expand" @@ -46,6 +47,13 @@ type State interface { RecordSessionOp(op string, duration time.Duration, opErr error, timedOut bool) MergeSessionStat(op string, add SessionStoreStat) SessionStoreStats() map[string]SessionStoreStat + // AddSourceCacheStats merges delta into the sync's source-cache + // counters (replay effectiveness, tombstones, spawn fan-out, lookup + // continuation). Token-persisted, so a resumed sync keeps its counts. + AddSourceCacheStats(delta SourceCacheStats) + // SourceCacheStatsSnapshot returns a copy of the accumulated + // source-cache counters, or nil when nothing was recorded. + SourceCacheStatsSnapshot() *SourceCacheStats // CheckAndSetExclusionGroupResourceType records that exclusionGroupID is // used on resourceTypeID. If the group was already recorded against a // different resource type, the prior value is returned with conflict=true @@ -203,29 +211,50 @@ const ( // Action stores the current operation, page token, and optional fields for which resource is being worked with. type Action struct { - ID string `json:"id,omitempty"` - Op ActionOp `json:"operation,omitempty"` - PageToken string `json:"page_token,omitempty"` - ResourceTypeID string `json:"resource_type_id,omitempty"` - ResourceID string `json:"resource_id,omitempty"` - ParentResourceTypeID string `json:"parent_resource_type_id,omitempty"` - ParentResourceID string `json:"parent_resource_id,omitempty"` + ID string `json:"id,omitempty"` + Op ActionOp `json:"operation,omitempty"` + PageToken string `json:"page_token,omitempty"` + ResourceTypeID string `json:"resource_type_id,omitempty"` + // ResourceID == "" on a SyncGrantsOp / SyncEntitlementsOp action is + // the TYPE-SCOPED discriminator: the action enumerates the whole + // resource type (TypeScopedGrants / TypeScopedEntitlements) instead + // of one resource. Deliberately not a separate field — actions are + // JSON-serialized into checkpointed sync tokens, and the planners + // never emit an empty ResourceID for those ops otherwise, so the + // sentinel is unambiguous and keeps old tokens decoding unchanged. + ResourceID string `json:"resource_id,omitempty"` + ParentResourceTypeID string `json:"parent_resource_type_id,omitempty"` + ParentResourceID string `json:"parent_resource_id,omitempty"` + + // Spawned marks an action enqueued by a connector's EnqueuePageTokens + // annotation (a sibling cursor) rather than by the syncer's own + // planners. Progress accounting uses it: per-resource grant / + // entitlement coverage counts a resource once, when its ORIGIN + // action's chain ends — spawned siblings (and their NextPage + // continuations, which inherit the action) don't count, or one + // resource would count N times. + // omitempty keeps old checkpointed sync tokens decoding unchanged. + Spawned bool `json:"spawned,omitempty"` } var _ State = &state{} // state is an object used for tracking the current status of a connector sync. It operates like a stack. type state struct { - mtx sync.RWMutex - actions map[string]Action - actionOrder []string - currentActionID uint64 // Counter for generating new action IDs. - entitlementGraph *expand.EntitlementGraph - needsExpansion bool - hasExternalResourceGrants bool - shouldFetchRelatedResources bool - shouldSkipEntitlementsAndGrants bool - shouldSkipGrants bool + mtx sync.RWMutex + actions map[string]Action + actionOrder []string + currentActionID uint64 // Counter for generating new action IDs. + entitlementGraph *expand.EntitlementGraph + // Monotone set-once flags, written concurrently by parallel sync + // workers (grant pages arm needsExpansion / hasExternalResourceGrants + // from any worker goroutine) — atomic, or -race flags every parallel + // sync. Set-once semantics make the ordering irrelevant. + needsExpansion atomic.Bool + hasExternalResourceGrants atomic.Bool + shouldFetchRelatedResources atomic.Bool + shouldSkipEntitlementsAndGrants atomic.Bool + shouldSkipGrants atomic.Bool completedActionsCount uint64 exclusionGroupResourceTypes map[string]string exclusionGroupDefaults map[string]string @@ -233,6 +262,7 @@ type state struct { stepDurationsMs map[string]int64 connectorCallStats map[string]*ConnectorCallStat sessionStoreStats map[string]*SessionStoreStat + sourceCacheStats *SourceCacheStats // compaction is provenance written by the sync compactor via // BuildCompactedToken; the syncer itself never sets it. Kept on the // state so Unmarshal→Marshal round trips (e.g. expansion replay @@ -291,6 +321,7 @@ type serializedTokenV1 struct { StepDurationsMs map[string]int64 `json:"step_durations_ms,omitempty"` ConnectorCallStats map[string]*ConnectorCallStat `json:"connector_call_stats,omitempty"` SessionStoreStats map[string]*SessionStoreStat `json:"session_store_stats,omitempty"` + SourceCacheStats *SourceCacheStats `json:"source_cache_stats,omitempty"` Compaction *CompactionTokenStats `json:"compaction,omitempty"` Version uint64 `json:"version"` } @@ -301,7 +332,6 @@ func newState() *state { actionOrder: []string{}, currentActionID: 0, entitlementGraph: nil, - needsExpansion: false, exclusionGroupResourceTypes: make(map[string]string), exclusionGroupDefaults: make(map[string]string), exclusionGroupCounts: make(map[string]uint32), @@ -425,12 +455,12 @@ func (st *state) Unmarshal(input string) error { st.actionOrder = []string{} } st.currentActionID = token.CurrentActionID - st.needsExpansion = token.NeedsExpansion + st.needsExpansion.Store(token.NeedsExpansion) st.entitlementGraph = token.EntitlementGraph - st.hasExternalResourceGrants = token.HasExternalResourceGrants - st.shouldSkipEntitlementsAndGrants = token.ShouldSkipEntitlementsAndGrants - st.shouldSkipGrants = token.ShouldSkipGrants - st.shouldFetchRelatedResources = token.ShouldFetchRelatedResources + st.hasExternalResourceGrants.Store(token.HasExternalResourceGrants) + st.shouldSkipEntitlementsAndGrants.Store(token.ShouldSkipEntitlementsAndGrants) + st.shouldSkipGrants.Store(token.ShouldSkipGrants) + st.shouldFetchRelatedResources.Store(token.ShouldFetchRelatedResources) st.completedActionsCount = token.CompletedActionsCount st.exclusionGroupResourceTypes = token.ExclusionGroupResourceTypes if st.exclusionGroupResourceTypes == nil { @@ -456,6 +486,7 @@ func (st *state) Unmarshal(input string) error { if st.sessionStoreStats == nil { st.sessionStoreStats = make(map[string]*SessionStoreStat) } + st.sourceCacheStats = token.SourceCacheStats st.compaction = token.Compaction } else { st.actions = make(map[string]Action) @@ -476,6 +507,7 @@ func (st *state) Unmarshal(input string) error { st.stepDurationsMs = make(map[string]int64) st.connectorCallStats = make(map[string]*ConnectorCallStat) st.sessionStoreStats = make(map[string]*SessionStoreStat) + st.sourceCacheStats = nil st.compaction = nil } @@ -491,12 +523,12 @@ func (st *state) Marshal() (string, error) { ActionsMap: st.actions, ActionOrder: st.actionOrder, CurrentActionID: st.currentActionID, - NeedsExpansion: st.needsExpansion, + NeedsExpansion: st.needsExpansion.Load(), EntitlementGraph: st.entitlementGraph, - HasExternalResourceGrants: st.hasExternalResourceGrants, - ShouldFetchRelatedResources: st.shouldFetchRelatedResources, - ShouldSkipEntitlementsAndGrants: st.shouldSkipEntitlementsAndGrants, - ShouldSkipGrants: st.shouldSkipGrants, + HasExternalResourceGrants: st.hasExternalResourceGrants.Load(), + ShouldFetchRelatedResources: st.shouldFetchRelatedResources.Load(), + ShouldSkipEntitlementsAndGrants: st.shouldSkipEntitlementsAndGrants.Load(), + ShouldSkipGrants: st.shouldSkipGrants.Load(), CompletedActionsCount: st.completedActionsCount, ExclusionGroupResourceTypes: st.exclusionGroupResourceTypes, ExclusionGroupDefaults: st.exclusionGroupDefaults, @@ -504,6 +536,7 @@ func (st *state) Marshal() (string, error) { StepDurationsMs: st.stepDurationsMs, ConnectorCallStats: st.connectorCallStats, SessionStoreStats: st.sessionStoreStats, + SourceCacheStats: st.sourceCacheStats, Compaction: st.compaction, Version: 1, }) @@ -695,43 +728,43 @@ func (st *state) NextPage(ctx context.Context, actionID string, pageToken string } func (st *state) NeedsExpansion() bool { - return st.needsExpansion + return st.needsExpansion.Load() } func (st *state) SetNeedsExpansion() { - st.needsExpansion = true + st.needsExpansion.Store(true) } func (st *state) HasExternalResourcesGrants() bool { - return st.hasExternalResourceGrants + return st.hasExternalResourceGrants.Load() } func (st *state) SetHasExternalResourcesGrants() { - st.hasExternalResourceGrants = true + st.hasExternalResourceGrants.Store(true) } func (st *state) ShouldFetchRelatedResources() bool { - return st.shouldFetchRelatedResources + return st.shouldFetchRelatedResources.Load() } func (st *state) SetShouldFetchRelatedResources() { - st.shouldFetchRelatedResources = true + st.shouldFetchRelatedResources.Store(true) } func (st *state) ShouldSkipEntitlementsAndGrants() bool { - return st.shouldSkipEntitlementsAndGrants + return st.shouldSkipEntitlementsAndGrants.Load() } func (st *state) SetShouldSkipEntitlementsAndGrants() { - st.shouldSkipEntitlementsAndGrants = true + st.shouldSkipEntitlementsAndGrants.Store(true) } func (st *state) ShouldSkipGrants() bool { - return st.shouldSkipGrants + return st.shouldSkipGrants.Load() } func (st *state) SetShouldSkipGrants() { - st.shouldSkipGrants = true + st.shouldSkipGrants.Store(true) } // EntitlementGraph returns the entitlement graph for the current action. diff --git a/pkg/sync/syncer.go b/pkg/sync/syncer.go index 39e7adc88..2d5530832 100644 --- a/pkg/sync/syncer.go +++ b/pkg/sync/syncer.go @@ -20,6 +20,7 @@ import ( "github.com/conductorone/baton-sdk/pkg/bid" "github.com/conductorone/baton-sdk/pkg/dotc1z" "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" + "github.com/conductorone/baton-sdk/pkg/sourcecache" "github.com/conductorone/baton-sdk/pkg/sync/expand" "github.com/conductorone/baton-sdk/pkg/types/entitlement" batonGrant "github.com/conductorone/baton-sdk/pkg/types/grant" @@ -142,33 +143,61 @@ type syncer struct { store c1zstore.Store externalResourceReader connectorstore.Reader previousSyncReader connectorstore.Reader - connector types.ConnectorClient - state State - runDuration time.Duration - transitionHandler func(s Action) - progressHandler func(p *Progress) - tmpDir string - storageEngine c1zstore.Engine - skipFullSync bool - lastCheckPointTime time.Time - counts *progresslog.ProgressLog - targetedSyncResources []*v2.Resource - onlyExpandGrants bool - dontExpandGrants bool - syncID string - skipEGForResourceType syncMap[string, bool] - skipEntitlementsForResourceType syncMap[string, bool] - skipEntitlementsAndGrants bool - skipGrants bool - resourceTypeTraits syncMap[string, []v2.ResourceType_Trait] - syncType connectorstore.SyncType - injectSyncIDAnnotation bool - setSessionStore sessions.SetSessionStore - syncResourceTypes []string - workerCount int // If 1, sync is sequential (default). If > 1, sync operations are done in parallel. - metricsHandler metrics.Handler - syncIdentity uotel.SyncIdentity - recordStats bool + // Ingestion-invariant state (see ingest_invariants.go): + // childSchedule is the monotone record backing invariant I4; + // resourcesPhaseRanHere gates I4 to processes that actually ran the + // resources phase; failFastInvariants promotes every invariant + // verdict to a hard, plainly-attributed sync failure — tolerated + // warns fail, drop arms fail before dropping, repair arms are + // skipped so the underlying bug is named — and enables I4 (skipped + // entirely in default mode). Tests and the equivalence harness set + // it; production default follows the per-invariant policy in + // ingest_invariants.go (drop/warn/repair/fail with the warm-cold + // ErrReplayIntegrity ladder on FAIL-class verdicts). + childSchedule childScheduleSet + resourcesPhaseRanHere bool + failFastInvariants bool + // expandDropStats aggregates expansion edges dropped over missing + // entitlements across the whole sync (see expand.DroppedEdgeStats); + // summarized once when expansion completes. + expandDropStats *expand.DroppedEdgeStats + // testSourceCacheHaltHook, when non-nil, fires at named seams of the + // scope lifecycle (replay-copied, rows-committed, tombstones-applied, + // manifest-written); returning an error fails the sync at exactly + // that boundary. The halt-point sweep (source_cache_halt_test.go) + // uses it to prove crash/resume equivalence at every + // ordering-sensitive point. Nil in production: one pointer check. + testSourceCacheHaltHook func(stage, scopeKey string) error + connector types.ConnectorClient + state State + runDuration time.Duration + transitionHandler func(s Action) + progressHandler func(p *Progress) + tmpDir string + storageEngine c1zstore.Engine + skipFullSync bool + lastCheckPointTime time.Time + counts *progresslog.ProgressLog + targetedSyncResources []*v2.Resource + onlyExpandGrants bool + dontExpandGrants bool + syncID string + skipEGForResourceType syncMap[string, bool] + skipEntitlementsForResourceType syncMap[string, bool] + typeScopedGrantsForResourceType syncMap[string, bool] + typeScopedEntitlementsForResourceType syncMap[string, bool] + skipEntitlementsAndGrants bool + skipGrants bool + resourceTypeTraits syncMap[string, []v2.ResourceType_Trait] + syncType connectorstore.SyncType + injectSyncIDAnnotation bool + setSessionStore sessions.SetSessionStore + syncResourceTypes []string + workerCount int // If 1, sync is sequential (default). If > 1, sync operations are done in parallel. + metricsHandler metrics.Handler + syncIdentity uotel.SyncIdentity + recordStats bool + sourceCache syncerSourceCache } var _ Syncer = (*syncer)(nil) @@ -496,6 +525,9 @@ func (s *syncer) syncSummaryFields(span trace.Span) []zap.Field { if logSession { fields = append(fields, zap.Any("session_store_stats", sessionForLog)) } + if scStats := s.state.SourceCacheStatsSnapshot(); scStats != nil { + fields = append(fields, zap.Any("source_cache_stats", scStats)) + } return fields } @@ -573,60 +605,6 @@ func (s *syncer) handleProgress(ctx context.Context, a *Action, c int) { // single exclusion_group_id. Phase 1 limit. const maxEntitlementsPerExclusionGroup = 50 -// recordEntitlementExclusionGroup enforces the invariants on an exclusion -// group membership: a given exclusion_group_id must stay within one resource -// type, a group may have at most one entitlement marked is_default, and a group -// may contain at most maxEntitlementsPerExclusionGroup entitlements. Empty -// group ids are treated as "no exclusion group" and skipped. -func (s *syncer) recordEntitlementExclusionGroup(eg *v2.EntitlementExclusionGroup, entitlementID, resourceTypeID string) error { - groupID := eg.GetExclusionGroupId() - if groupID == "" { - return nil - } - if existing, conflict := s.state.CheckAndSetExclusionGroupResourceType(groupID, resourceTypeID); conflict { - return fmt.Errorf("exclusion group %q is used on multiple resource types (%q and %q); "+ - "exclusion groups may span resources but must be scoped to a single resource type", - groupID, existing, resourceTypeID) - } - if eg.GetIsDefault() { - if existing, conflict := s.state.CheckAndSetExclusionGroupDefault(groupID, entitlementID); conflict { - return fmt.Errorf("exclusion group %q has multiple default entitlements (%q and %q); "+ - "at most one entitlement per exclusion group may set is_default=true", - groupID, existing, entitlementID) - } - } - if count := s.state.IncrementExclusionGroupCount(groupID); count > maxEntitlementsPerExclusionGroup { - return fmt.Errorf("exclusion group %q has too many entitlements (%d); "+ - "at most %d entitlements are allowed per exclusion group", - groupID, count, maxEntitlementsPerExclusionGroup) - } - return nil -} - -// validateEntitlementExclusionGroups picks the exclusion group annotation off -// each entitlement (if present) and forwards to recordEntitlementExclusionGroup. -// Use this on lists of entitlements that may independently carry exclusion -// group annotations (e.g., the dynamic ListEntitlements path); callers that -// already have the annotation in hand should call recordEntitlementExclusionGroup -// directly to avoid the per-entitlement Pick. -func (s *syncer) validateEntitlementExclusionGroups(ents []*v2.Entitlement) error { - for _, ent := range ents { - eg := &v2.EntitlementExclusionGroup{} - entAnnos := annotations.Annotations(ent.GetAnnotations()) - ok, err := entAnnos.Pick(eg) - if err != nil { - return err - } - if !ok { - continue - } - if err := s.recordEntitlementExclusionGroup(eg, ent.GetId(), ent.GetResource().GetId().GetResourceType()); err != nil { - return err - } - } - return nil -} - // nextPageOrFinishAction updates the action with the next page token, or if there is no next page, finishes the action. // It also pushes any child actions before updating/finishing the action. // This is useful for pagination, and for actions that create other actions. @@ -770,6 +748,11 @@ func (s *syncer) Sync(ctx context.Context) error { } } + err = s.configureSourceCache(ctx, resp) + if err != nil { + return err + } + syncResourceTypeMap := make(map[string]bool) if len(s.syncResourceTypes) > 0 { for _, rt := range s.syncResourceTypes { @@ -810,6 +793,16 @@ func (s *syncer) Sync(ctx context.Context) error { l.Debug("resuming previous sync", zap.String("sync_id", syncID)) } + if s.sourceCache.enabled { + // Persist the replay-compatibility key before any page can write + // a manifest entry (idempotent overwrite on resume): an artifact + // must never hold validators without the key that scopes their + // validity. See configureSourceCache. + if err := s.sourceCache.current.PutSourceCacheCompat(ctx, s.sourceCache.compatKey); err != nil { + return s.returnSyncError(l, span, fmt.Errorf("error recording source-cache replay-compatibility key: %w", err)) + } + } + currentStep, err := s.store.CurrentSyncStep(ctx) if err != nil { return err @@ -871,6 +864,24 @@ func (s *syncer) Sync(ctx context.Context) error { return s.returnSyncError(l, span, err) } + // Post-collection ingestion invariants (ingest_invariants.go): every + // ingestion path has finished writing, so the store-derived + // definitions of the sync's side effects are checkable. Runs before + // EndSync so a violating sync is never sealed as complete. + if err := s.runIngestionInvariants(ctx); err != nil { + return s.returnSyncError(l, span, err) + } + if s.testSourceCacheHaltHook != nil { + // The seam AFTER the invariants' drops/repairs committed and + // BEFORE the checkpoint/EndSync below: a resumed sync re-runs the + // whole invariant pass over already-dropped state, so drops must + // be idempotent and their manifest invalidations must already be + // durable (see stageSourceCacheScopeInvalidationLocked). + if err := s.testSourceCacheHaltHook("ingest-invariants-complete", ""); err != nil { + return s.returnSyncError(l, span, err) + } + } + // Force a checkpoint to clear completed actions & entitlement graph in sync_token. s.state.ClearEntitlementGraph(ctx) s.state.ClearExclusionGroupTracking(ctx) @@ -895,6 +906,7 @@ func (s *syncer) Sync(ctx context.Context) error { return s.returnSyncError(l, span, err) } + s.logSourceCacheStats(ctx) if s.recordStats { l.Info("Sync complete.", s.syncSummaryFields(span)...) } else { @@ -1087,11 +1099,7 @@ func (s *syncer) hasChildResources(resource *v2.Resource) bool { // At sync scale (100k+ resources per trace) the span overhead and trace bloat // outweighed any debugging value. func (s *syncer) getSubResources(ctx context.Context, parent *v2.Resource) error { - syncResourceTypeMap := make(map[string]bool) - for _, rt := range s.syncResourceTypes { - syncResourceTypeMap[rt] = true - } - + var childTypeIDs []string for _, a := range parent.GetAnnotations() { if a.MessageIs((*v2.ChildResourceType)(nil)) { crt := &v2.ChildResourceType{} @@ -1099,24 +1107,42 @@ func (s *syncer) getSubResources(ctx context.Context, parent *v2.Resource) error if err != nil { return err } - if len(s.syncResourceTypes) > 0 { - if shouldSync := syncResourceTypeMap[crt.GetResourceTypeId()]; !shouldSync { - continue - } - } - childAction := Action{ - Op: SyncResourcesOp, - ResourceTypeID: crt.GetResourceTypeId(), - ParentResourceID: parent.GetId().GetResource(), - ParentResourceTypeID: parent.GetId().GetResourceType(), - } - s.state.PushAction(ctx, childAction) + childTypeIDs = append(childTypeIDs, crt.GetResourceTypeId()) } } - + s.pushChildResourceActions(ctx, childTypeIDs, parent.GetId().GetResourceType(), parent.GetId().GetResource()) return nil } +// pushChildResourceActions queues child resource syncs for one parent, +// honoring the sync's resource-type filter. Shared by getSubResources +// (response rows) and the source-cache replay path, which must schedule +// children for replayed parents itself (a replayed page has no response +// rows for getSubResources to inspect). +func (s *syncer) pushChildResourceActions(ctx context.Context, childTypeIDs []string, parentTypeID, parentID string) { + for _, childTypeID := range childTypeIDs { + if len(s.syncResourceTypes) > 0 && !slices.Contains(s.syncResourceTypes, childTypeID) { + continue + } + // Monotone evidence for ingestion invariant I4 (see + // ingest_invariants.go), doubling as the per-sync scheduling + // dedupe: a parent discovered by BOTH the replay copy and a + // delta overlay re-emission (or re-emitted within a cold delta + // walk) schedules its children exactly once. In-memory, so a + // resumed process re-schedules — the existing at-least-once + // semantic, just not N-times-per-sync. + if !s.childSchedule.recordIfNew(childTypeID, parentTypeID, parentID) { + continue + } + s.state.PushAction(ctx, Action{ + Op: SyncResourcesOp, + ResourceTypeID: childTypeID, + ParentResourceID: parentID, + ParentResourceTypeID: parentTypeID, + }) + } +} + func (s *syncer) getResourceFromConnector(ctx context.Context, resourceID *v2.ResourceId, parentResourceID *v2.ResourceId) (*v2.Resource, error) { ctx, span := tracer.Start(ctx, "syncer.getResource") var err error @@ -1189,18 +1215,29 @@ func (s *syncer) SyncTargetedResource(ctx context.Context, action *Action) error s.state.FinishAction(ctx, action) - // Actions happen in reverse order. We want to sync child resources, then entitlements, then grants + // Actions happen in reverse order. We want to sync child resources, then entitlements, then grants. + // Type-scoped types are excluded from per-resource fan-out here as in + // the full-sync planners: SyncTargetedResource must not enqueue + // ListGrants/ListEntitlements for a resource whose type is annotated + // TypeScopedGrants / TypeScopedEntitlements (those paths use a single + // type-scoped action owned by the full phase planners). shouldSkipGrants, err := s.shouldSkipGrants(ctx, resource) if err != nil { return err } if !shouldSkipGrants { - s.state.PushAction(ctx, Action{ - Op: SyncGrantsOp, - ResourceTypeID: resourceTypeID, - ResourceID: resourceID, - }) + typeScopedGrants, err := s.resourceTypeHasTypeScopedGrants(ctx, resourceTypeID) + if err != nil { + return err + } + if !typeScopedGrants { + s.state.PushAction(ctx, Action{ + Op: SyncGrantsOp, + ResourceTypeID: resourceTypeID, + ResourceID: resourceID, + }) + } } shouldSkipEnts, err := s.shouldSkipEntitlements(ctx, resource) @@ -1209,11 +1246,17 @@ func (s *syncer) SyncTargetedResource(ctx context.Context, action *Action) error } if !shouldSkipEnts { - s.state.PushAction(ctx, Action{ - Op: SyncEntitlementsOp, - ResourceTypeID: resourceTypeID, - ResourceID: resourceID, - }) + typeScopedEnts, err := s.resourceTypeHasTypeScopedEntitlements(ctx, resourceTypeID) + if err != nil { + return err + } + if !typeScopedEnts { + s.state.PushAction(ctx, Action{ + Op: SyncEntitlementsOp, + ResourceTypeID: resourceTypeID, + ResourceID: resourceID, + }) + } } err = s.getSubResources(ctx, resource) @@ -1236,6 +1279,10 @@ func (s *syncer) SyncResources(ctx context.Context, action *Action) error { if action.PageToken == "" { ctxzap.Extract(ctx).Info("Syncing resources...") s.handleInitialActionForStep(ctx, *action) + // Ingestion invariant I4 is only verifiable when the + // resources phase started in THIS process (the scheduled + // set is in-memory); see ingest_invariants.go. + s.resourcesPhaseRanHere = true } resp, err := s.store.ListResourceTypes(ctx, v2.ResourceTypesServiceListResourceTypesRequest_builder{ @@ -1269,26 +1316,57 @@ func (s *syncer) SyncResources(ctx context.Context, action *Action) error { // owns a span — the duplicate inflated trace span counts without adding // information. func (s *syncer) syncResources(ctx context.Context, action *Action) error { - req := v2.ResourcesServiceListResourcesRequest_builder{ - ResourceTypeId: action.ResourceTypeID, - PageToken: action.PageToken, - ActiveSyncId: s.getActiveSyncID(), - }.Build() - if action.ParentResourceTypeID != "" && action.ParentResourceID != "" { - req.SetParentResourceId(v2.ResourceId_builder{ - ResourceType: action.ParentResourceTypeID, - Resource: action.ParentResourceID, - }.Build()) + var resp *v2.ResourcesServiceListResourcesResponse + err := s.withSourceCacheContinuation(ctx, "sync-resources", func(extra annotations.Annotations, attempt int) (listAttempt, error) { + req := v2.ResourcesServiceListResourcesRequest_builder{ + ResourceTypeId: action.ResourceTypeID, + PageToken: action.PageToken, + ActiveSyncId: s.getActiveSyncID(), + Annotations: extra, + }.Build() + if action.ParentResourceTypeID != "" && action.ParentResourceID != "" { + req.SetParentResourceId(v2.ResourceId_builder{ + ResourceType: action.ParentResourceTypeID, + Resource: action.ParentResourceID, + }.Build()) + } + start := time.Now() + r, err := s.connector.ListResources(ctx, req) + s.observeConnectorCall(ctx, continuationCallMethod("list-resources", attempt), start, action.ResourceTypeID, action.ResourceID) + if err != nil { + return listAttempt{}, err + } + s.recordSessionUsage(r.GetAnnotations()) + resp = r + return listAttempt{ + annos: annotations.Annotations(r.GetAnnotations()), + rows: len(r.GetList()), + nextToken: r.GetNextPageToken(), + }, nil + }) + if err != nil { + return err } - start := time.Now() - resp, err := s.connector.ListResources(ctx, req) - s.observeConnectorCall(ctx, "list-resources", start, action.ResourceTypeID, action.ResourceID) - s.recordSessionUsage(resp.GetAnnotations()) + putCtx, scPage, err := s.beginSourceCachePage(ctx, sourcecache.RowKindResources, annotations.Annotations(resp.GetAnnotations()), len(resp.GetList())) if err != nil { return err } + // On any source-cache-scoped page the "already synced this sync" + // dedupe below is wrong — scoped pages are upsert streams whose rows + // are always authoritative: + // - Replayed rounds copy the previous sync's rows in on the round's + // FIRST page, so an overlay row's identity ALWAYS hits the store, + // on every page of the round — but the stored row is the stale + // base and the overlay row is the update that must overwrite it. + // Keying off the replay annotation alone would only protect page + // one (the annotation fires once per round). + // - Cold delta enumerations may legally return the same object + // multiple times (changed mid-walk), later occurrences + // authoritative; deduping would keep the STALE first occurrence. + pageScoped := scPage != nil + bulkPutResoruces := []*v2.Resource{} for _, r := range resp.GetList() { validatedResource := false @@ -1305,7 +1383,7 @@ func (s *syncer) syncResources(ctx context.Context, action *Action) error { validatedResource = true // We must *ALSO* check if we have any child resources. - if !s.hasChildResources(r) { + if !pageScoped && !s.hasChildResources(r) { // Since we only have the resource type IDs of child resources, // we can't tell if we already have synced those child resources. // Those children may also have their own child resources, @@ -1334,12 +1412,16 @@ func (s *syncer) syncResources(ctx context.Context, action *Action) error { } if len(bulkPutResoruces) > 0 { - err = s.store.PutResources(ctx, bulkPutResoruces...) + err = s.store.PutResources(putCtx, bulkPutResoruces...) if err != nil { return err } } + if err := s.finishSourceCachePage(ctx, scPage); err != nil { + return err + } + s.handleProgress(ctx, action, len(resp.GetList())) s.counts.AddResources(action.ResourceTypeID, len(resp.GetList())) if resp.GetNextPageToken() == "" { @@ -1486,6 +1568,9 @@ func (s *syncer) shouldSkipEntitlements(ctx context.Context, r *v2.Resource) (bo // SyncEntitlements fetches the entitlements from the connector. It first lists each resource from the datastore, // and pushes an action to fetch the entitlements for each resource. +// Resource types annotated with TypeScopedEntitlements are excluded from the per-resource fan-out and +// get a single type-scoped action instead (empty ResourceID); the connector enumerates the whole +// type, optionally spawning additional cursors via the EnqueuePageTokens annotation. func (s *syncer) SyncEntitlements(ctx context.Context, action *Action) error { ctx, span := uotel.StartWithLink(ctx, tracer, "syncer.SyncEntitlements") uotel.SetSyncIdentityAttrs(ctx, span) @@ -1493,11 +1578,22 @@ func (s *syncer) SyncEntitlements(ctx context.Context, action *Action) error { defer func() { uotel.EndSpanWithError(span, err) }() if action.ResourceTypeID == "" && action.ResourceID == "" { + actions := make([]Action, 0) pageToken := action.PageToken if pageToken == "" { ctxzap.Extract(ctx).Info("Syncing entitlements...") s.handleInitialActionForStep(ctx, *action) + + // One type-scoped action per annotated resource type, enqueued + // exactly once (the planner's first page). + typeScoped, err := s.typeScopedEntitlementsResourceTypes(ctx) + if err != nil { + return fmt.Errorf("sync-entitlements: error listing type-scoped resource types: %w", err) + } + for _, rtID := range typeScoped { + actions = append(actions, Action{Op: SyncEntitlementsOp, ResourceTypeID: rtID}) + } } resp, err := s.store.ListResources(ctx, v2.ResourcesServiceListResourcesRequest_builder{ @@ -1508,7 +1604,6 @@ func (s *syncer) SyncEntitlements(ctx context.Context, action *Action) error { return err } - actions := make([]Action, 0) for _, r := range resp.GetList() { shouldSkipEntitlements, err := s.shouldSkipEntitlements(ctx, r) if err != nil { @@ -1517,6 +1612,17 @@ func (s *syncer) SyncEntitlements(ctx context.Context, action *Action) error { if shouldSkipEntitlements { continue } + + // Types with type-scoped entitlements are excluded from the + // per-resource fan-out; their single action is enqueued above. + typeScoped, err := s.resourceTypeHasTypeScopedEntitlements(ctx, r.GetId().GetResourceType()) + if err != nil { + return err + } + if typeScoped { + continue + } + actions = append(actions, Action{Op: SyncEntitlementsOp, ResourceID: r.GetId().GetResource(), ResourceTypeID: r.GetId().GetResourceType()}) } @@ -1531,48 +1637,118 @@ func (s *syncer) SyncEntitlements(ctx context.Context, action *Action) error { return nil } +// resourceTypeHasTypeScopedEntitlements reports (cached per sync) whether the +// resource type carries the TypeScopedEntitlements annotation. +func (s *syncer) resourceTypeHasTypeScopedEntitlements(ctx context.Context, resourceTypeID string) (bool, error) { + return s.resourceTypeCarries(ctx, resourceTypeID, &v2.TypeScopedEntitlements{}, &s.typeScopedEntitlementsForResourceType) +} + +// typeScopedEntitlementsResourceTypes lists every synced resource type annotated +// with TypeScopedEntitlements. +func (s *syncer) typeScopedEntitlementsResourceTypes(ctx context.Context) ([]string, error) { + return s.resourceTypesCarrying(ctx, &v2.TypeScopedEntitlements{}, &s.typeScopedEntitlementsForResourceType) +} + // syncEntitlementsForResource fetches the entitlements for a specific resource from the connector. +// An action with an empty ResourceID is a TYPE-SCOPED entitlements cursor: the connector +// enumerates entitlements for the whole resource type (no single resource backs the call), +// and may spawn sibling cursors via the EnqueuePageTokens response annotation. // No span here: only call site is SyncEntitlements, which already owns a span. func (s *syncer) syncEntitlementsForResource(ctx context.Context, action *Action) error { + typeScoped := action.ResourceID == "" resourceID := v2.ResourceId_builder{ ResourceType: action.ResourceTypeID, Resource: action.ResourceID, }.Build() - resourceResponse, err := s.store.GetResource(ctx, reader_v2.ResourcesReaderServiceGetResourceRequest_builder{ - ResourceId: resourceID, - }.Build()) - if err != nil { - return err - } - resource := resourceResponse.GetResource() + var resource *v2.Resource + var reqAnnos annotations.Annotations + if typeScoped { + resource, reqAnnos = typeScopedRequestStub(action.ResourceTypeID, &v2.TypeScopedEntitlements{}) + } else { + resourceResponse, err := s.store.GetResource(ctx, reader_v2.ResourcesReaderServiceGetResourceRequest_builder{ + ResourceId: resourceID, + }.Build()) + if err != nil { + return err + } + resource = resourceResponse.GetResource() + } - start := time.Now() - resp, err := s.connector.ListEntitlements(ctx, v2.EntitlementsServiceListEntitlementsRequest_builder{ - Resource: resource, - PageToken: action.PageToken, - ActiveSyncId: s.getActiveSyncID(), - }.Build()) - s.observeConnectorCall(ctx, "list-entitlements", start, action.ResourceTypeID, action.ResourceID) - s.recordSessionUsage(resp.GetAnnotations()) + var resp *v2.EntitlementsServiceListEntitlementsResponse + err := s.withSourceCacheContinuation(ctx, "sync-entitlements", func(extra annotations.Annotations, attempt int) (listAttempt, error) { + annos := make(annotations.Annotations, 0, len(reqAnnos)+len(extra)) + annos = append(annos, reqAnnos...) + annos = append(annos, extra...) + start := time.Now() + r, err := s.connector.ListEntitlements(ctx, v2.EntitlementsServiceListEntitlementsRequest_builder{ + Resource: resource, + PageToken: action.PageToken, + ActiveSyncId: s.getActiveSyncID(), + Annotations: annos, + }.Build()) + s.observeConnectorCall(ctx, continuationCallMethod("list-entitlements", attempt), start, action.ResourceTypeID, action.ResourceID) + if err != nil { + return listAttempt{}, err + } + s.recordSessionUsage(r.GetAnnotations()) + resp = r + return listAttempt{ + annos: annotations.Annotations(r.GetAnnotations()), + rows: len(r.GetList()), + nextToken: r.GetNextPageToken(), + }, nil + }) if err != nil { return err } - if err := s.validateEntitlementExclusionGroups(resp.GetList()); err != nil { + respAnnos := annotations.Annotations(resp.GetAnnotations()) + putCtx, scPage, err := s.beginSourceCachePage(ctx, sourcecache.RowKindEntitlements, respAnnos, len(resp.GetList())) + if err != nil { return err } - err = s.store.PutEntitlements(ctx, resp.GetList()...) + + err = s.store.PutEntitlements(putCtx, resp.GetList()...) if err != nil { return err } + if err := s.finishSourceCachePage(ctx, scPage); err != nil { + return err + } + s.handleProgress(ctx, action, len(resp.GetList())) - if resp.GetNextPageToken() == "" { - s.counts.AddEntitlementsProgress(resourceID.ResourceType, 1) + + if typeScoped { + // Cursors don't map 1:1 to resources, so the per-resource + // "N of M resources covered" accounting is meaningless here. + // Row progress counts response rows PLUS replayed rows: a warm + // 304 page carries zero response rows while landing the scope's + // whole replayed row set, and counting only the former reported + // near-zero progress for healthy warm syncs. + rows := len(resp.GetList()) + if scPage != nil { + rows += int(scPage.replayRows) + } + s.counts.SetEntitlementsCountOnly(resourceID.GetResourceType()) + s.counts.AddEntitlementsProgress(resourceID.GetResourceType(), rows) + s.counts.LogEntitlementsProgress(ctx, resourceID.GetResourceType()) + } else if resp.GetNextPageToken() == "" && !action.Spawned { + // A resource counts as covered exactly once: when its ORIGIN + // action's page chain ends. Spawned sibling cursors must not count. + s.counts.AddEntitlementsProgress(resourceID.GetResourceType(), 1) s.counts.LogEntitlementsProgress(ctx, resourceID.GetResourceType()) } - return s.nextPageOrFinishAction(ctx, action, resp.GetNextPageToken()) + // EnqueuePageTokens: enqueue sibling cursors (type-scoped shards or + // per-resource parallel pages). Shared contract: see + // collectEnqueuedPageTokens (type_scoped.go). + spawned, err := s.collectEnqueuedPageTokens(ctx, "sync-entitlements", SyncEntitlementsOp, action, respAnnos) + if err != nil { + return err + } + + return s.nextPageOrFinishAction(ctx, action, resp.GetNextPageToken(), spawned...) } func (s *syncer) SyncStaticEntitlements(ctx context.Context, action *Action) error { @@ -1665,11 +1841,6 @@ func (s *syncer) syncStaticEntitlementsForResourceType(ctx context.Context, acti } entID := entitlement.NewEntitlementID(resource, ent.GetSlug()) - if hasExclusionGroup { - if err := s.recordEntitlementExclusionGroup(exclusionGroup, entID, resource.GetId().GetResourceType()); err != nil { - return err - } - } entitlements = append(entitlements, &v2.Entitlement{ Resource: resource, @@ -1920,6 +2091,13 @@ func (s *syncer) loadEntitlementGraph(ctx context.Context, action *Action, graph // Only skip not-found entitlements; propagate other errors // to avoid silently dropping edges and yielding incorrect expansions. if status.Code(err) == codes.NotFound { + // Counted on the same sync-wide aggregate as the + // expander's drops (this seam was previously + // Debug-only — invisible in production). + if s.expandDropStats == nil { + s.expandDropStats = &expand.DroppedEdgeStats{} + } + s.expandDropStats.RecordSourceMissing(srcEntitlementID) l.Debug("source entitlement not found, skipping edge", zap.String("src_entitlement_id", srcEntitlementID), zap.String("dst_entitlement_id", dstEntitlementID), @@ -1999,6 +2177,9 @@ func (s *syncer) fixEntitlementGraphCycles(ctx context.Context, graph *expand.En // SyncGrants fetches the grants for each resource from the connector. It iterates each resource // from the datastore, and pushes a new action to sync the grants for each individual resource. +// Resource types annotated with TypeScopedGrants are excluded from the per-resource fan-out and +// get a single type-scoped action instead (empty ResourceID); the connector enumerates the whole +// type, optionally spawning additional cursors via the EnqueuePageTokens annotation. func (s *syncer) SyncGrants(ctx context.Context, action *Action) error { ctx, span := uotel.StartWithLink(ctx, tracer, "syncer.SyncGrants") uotel.SetSyncIdentityAttrs(ctx, span) @@ -2006,9 +2187,20 @@ func (s *syncer) SyncGrants(ctx context.Context, action *Action) error { defer func() { uotel.EndSpanWithError(span, err) }() if action.ResourceTypeID == "" && action.ResourceID == "" { + actions := make([]Action, 0) if action.PageToken == "" { ctxzap.Extract(ctx).Info("Syncing grants...") s.handleInitialActionForStep(ctx, *action) + + // One type-scoped action per annotated resource type, enqueued + // exactly once (the planner's first page). + typeScoped, err := s.typeScopedGrantsResourceTypes(ctx) + if err != nil { + return fmt.Errorf("sync-grants: error listing type-scoped resource types: %w", err) + } + for _, rtID := range typeScoped { + actions = append(actions, Action{Op: SyncGrantsOp, ResourceTypeID: rtID}) + } } resp, err := s.store.ListResources(ctx, v2.ResourcesServiceListResourcesRequest_builder{ @@ -2019,16 +2211,25 @@ func (s *syncer) SyncGrants(ctx context.Context, action *Action) error { return fmt.Errorf("sync-grants: error listing resources: %w", err) } - actions := make([]Action, 0) for _, r := range resp.GetList() { shouldSkip, err := s.shouldSkipGrants(ctx, r) if err != nil { return err } - if shouldSkip { continue } + + // Types with type-scoped grants are excluded from the + // per-resource fan-out; their single action is enqueued above. + typeScoped, err := s.resourceTypeHasTypeScopedGrants(ctx, r.GetId().GetResourceType()) + if err != nil { + return err + } + if typeScoped { + continue + } + actions = append(actions, Action{Op: SyncGrantsOp, ResourceID: r.GetId().GetResource(), ResourceTypeID: r.GetId().GetResourceType()}) } @@ -2042,30 +2243,68 @@ func (s *syncer) SyncGrants(ctx context.Context, action *Action) error { return nil } +// resourceTypeHasTypeScopedGrants reports (cached per sync) whether the +// resource type carries the TypeScopedGrants annotation. +func (s *syncer) resourceTypeHasTypeScopedGrants(ctx context.Context, resourceTypeID string) (bool, error) { + return s.resourceTypeCarries(ctx, resourceTypeID, &v2.TypeScopedGrants{}, &s.typeScopedGrantsForResourceType) +} + +// typeScopedGrantsResourceTypes lists every synced resource type annotated +// with TypeScopedGrants. +func (s *syncer) typeScopedGrantsResourceTypes(ctx context.Context) ([]string, error) { + return s.resourceTypesCarrying(ctx, &v2.TypeScopedGrants{}, &s.typeScopedGrantsForResourceType) +} + // syncGrantsForResource fetches the grants for a specific resource from the connector. +// An action with an empty ResourceID is a TYPE-SCOPED grants cursor: the connector +// enumerates grants for the whole resource type (no single resource backs the call), +// and may spawn sibling cursors via the EnqueuePageTokens response annotation. // No span here: only call site is SyncGrants, which already owns a span. func (s *syncer) syncGrantsForResource(ctx context.Context, action *Action) error { + typeScoped := action.ResourceID == "" resourceID := v2.ResourceId_builder{ ResourceType: action.ResourceTypeID, Resource: action.ResourceID, }.Build() - resourceResponse, err := s.store.GetResource(ctx, reader_v2.ResourcesReaderServiceGetResourceRequest_builder{ - ResourceId: resourceID, - }.Build()) - if err != nil { - return fmt.Errorf("sync-grants-for-resource: error getting resource: %w", err) - } - resource := resourceResponse.GetResource() + var resource *v2.Resource + var reqAnnos annotations.Annotations + if typeScoped { + resource, reqAnnos = typeScopedRequestStub(action.ResourceTypeID, &v2.TypeScopedGrants{}) + } else { + resourceResponse, err := s.store.GetResource(ctx, reader_v2.ResourcesReaderServiceGetResourceRequest_builder{ + ResourceId: resourceID, + }.Build()) + if err != nil { + return fmt.Errorf("sync-grants-for-resource: error getting resource: %w", err) + } + resource = resourceResponse.GetResource() + } - start := time.Now() - resp, err := s.connector.ListGrants(ctx, v2.GrantsServiceListGrantsRequest_builder{ - Resource: resource, - PageToken: action.PageToken, - ActiveSyncId: s.getActiveSyncID(), - }.Build()) - s.observeConnectorCall(ctx, "list-grants", start, action.ResourceTypeID, action.ResourceID) - s.recordSessionUsage(resp.GetAnnotations()) + var resp *v2.GrantsServiceListGrantsResponse + err := s.withSourceCacheContinuation(ctx, "sync-grants-for-resource", func(extra annotations.Annotations, attempt int) (listAttempt, error) { + annos := make(annotations.Annotations, 0, len(reqAnnos)+len(extra)) + annos = append(annos, reqAnnos...) + annos = append(annos, extra...) + start := time.Now() + r, err := s.connector.ListGrants(ctx, v2.GrantsServiceListGrantsRequest_builder{ + Resource: resource, + PageToken: action.PageToken, + ActiveSyncId: s.getActiveSyncID(), + Annotations: annos, + }.Build()) + s.observeConnectorCall(ctx, continuationCallMethod("list-grants", attempt), start, action.ResourceTypeID, action.ResourceID) + if err != nil { + return listAttempt{}, err + } + s.recordSessionUsage(r.GetAnnotations()) + resp = r + return listAttempt{ + annos: annotations.Annotations(r.GetAnnotations()), + rows: len(r.GetList()), + nextToken: r.GetNextPageToken(), + }, nil + }) if err != nil { return fmt.Errorf("sync-grants-for-resource: error listing grants: %w", err) } @@ -2077,6 +2316,15 @@ func (s *syncer) syncGrantsForResource(ctx context.Context, action *Action) erro respAnnos := annotations.Annotations(resp.GetAnnotations()) insertResourceGrants := respAnnos.Contains(&v2.InsertResourceGrants{}) + // Source-cache replay/stamping for this grants page. putCtx applies + // ONLY to the PutGrants call below — the related-resource PutResources + // writes in this function are resource rows and must not inherit a + // grants-scope stamp. + putCtx, scPage, err := s.beginSourceCachePage(ctx, sourcecache.RowKindGrants, respAnnos, len(grants)) + if err != nil { + return fmt.Errorf("sync-grants-for-resource: %w", err) + } + // Stamp InsertResourceGrants per-grant so the slim-blob writer's // gate sees it. The annotation is response-level, but the writer // needs it per-row to avoid stripping the Resource this path @@ -2159,19 +2407,56 @@ func (s *syncer) syncGrantsForResource(ctx context.Context, action *Action) erro } } - err = s.store.PutGrants(ctx, grants...) + err = s.store.PutGrants(putCtx, grants...) if err != nil { return fmt.Errorf("sync-grants-for-resource: error putting grants: %w", err) } + if err := s.finishSourceCachePage(ctx, scPage); err != nil { + return fmt.Errorf("sync-grants-for-resource: %w", err) + } + s.handleProgress(ctx, action, len(grants)) - if resp.GetNextPageToken() == "" { + if typeScoped { + // Cursors don't map 1:1 to resources (one cursor can cover many + // groups and may also emit cross-type grants), so the per-resource + // "N of M resources covered" accounting is meaningless here and + // would trip the "more grant resources than resources" warning on + // healthy syncs. Count raw grant rows instead — response rows PLUS + // replayed rows: a warm 304 page carries zero response rows while + // landing the scope's whole replayed row set, and counting only + // the former reported near-zero progress for healthy warm syncs. + rows := len(grants) + if scPage != nil { + rows += int(scPage.replayRows) + } + s.counts.SetGrantsCountOnly(resourceID.GetResourceType()) + s.counts.AddGrantsProgress(resourceID.GetResourceType(), rows) + s.counts.LogGrantsProgress(ctx, resourceID.GetResourceType()) + } else if resp.GetNextPageToken() == "" && !action.Spawned { + // A resource counts as covered exactly once: when its ORIGIN + // action's page chain ends. Spawned sibling cursors for the same + // resource also end with an empty token but must not count, or a + // resource with N spawned pages would count N times and trip the + // progress anomaly warning on healthy syncs. s.counts.AddGrantsProgress(resourceID.GetResourceType(), 1) s.counts.LogGrantsProgress(ctx, resourceID.GetResourceType()) } - return s.nextPageOrFinishAction(ctx, action, resp.GetNextPageToken()) + // EnqueuePageTokens: the response may enqueue sibling cursors — + // type-scoped shards or per-resource parallel pages. Spawned cursors + // are ORDINARY pages that happen to be enqueued eagerly: the SDK + // assumes nothing about replay — a spawned page may hit its lookup + // and replay, miss and fetch cold (page boundary shifted since last + // sync), and may chain further via NextPageToken. Shared contract: + // see collectEnqueuedPageTokens (type_scoped.go). + spawned, err := s.collectEnqueuedPageTokens(ctx, "sync-grants-for-resource", SyncGrantsOp, action, respAnnos) + if err != nil { + return err + } + + return s.nextPageOrFinishAction(ctx, action, resp.GetNextPageToken(), spawned...) } func (s *syncer) SyncExternalResources(ctx context.Context, action *Action) error { @@ -2183,6 +2468,13 @@ func (s *syncer) SyncExternalResources(ctx context.Context, action *Action) erro l := ctxzap.Extract(ctx) l.Info("Syncing external resources") + // Ingestion invariant I2 (repair): reconcile the match-processing + // flag with the store-derived fact before anything gates on it — + // replayed rows arm the engine's existence bit, never the stream arm. + if err := s.repairExternalMatchFlag(ctx); err != nil { + return err + } + if s.externalResourceEntitlementIdFilter != "" { err := s.SyncExternalResourcesWithGrantToEntitlement(ctx, s.externalResourceEntitlementIdFilter) if err != nil { @@ -2580,7 +2872,37 @@ func (s *syncer) processGrantsWithExternalPrincipals(ctx context.Context, princi } grantsToDelete := make([]*v2.Grant, 0) - expandedGrants := make([]*v2.Grant, 0) + // expanded dedupes transformed grants by id with DETERMINISTIC + // collision resolution. Overlapping match rules (e.g. MatchID and + // MatchAll on one entitlement resolving to the same principal) + // produce the same grant id with different annotations; naive + // last-writer-wins depends on store iteration order, which differs + // between a cold sync (iterating source grants) and a warm sync + // (iterating replayed transformed grants) — breaking replay + // equivalence. The more specific rule wins (MatchID > key/value + // Match > MatchAll); equal-rank conflicts keep the smaller + // deterministic encoding. Each entry also carries its SOURCE grant's + // source-cache scope: the source grant is deleted below, and a scope + // left holding neither source nor transformed rows would silently + // replay empty on the next sync. + const ( + matchRankAll = iota + 1 + matchRankKeyValue + matchRankID + ) + type expandedEntry struct { + grant *v2.Grant + scope string + rank int + } + expanded := make(map[string]*expandedEntry) + // matchAllSeen guards MatchAll reprocessing. On a warm sync the + // previous sync's transformed grants replay and re-enter this loop, + // each still carrying the original MatchAll annotation; without + // dedupe, N transformed grants would each regenerate all N + // principals' grants (O(N²)). One regeneration per (entitlement, + // trait) produces the complete set. + matchAllSeen := make(map[string]struct{}) for ga, err := range s.store.Grants().ListWithAnnotations(ctx) { if err != nil { @@ -2592,6 +2914,23 @@ func (s *syncer) processGrantsWithExternalPrincipals(ctx context.Context, princi if !annos.ContainsAny(&v2.ExternalResourceMatchAll{}, &v2.ExternalResourceMatch{}, &v2.ExternalResourceMatchID{}) { continue } + sourceScope := ga.SourceScopeKey + addExpanded := func(rank int, g *v2.Grant) { + entry := &expandedEntry{grant: g, scope: sourceScope, rank: rank} + prev, ok := expanded[g.GetId()] + if !ok || entry.rank > prev.rank { + expanded[g.GetId()] = entry + return + } + if entry.rank < prev.rank || prev.grant == g { + // Lower specificity loses; re-adding the same object (the + // MatchID+expandable path mutates and re-adds) is a no-op. + return + } + if grantCollisionKey(entry.scope, g) < grantCollisionKey(prev.scope, prev.grant) { + expanded[g.GetId()] = entry + } + } // Match all matchResourceMatchAllAnno, err := GetExternalResourceMatchAllAnnotation(annos) @@ -2599,21 +2938,31 @@ func (s *syncer) processGrantsWithExternalPrincipals(ctx context.Context, princi return err } if matchResourceMatchAllAnno != nil { - var processPrincipals []*v2.Resource - switch matchResourceMatchAllAnno.GetResourceType() { - case v2.ResourceType_TRAIT_USER: - processPrincipals = userPrincipals - case v2.ResourceType_TRAIT_GROUP: - processPrincipals = groupPrincipals - default: - l.Error("unexpected external resource type trait", zap.Any("trait", matchResourceMatchAllAnno.GetResourceType())) - } - for _, principal := range processPrincipals { - newGrant := newGrantForExternalPrincipal(grant, principal) - expandedGrants = append(expandedGrants, newGrant) + matchAllKey := grant.GetEntitlement().GetId() + "\x00" + matchResourceMatchAllAnno.GetResourceType().String() + if _, done := matchAllSeen[matchAllKey]; !done { + matchAllSeen[matchAllKey] = struct{}{} + var processPrincipals []*v2.Resource + switch matchResourceMatchAllAnno.GetResourceType() { + case v2.ResourceType_TRAIT_USER: + processPrincipals = userPrincipals + case v2.ResourceType_TRAIT_GROUP: + processPrincipals = groupPrincipals + default: + l.Error("unexpected external resource type trait", zap.Any("trait", matchResourceMatchAllAnno.GetResourceType())) + } + for _, principal := range processPrincipals { + newGrant := newGrantForExternalPrincipal(grant, principal) + addExpanded(matchRankAll, newGrant) + } } grantsToDelete = append(grantsToDelete, grant) - continue + // No `continue`: a grant carrying MatchAll AND MatchID/Match + // annotations must still evaluate the more specific rules so + // rank resolution (MatchID > key/value > MatchAll) applies + // within one grant, not just across grants. Transformed + // grant IDs coincide per principal, so addExpanded picks the + // higher-rank version; grantsToDelete double-appends are + // idempotent (same shape as MatchID+Match today). } // Expansion annotation (may be nil for non-expandable grants). @@ -2649,7 +2998,7 @@ func (s *syncer) processGrantsWithExternalPrincipals(ctx context.Context, princi if matchResourceMatchIDAnno != nil { if principal, ok := principalMap[matchResourceMatchIDAnno.GetId()]; ok { newGrant := newGrantForExternalPrincipal(grant, principal) - expandedGrants = append(expandedGrants, newGrant) + addExpanded(matchRankID, newGrant) newGrantAnnos := annotations.Annotations(newGrant.GetAnnotations()) @@ -2682,7 +3031,7 @@ func (s *syncer) processGrantsWithExternalPrincipals(ctx context.Context, princi }.Build() newGrantAnnos.Update(newExpandableAnno) newGrant.SetAnnotations(newGrantAnnos) - expandedGrants = append(expandedGrants, newGrant) + addExpanded(matchRankID, newGrant) } } @@ -2709,7 +3058,7 @@ func (s *syncer) processGrantsWithExternalPrincipals(ctx context.Context, princi if matchExternalResource.GetKey() == "email" { if userTraitContainsEmail(userTrait.GetEmails(), matchExternalResource.GetValue()) { newGrant := newGrantForExternalPrincipal(grant, userPrincipal) - expandedGrants = append(expandedGrants, newGrant) + addExpanded(matchRankKeyValue, newGrant) // continue to next principal since we found an email match continue } @@ -2717,7 +3066,7 @@ func (s *syncer) processGrantsWithExternalPrincipals(ctx context.Context, princi profileVal, ok := resource.GetProfileStringValue(userTrait.GetProfile(), matchExternalResource.GetKey()) if ok && strings.EqualFold(profileVal, matchExternalResource.GetValue()) { newGrant := newGrantForExternalPrincipal(grant, userPrincipal) - expandedGrants = append(expandedGrants, newGrant) + addExpanded(matchRankKeyValue, newGrant) } } case v2.ResourceType_TRAIT_GROUP: @@ -2761,7 +3110,7 @@ func (s *syncer) processGrantsWithExternalPrincipals(ctx context.Context, princi }.Build() newGrantAnnos.Update(newExpandableAnno) newGrant.SetAnnotations(newGrantAnnos) - expandedGrants = append(expandedGrants, newGrant) + addExpanded(matchRankKeyValue, newGrant) } } } @@ -2775,13 +3124,24 @@ func (s *syncer) processGrantsWithExternalPrincipals(ctx context.Context, princi } newGrantIDs := mapset.NewSet[string]() - for _, ng := range expandedGrants { - newGrantIDs.Add(ng.GetId()) + expandedByScope := make(map[string][]*v2.Grant) + for id, e := range expanded { + newGrantIDs.Add(id) + expandedByScope[e.scope] = append(expandedByScope[e.scope], e.grant) } - err = s.store.PutGrants(ctx, expandedGrants...) - if err != nil { - return err + // Stamped groups write under their source grant's scope so the next + // sync's replay of that scope carries the transformed grants forward; + // the "" group is the plain unstamped write. + for scope, grants := range expandedByScope { + putCtx := ctx + if scope != "" { + putCtx = sourcecache.WithScope(ctx, scope) + } + err = s.store.PutGrants(putCtx, grants...) + if err != nil { + return err + } } // Prefer the refs-based delete (exact structural identity) when the @@ -2818,6 +3178,17 @@ type grantByRefsDeleter interface { DeleteGrantByRefs(ctx context.Context, grant *v2.Grant) error } +// grantCollisionKey gives equal-rank transformed-grant collisions a +// deterministic total order that does not depend on store iteration +// order (which differs between cold syncs and warm replayed syncs). +func grantCollisionKey(scope string, g *v2.Grant) string { + b, err := proto.MarshalOptions{Deterministic: true}.Marshal(g) + if err != nil { + return scope + } + return scope + "\x00" + string(b) +} + func newGrantForExternalPrincipal(grant *v2.Grant, principal *v2.Resource) *v2.Grant { newGrant := v2.Grant_builder{ Entitlement: grant.GetEntitlement(), @@ -2881,7 +3252,11 @@ func (s *syncer) expandGrantsForEntitlements(ctx context.Context, action *Action // The expander needs Reader methods (on s.store) plus StoreExpandedGrants // (on s.store.Grants()). An inline adapter composes them so expand // stays decoupled from C1ZStore. + if s.expandDropStats == nil { + s.expandDropStats = &expand.DroppedEdgeStats{} + } expander := expand.NewExpander(expanderStoreAdapter{s.store}, graph) + expander.SetDropStats(s.expandDropStats) err = expander.RunSingleStep(ctx) if err != nil { l.Error("expandGrantsForEntitlements: error during expansion", zap.Error(err)) @@ -2895,6 +3270,9 @@ func (s *syncer) expandGrantsForEntitlements(ctx context.Context, action *Action if expander.IsDone(ctx) { l.Debug("expandGrantsForEntitlements: graph is expanded") + // The sync's one aggregated dropped-edge report (totals + + // distinct-id examples) — per-edge drops log at Debug only. + s.expandDropStats.LogSummary(ctx) s.state.FinishAction(ctx, action) } @@ -2984,6 +3362,10 @@ func (s *syncer) Close(ctx context.Context) error { var errs []error + // Detach the source-cache lookup before the stores go away so a late + // connector RPC can't read a store the syncer no longer owns. + s.clearSourceCacheLookup(ctx) + var storeCloseErr error if s.store != nil { storeCloseErr = s.store.Close(finalizeCtx) @@ -3077,6 +3459,21 @@ func WithStorageEngine(engine c1zstore.Engine) SyncOpt { } } +// WithFailFastInvariants promotes every ingestion-invariant verdict +// (see ingest_invariants.go) to a hard, plainly-attributed sync +// failure: tolerated warns fail, drop arms fail before dropping, repair +// arms are skipped so the underlying bug is named, and I4 (skipped in +// default mode) runs. Tests and the replay-equivalence harness enable +// it; production default follows the per-invariant policy — drops with +// aggregated warnings for disabled-type danglings, warm repair for I3, +// hard failure (with the warm/cold ErrReplayIntegrity ladder) for +// enabled-type danglings, I5, and I6. +func WithFailFastInvariants() SyncOpt { + return func(s *syncer) { + s.failFastInvariants = true + } +} + // WithSkipFullSync skips syncing entirely. func WithSkipFullSync() SyncOpt { return func(s *syncer) { @@ -3103,8 +3500,10 @@ func WithExternalResourceC1ZPath(path string) SyncOpt { // multi-sync behavior), so existing callers are unaffected. // // The file is opened read-only and engine-agnostically (the magic byte -// selects SQLite or Pebble), so the previous-sync c1z may use either -// engine. +// selects SQLite or Pebble), but source-cache replay is served ONLY by +// Pebble artifacts: a SQLite previous sync opens fine and then degrades +// to a cold sync (every lookup misses), with a warning naming the +// reason. SQLite replay is an explicit non-goal. func WithPreviousSyncC1ZPath(path string) SyncOpt { return func(s *syncer) { s.previousSyncC1ZPath = path @@ -3127,6 +3526,19 @@ func WithOptionalPreviousSyncC1ZPath(path string) SyncOpt { } } +// WithoutPreviousSync clears any previous-sync replay source, forcing a +// cold sync. Applied LAST it overrides an earlier +// With(Optional)PreviousSyncC1ZPath — the runners' cold-retry appends it +// to the original option set after a replay-integrity failure +// (ErrReplayIntegrity) so the re-run cannot consult the implicated +// artifact. +func WithoutPreviousSync() SyncOpt { + return func(s *syncer) { + s.previousSyncC1ZPath = "" + s.previousSyncC1ZPathOptional = false + } +} + func WithExternalResourceEntitlementIdFilter(entitlementId string) SyncOpt { return func(s *syncer) { s.externalResourceEntitlementIdFilter = entitlementId @@ -3290,8 +3702,11 @@ func NewSyncer(ctx context.Context, c types.ConnectorClient, opts ...SyncOpt) (S if s.previousSyncC1ZPath != "" { // Open the previous-sync c1z read-only and engine-agnostically - // (NewStore selects the engine from the file's magic byte), so a - // Pebble or SQLite prior run both work as a replay source. + // (NewStore selects the engine from the file's magic byte). Note + // that opening is NOT eligibility: only Pebble artifacts serve + // source-cache replay — a SQLite prior run opens fine and then + // degrades to cold in configureSourceCache (SQLite replay is an + // explicit non-goal). previousSyncStore, err := dotc1z.NewStore(ctx, s.previousSyncC1ZPath, dotc1z.WithReadOnly(true), dotc1z.WithTmpDir(s.tmpDir), @@ -3305,7 +3720,7 @@ func NewSyncer(ctx context.Context, c types.ConnectorClient, opts ...SyncOpt) (S // without ETag replay, never a failed sync. The caller that // maintains the cache replaces it after its next successful // upload, so a bad file self-heals. - ctxzap.Extract(ctx).Warn("previous-sync c1z unusable; syncing without etag replay", + ctxzap.Extract(ctx).Warn("previous-sync c1z unusable; syncing without source-cache replay", zap.String("previous_sync_c1z_path", s.previousSyncC1ZPath), zap.Error(err), ) diff --git a/pkg/sync/syncer_test.go b/pkg/sync/syncer_test.go index c3d0cc556..704847891 100644 --- a/pkg/sync/syncer_test.go +++ b/pkg/sync/syncer_test.go @@ -2022,6 +2022,18 @@ func TestSyncGrants_DoesNotPropagateAnnotationWhenAbsent(t *testing.T) { } func TestValidateEntitlementExclusionGroups(t *testing.T) { + // Drives the exclusionGroupTracker core (ingest invariant I5) over a + // slice, standing in for the store scan validateStoredExclusionGroups + // performs post-collection. + validate := func(ents []*v2.Entitlement) error { + tracker := &exclusionGroupTracker{} + for _, ent := range ents { + if err := tracker.record(ent); err != nil { + return err + } + } + return nil + } newEntitlement := func(t *testing.T, rt *v2.ResourceType, resourceID, slug string, opts ...et.EntitlementOption) *v2.Entitlement { t.Helper() r, err := rs.NewResource(resourceID, rt, resourceID) @@ -2030,78 +2042,67 @@ func TestValidateEntitlementExclusionGroups(t *testing.T) { } t.Run("empty input is a no-op", func(t *testing.T) { - s := &syncer{state: newState()} - require.NoError(t, s.validateEntitlementExclusionGroups(nil)) + require.NoError(t, validate(nil)) }) t.Run("entitlements without exclusion group annotation pass", func(t *testing.T) { - s := &syncer{state: newState()} ents := []*v2.Entitlement{ newEntitlement(t, userResourceType, "user1", "member"), newEntitlement(t, groupResourceType, "group1", "admin"), } - require.NoError(t, s.validateEntitlementExclusionGroups(ents)) + require.NoError(t, validate(ents)) }) t.Run("shared group id within one resource type passes", func(t *testing.T) { - s := &syncer{state: newState()} ents := []*v2.Entitlement{ newEntitlement(t, userResourceType, "user1", "viewer", et.WithExclusionGroup("role")), newEntitlement(t, userResourceType, "user2", "editor", et.WithExclusionGroup("role")), newEntitlement(t, userResourceType, "user3", "owner", et.WithExclusionGroup("role")), } - require.NoError(t, s.validateEntitlementExclusionGroups(ents)) + require.NoError(t, validate(ents)) }) t.Run("distinct group ids across resource types are independent", func(t *testing.T) { - s := &syncer{state: newState()} ents := []*v2.Entitlement{ newEntitlement(t, userResourceType, "user1", "viewer", et.WithExclusionGroup("user-role")), newEntitlement(t, groupResourceType, "group1", "admin", et.WithExclusionGroup("group-role")), } - require.NoError(t, s.validateEntitlementExclusionGroups(ents)) + require.NoError(t, validate(ents)) }) t.Run("same group id on two resource types fails", func(t *testing.T) { - s := &syncer{state: newState()} ents := []*v2.Entitlement{ newEntitlement(t, userResourceType, "user1", "viewer", et.WithExclusionGroup("role")), newEntitlement(t, groupResourceType, "group1", "admin", et.WithExclusionGroup("role")), } - err := s.validateEntitlementExclusionGroups(ents) + err := validate(ents) require.Error(t, err) require.Contains(t, err.Error(), `"role"`) require.Contains(t, err.Error(), `"user"`) require.Contains(t, err.Error(), `"group"`) }) - t.Run("cross-resource-type conflict is detected across separate batches via persisted state", func(t *testing.T) { - s := &syncer{state: newState()} - require.NoError(t, s.validateEntitlementExclusionGroups([]*v2.Entitlement{ - newEntitlement(t, userResourceType, "user1", "viewer", et.WithExclusionGroup("role")), - })) - err := s.validateEntitlementExclusionGroups([]*v2.Entitlement{ - newEntitlement(t, groupResourceType, "group1", "admin", et.WithExclusionGroup("role")), - }) + t.Run("cross-resource-type conflict is detected across pages of one scan", func(t *testing.T) { + tracker := &exclusionGroupTracker{} + require.NoError(t, tracker.record(newEntitlement(t, userResourceType, "user1", "viewer", et.WithExclusionGroup("role")))) + err := tracker.record(newEntitlement(t, groupResourceType, "group1", "admin", et.WithExclusionGroup("role"))) require.Error(t, err) require.Contains(t, err.Error(), `"role"`) }) t.Run("single default per group passes", func(t *testing.T) { - s := &syncer{state: newState()} ents := []*v2.Entitlement{ newEntitlement(t, userResourceType, "user1", "viewer", et.WithExclusionGroupDefault("role", 1)), newEntitlement(t, userResourceType, "user2", "editor", et.WithExclusionGroupOrder("role", 2)), newEntitlement(t, userResourceType, "user3", "owner", et.WithExclusionGroupOrder("role", 3)), } - require.NoError(t, s.validateEntitlementExclusionGroups(ents)) + require.NoError(t, validate(ents)) }) t.Run("two defaults in the same group fail", func(t *testing.T) { - s := &syncer{state: newState()} viewer := newEntitlement(t, userResourceType, "user1", "viewer", et.WithExclusionGroupDefault("role", 1)) editor := newEntitlement(t, userResourceType, "user2", "editor", et.WithExclusionGroupDefault("role", 2)) - err := s.validateEntitlementExclusionGroups([]*v2.Entitlement{viewer, editor}) + err := validate([]*v2.Entitlement{viewer, editor}) require.Error(t, err) require.Contains(t, err.Error(), `"role"`) require.Contains(t, err.Error(), viewer.GetId()) @@ -2109,60 +2110,55 @@ func TestValidateEntitlementExclusionGroups(t *testing.T) { require.Contains(t, err.Error(), "is_default") }) - t.Run("default conflict is detected across separate batches via persisted state", func(t *testing.T) { - s := &syncer{state: newState()} + t.Run("default conflict is detected across pages of one scan", func(t *testing.T) { + tracker := &exclusionGroupTracker{} viewer := newEntitlement(t, userResourceType, "user1", "viewer", et.WithExclusionGroupDefault("role", 1)) - require.NoError(t, s.validateEntitlementExclusionGroups([]*v2.Entitlement{viewer})) + require.NoError(t, tracker.record(viewer)) editor := newEntitlement(t, userResourceType, "user2", "editor", et.WithExclusionGroupDefault("role", 2)) - err := s.validateEntitlementExclusionGroups([]*v2.Entitlement{editor}) + err := tracker.record(editor) require.Error(t, err) require.Contains(t, err.Error(), viewer.GetId()) require.Contains(t, err.Error(), editor.GetId()) }) t.Run("defaults in different groups are independent", func(t *testing.T) { - s := &syncer{state: newState()} ents := []*v2.Entitlement{ newEntitlement(t, userResourceType, "user1", "viewer", et.WithExclusionGroupDefault("role-a", 1)), newEntitlement(t, userResourceType, "user2", "admin", et.WithExclusionGroupDefault("role-b", 1)), } - require.NoError(t, s.validateEntitlementExclusionGroups(ents)) + require.NoError(t, validate(ents)) }) t.Run("group at the cap passes", func(t *testing.T) { - s := &syncer{state: newState()} ents := make([]*v2.Entitlement, 0, maxEntitlementsPerExclusionGroup) for i := 0; i < maxEntitlementsPerExclusionGroup; i++ { ents = append(ents, newEntitlement(t, userResourceType, fmt.Sprintf("user%d", i), "member", et.WithExclusionGroup("role"))) } - require.NoError(t, s.validateEntitlementExclusionGroups(ents)) + require.NoError(t, validate(ents)) }) t.Run("group over the cap fails", func(t *testing.T) { - s := &syncer{state: newState()} ents := make([]*v2.Entitlement, 0, maxEntitlementsPerExclusionGroup+1) for i := 0; i < maxEntitlementsPerExclusionGroup+1; i++ { ents = append(ents, newEntitlement(t, userResourceType, fmt.Sprintf("user%d", i), "member", et.WithExclusionGroup("role"))) } - err := s.validateEntitlementExclusionGroups(ents) + err := validate(ents) require.Error(t, err) require.Contains(t, err.Error(), `"role"`) require.Contains(t, err.Error(), "too many entitlements") }) - t.Run("cap is enforced across separate batches via persisted state", func(t *testing.T) { - s := &syncer{state: newState()} - // First batch fills the group exactly to the cap. - first := make([]*v2.Entitlement, 0, maxEntitlementsPerExclusionGroup) + t.Run("cap is enforced across pages of one scan", func(t *testing.T) { + // First page fills the group exactly to the cap. + tracker := &exclusionGroupTracker{} for i := 0; i < maxEntitlementsPerExclusionGroup; i++ { - first = append(first, newEntitlement(t, userResourceType, fmt.Sprintf("user%d", i), "member", et.WithExclusionGroup("role"))) + require.NoError(t, tracker.record(newEntitlement(t, userResourceType, fmt.Sprintf("user%d", i), "member", et.WithExclusionGroup("role")))) } - require.NoError(t, s.validateEntitlementExclusionGroups(first)) - // A later batch adding one more member to the same group pushes it over. - err := s.validateEntitlementExclusionGroups([]*v2.Entitlement{ + // A later page adding one more member to the same group pushes it over. + err := tracker.record( newEntitlement(t, userResourceType, "extra", "member", et.WithExclusionGroup("role")), - }) + ) require.Error(t, err) require.Contains(t, err.Error(), "too many entitlements") }) diff --git a/pkg/sync/testdata/README-old-fold.md b/pkg/sync/testdata/README-old-fold.md new file mode 100644 index 000000000..c84162cdf --- /dev/null +++ b/pkg/sync/testdata/README-old-fold.md @@ -0,0 +1,33 @@ +# old-fold-mixed-version.c1z + +A REAL mixed-version compaction artifact: produced by the merge-base +SDK's fold compactor (pre-source-cache, pre-compacted-flag; commit +`604dae09`) running over two source-cache-stamped inputs written by the +CURRENT SDK. Frozen here so version-skew behavior is tested against +actual old-binary output bytes, not a synthesis of what we believe old +binaries did. + +Measured properties (which make it the dangerous shape): + +- run record: `type=full`, `compacted` UNSET (the old proto has no field); +- the base input's source-cache MANIFEST survived the fold intact, + validators and all — the old engine copies keyspaces it does not know; +- the sync token carries the compaction provenance section + (`CompactionTokenStats`), which every compactor version has written. + +So every record-level replay gate passes and only the token-provenance +gate refuses it. `source_cache_skew_fixture_test.go` pins that refusal. + +Regeneration (only if the fixture must be rebuilt): + +1. `BATON_GEN_SKEW_INPUTS=/tmp/skew-gen go test -run TestGenerateSkewInputs ./pkg/sync/` +2. `git worktree add /tmp/skew-base `; copy + `cmd/genskewfold` from the worktree history (or re-create per the + test's README reference) and run + `go run ./cmd/genskewfold /tmp/skew-gen/base.c1z /tmp/skew-gen/newer.c1z /tmp/skew-gen/out` + inside the worktree. +3. Copy the output over this file. + +Future releases should extend this corpus via a freeze-at-tag step +(generate artifacts with the just-released binary) rather than in CI, +which runs before tags exist. diff --git a/pkg/sync/testdata/old-fold-mixed-version.c1z b/pkg/sync/testdata/old-fold-mixed-version.c1z new file mode 100644 index 000000000..07ced5062 Binary files /dev/null and b/pkg/sync/testdata/old-fold-mixed-version.c1z differ diff --git a/pkg/sync/type_scoped.go b/pkg/sync/type_scoped.go new file mode 100644 index 000000000..65c140b7f --- /dev/null +++ b/pkg/sync/type_scoped.go @@ -0,0 +1,158 @@ +package sync //nolint:revive,nolintlint // we can't change the package name for backwards compatibility + +// Shared plumbing for TYPE-SCOPED grants and entitlements (see +// proto/c1/connector/v2/annotation_type_scoped_grants.proto and its +// entitlements sibling). Both phases have identical mechanics — a +// resource-type annotation opts a type out of the per-resource fan-out, +// the syncer issues one planner action whose request carries the marker +// annotation over a {type, type} stub resource, and the connector may +// fan out sibling cursors via EnqueuePageTokens — so the mechanics live +// here once, parameterized by marker/cache/op, and the phase functions +// in syncer.go stay thin. + +import ( + "context" + "fmt" + + "go.uber.org/zap" + "google.golang.org/protobuf/proto" + + "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" + + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + reader_v2 "github.com/conductorone/baton-sdk/pb/c1/reader/v2" + "github.com/conductorone/baton-sdk/pkg/annotations" +) + +// resourceTypeCarries reports (cached per sync in cache) whether the +// resource type's stored annotations contain marker. +func (s *syncer) resourceTypeCarries(ctx context.Context, resourceTypeID string, marker proto.Message, cache *syncMap[string, bool]) (bool, error) { + if v, ok := cache.Load(resourceTypeID); ok { + return v, nil + } + rt, err := s.store.GetResourceType(ctx, reader_v2.ResourceTypesReaderServiceGetResourceTypeRequest_builder{ + ResourceTypeId: resourceTypeID, + }.Build()) + if err != nil { + return false, err + } + rtAnnos := annotations.Annotations(rt.GetResourceType().GetAnnotations()) + carries := rtAnnos.Contains(marker) + cache.Store(resourceTypeID, carries) + return carries, nil +} + +// resourceTypesCarrying lists every synced resource type whose stored +// annotations contain marker, warming cache for every type seen. +func (s *syncer) resourceTypesCarrying(ctx context.Context, marker proto.Message, cache *syncMap[string, bool]) ([]string, error) { + var out []string + pageToken := "" + for { + resp, err := s.store.ListResourceTypes(ctx, v2.ResourceTypesServiceListResourceTypesRequest_builder{ + PageToken: pageToken, + }.Build()) + if err != nil { + return nil, err + } + for _, rt := range resp.GetList() { + rtAnnos := annotations.Annotations(rt.GetAnnotations()) + carries := rtAnnos.Contains(marker) + cache.Store(rt.GetId(), carries) + if carries { + out = append(out, rt.GetId()) + } + } + pageToken = resp.GetNextPageToken() + if pageToken == "" { + return out, nil + } + } +} + +// typeScopedRequestStub builds the wire shape of a type-scoped list +// request: wire validation requires a non-empty resource id, so the stub +// resource is self-referential ({type, type}) and the marker annotation +// on the REQUEST is the routing signal the builder dispatches on. +func typeScopedRequestStub(resourceTypeID string, marker proto.Message) (*v2.Resource, annotations.Annotations) { + stub := v2.Resource_builder{ + Id: v2.ResourceId_builder{ + ResourceType: resourceTypeID, + Resource: resourceTypeID, + }.Build(), + }.Build() + var reqAnnos annotations.Annotations + reqAnnos.Update(marker) + return stub, reqAnnos +} + +// collectEnqueuedPageTokens parses a response's EnqueuePageTokens +// annotation and converts its page tokens into sibling actions — each an +// ORDINARY page that happens to be enqueued eagerly: scheduled by the +// worker pool, rate-limited, and checkpointed like any other pagination. +// +// - Type-scoped calls: one cursor per connector-defined shard (e.g. +// 50-id delta chunks); actions carry only the resource type. +// - Per-resource calls: parallel page fan-out for page-numbered APIs; +// actions carry the resource identity. +// +// Every spawned action is persisted in the checkpointed state token, so +// fan-out is capped per response and oversized/empty tokens fail loudly +// (matching the proto contract) rather than bloating every checkpoint or +// silently dropping a shard. +func (s *syncer) collectEnqueuedPageTokens(ctx context.Context, phase string, op ActionOp, action *Action, respAnnos annotations.Annotations) ([]Action, error) { + spawn := &v2.EnqueuePageTokens{} + hasSpawn, err := respAnnos.Pick(spawn) + if err != nil { + return nil, fmt.Errorf("%s: error parsing enqueue-page-tokens annotation: %w", phase, err) + } + if !hasSpawn { + return nil, nil + } + if len(spawn.GetPageTokens()) > maxEnqueuePageTokensPerResponse { + return nil, fmt.Errorf( + "%s: EnqueuePageTokens carried %d page tokens (max %d per response); chain additional spawns across pages instead", + phase, len(spawn.GetPageTokens()), maxEnqueuePageTokensPerResponse) + } + spawned := make([]Action, 0, len(spawn.GetPageTokens())) + totalTokenBytes := 0 + for _, tok := range spawn.GetPageTokens() { + if tok == "" { + // An empty token would spawn a PLANNER call, not a page — + // a connector bug that would loop the fan-out. Loud, like + // the caps. + return nil, fmt.Errorf("%s: EnqueuePageTokens carried an empty page token", phase) + } + if len(tok) > maxEnqueuedPageTokenBytes { + return nil, fmt.Errorf( + "%s: EnqueuePageTokens page token is %d bytes (max %d)", + phase, len(tok), maxEnqueuedPageTokenBytes) + } + totalTokenBytes += len(tok) + if totalTokenBytes > maxEnqueuedPageTokenTotalBytes { + // The per-item caps alone admit ~1GiB of tokens per legal + // response, all persisted into every checkpoint until the + // actions drain — bound the aggregate too. + return nil, fmt.Errorf( + "%s: EnqueuePageTokens carries more than %d total page-token bytes; shrink the tokens or chain additional spawns across pages", + phase, maxEnqueuedPageTokenTotalBytes) + } + spawned = append(spawned, Action{ + Op: op, + ResourceTypeID: action.ResourceTypeID, + ResourceID: action.ResourceID, // empty on type-scoped actions + PageToken: tok, + Spawned: true, + }) + } + if len(spawned) > 0 { + s.state.AddSourceCacheStats(SourceCacheStats{EnqueuedPageTokens: int64(len(spawned))}) + } + ctxzap.Extract(ctx).Debug(phase+": enqueued sibling page-token cursors", + zap.String("resource_type_id", action.ResourceTypeID), + zap.String("resource_id", action.ResourceID), + zap.Int("cursors", len(spawned)), + // estimated_total is advisory (logging/observability only); + // nothing gates on it. + zap.Int64("estimated_total", spawn.GetEstimatedTotal())) + return spawned, nil +} diff --git a/pkg/sync/type_scoped_entitlements_test.go b/pkg/sync/type_scoped_entitlements_test.go new file mode 100644 index 000000000..c73aff503 --- /dev/null +++ b/pkg/sync/type_scoped_entitlements_test.go @@ -0,0 +1,520 @@ +package sync //nolint:revive,nolintlint // we can't change the package name for backwards compatibility + +// End-to-end harness for TYPE-SCOPED entitlements (v2.TypeScopedEntitlements + +// EnqueuePageTokens), mirroring type_scoped_grants_test.go: entitlement rows for a +// whole resource type are served through connector-defined chunk cursors +// instead of one ListEntitlements call per group. Each chunk is its own +// source-cache scope, so warm syncs replay unchanged chunks and +// overlay/tombstone the changed ones. + +import ( + "context" + "fmt" + "path/filepath" + "sort" + "strings" + "sync" + "testing" + + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + "github.com/conductorone/baton-sdk/pkg/annotations" + "github.com/conductorone/baton-sdk/pkg/dotc1z" + "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" + "github.com/conductorone/baton-sdk/pkg/logging" + "github.com/conductorone/baton-sdk/pkg/sourcecache" + et "github.com/conductorone/baton-sdk/pkg/types/entitlement" + rs "github.com/conductorone/baton-sdk/pkg/types/resource" +) + +// chunkedEntitlementsMockConnector serves group entitlements exclusively +// through type-scoped cursors. +type chunkedEntitlementsMockConnector struct { + *mockConnector + + mu sync.Mutex + lookup sourcecache.Lookup + + groupIDs []string // sorted; chunk i = groupIDs[i*50:(i+1)*50] + ents map[string][]string // groupID -> entitlement slugs (e.g. "member") + resByID map[string]*v2.Resource + + tokenByChunk map[int]string // last validator issued per chunk + generation int + + addsByChunk map[int][]*v2.Entitlement + deletedIDsByChunk map[int][]string + + unstampedExtra bool + + planningCalls int + coldChunkRounds int + warmChunkRounds int + perResourceGroupLE int + unstampedPages int +} + +var typeScopedEntGroupRT = v2.ResourceType_builder{ + Id: "group", + DisplayName: "Group", + Traits: []v2.ResourceType_Trait{v2.ResourceType_TRAIT_GROUP}, + Annotations: annotations.New(&v2.TypeScopedEntitlements{}), +}.Build() + +func newChunkedEntitlementsMockConnector(t *testing.T, groupCount int, entsPerGroup int) *chunkedEntitlementsMockConnector { + t.Helper() + mc := &chunkedEntitlementsMockConnector{ + mockConnector: newMockConnector(), + ents: map[string][]string{}, + resByID: map[string]*v2.Resource{}, + tokenByChunk: map[int]string{}, + addsByChunk: map[int][]*v2.Entitlement{}, + deletedIDsByChunk: map[int][]string{}, + } + mc.rtDB = append(mc.rtDB, typeScopedEntGroupRT, userResourceType) + + userIDs := make([]string, 0, 10) + for i := 0; i < 10; i++ { + uid := fmt.Sprintf("user-%02d", i) + u, err := rs.NewUserResource(uid, userResourceType, uid, nil, rs.WithAnnotation(&v2.SkipEntitlementsAndGrants{})) + require.NoError(t, err) + mc.AddResource(context.Background(), u) + mc.resByID[uid] = u + userIDs = append(userIDs, uid) + } + _ = userIDs + + slugs := []string{"member", "admin", "viewer"} + for i := 0; i < groupCount; i++ { + gid := fmt.Sprintf("group-%03d", i) + g, err := rs.NewGroupResource(gid, typeScopedEntGroupRT, gid, nil) + require.NoError(t, err) + mc.AddResource(context.Background(), g) + mc.resByID[gid] = g + mc.groupIDs = append(mc.groupIDs, gid) + + for e := 0; e < entsPerGroup; e++ { + mc.ents[gid] = append(mc.ents[gid], slugs[e%len(slugs)]) + } + } + sort.Strings(mc.groupIDs) + return mc +} + +func (mc *chunkedEntitlementsMockConnector) SetSourceCache(_ context.Context, lookup sourcecache.Lookup) { + mc.mu.Lock() + defer mc.mu.Unlock() + if lookup == nil { + lookup = sourcecache.NoopLookup{} + } + mc.lookup = lookup +} + +func (mc *chunkedEntitlementsMockConnector) Validate(context.Context, *v2.ConnectorServiceValidateRequest, ...grpc.CallOption) (*v2.ConnectorServiceValidateResponse, error) { + return v2.ConnectorServiceValidateResponse_builder{ + Annotations: annotations.New(v2.SourceCacheCapability_builder{ + Mode: v2.SourceCacheCapability_MODE_READ_WRITE, + }.Build()), + }.Build(), nil +} + +func (mc *chunkedEntitlementsMockConnector) chunkCount() int { + return (len(mc.groupIDs) + chunkSize - 1) / chunkSize +} + +func (mc *chunkedEntitlementsMockConnector) chunkGroups(chunk int) []string { + lo := chunk * chunkSize + hi := lo + chunkSize + if hi > len(mc.groupIDs) { + hi = len(mc.groupIDs) + } + return mc.groupIDs[lo:hi] +} + +func entChunkScope(chunk int) string { + return fmt.Sprintf("groups/ents/chunk/%03d?sig=testsig", chunk) +} + +func (mc *chunkedEntitlementsMockConnector) groupEnt(gid, slug string) *v2.Entitlement { + return et.NewAssignmentEntitlement(mc.resByID[gid], slug, et.WithGrantableTo(userResourceType)) +} + +func (mc *chunkedEntitlementsMockConnector) chunkEntitlements(chunk int) []*v2.Entitlement { + var out []*v2.Entitlement + for _, gid := range mc.chunkGroups(chunk) { + for _, slug := range mc.ents[gid] { + out = append(out, mc.groupEnt(gid, slug)) + } + } + return out +} + +func (mc *chunkedEntitlementsMockConnector) mutateAddEntitlement(gid, slug string) { + mc.mu.Lock() + defer mc.mu.Unlock() + mc.ents[gid] = append(mc.ents[gid], slug) + chunk := sort.SearchStrings(mc.groupIDs, gid) / chunkSize + mc.addsByChunk[chunk] = append(mc.addsByChunk[chunk], mc.groupEnt(gid, slug)) +} + +func (mc *chunkedEntitlementsMockConnector) mutateRemoveEntitlement(gid, slug string) { + mc.mu.Lock() + defer mc.mu.Unlock() + kept := mc.ents[gid][:0] + for _, s := range mc.ents[gid] { + if s != slug { + kept = append(kept, s) + } + } + mc.ents[gid] = kept + chunk := sort.SearchStrings(mc.groupIDs, gid) / chunkSize + mc.deletedIDsByChunk[chunk] = append(mc.deletedIDsByChunk[chunk], mc.groupEnt(gid, slug).GetId()) +} + +func (mc *chunkedEntitlementsMockConnector) clearDeltas() { + mc.mu.Lock() + defer mc.mu.Unlock() + mc.addsByChunk = map[int][]*v2.Entitlement{} + mc.deletedIDsByChunk = map[int][]string{} +} + +func (mc *chunkedEntitlementsMockConnector) ListEntitlements( + ctx context.Context, in *v2.EntitlementsServiceListEntitlementsRequest, _ ...grpc.CallOption, +) (*v2.EntitlementsServiceListEntitlementsResponse, error) { + rid := in.GetResource().GetId() + if rid.GetResourceType() != typeScopedEntGroupRT.GetId() { + return v2.EntitlementsServiceListEntitlementsResponse_builder{}.Build(), nil + } + reqAnnos := annotations.Annotations(in.GetAnnotations()) + if !reqAnnos.Contains(&v2.TypeScopedEntitlements{}) { + mc.mu.Lock() + mc.perResourceGroupLE++ + mc.mu.Unlock() + return v2.EntitlementsServiceListEntitlementsResponse_builder{}.Build(), nil + } + + tok := in.GetPageToken() + if tok == "" { + mc.mu.Lock() + mc.planningCalls++ + n := mc.chunkCount() + mc.mu.Unlock() + tokens := make([]string, 0, n) + for i := 0; i < n; i++ { + tokens = append(tokens, fmt.Sprintf("chunk=%d", i)) + } + return v2.EntitlementsServiceListEntitlementsResponse_builder{ + Annotations: annotations.New(v2.EnqueuePageTokens_builder{ + PageTokens: tokens, + EstimatedTotal: int64(len(mc.groupIDs)), + }.Build()), + }.Build(), nil + } + + if tok == "chunk=0&extra" { + mc.mu.Lock() + mc.unstampedPages++ + mc.mu.Unlock() + return v2.EntitlementsServiceListEntitlementsResponse_builder{ + List: mc.unstampedExtraEntitlements(), + }.Build(), nil + } + + var chunk, page int + if n, err := fmt.Sscanf(tok, "chunk=%d&page=%d", &chunk, &page); err != nil || n != 2 { + if _, err := fmt.Sscanf(tok, "chunk=%d", &chunk); err != nil { + return nil, fmt.Errorf("bad page token %q", tok) + } + page = 0 + } + scope := entChunkScope(chunk) + + mc.mu.Lock() + lookup := mc.lookup + lastTok := mc.tokenByChunk[chunk] + adds := mc.addsByChunk[chunk] + deleted := mc.deletedIDsByChunk[chunk] + gen := mc.generation + mc.mu.Unlock() + if lookup == nil { + lookup = sourcecache.NoopLookup{} + } + + newTok := fmt.Sprintf("tok-c%d-g%d", chunk, gen) + rotate := func() { + mc.mu.Lock() + mc.tokenByChunk[chunk] = newTok + mc.mu.Unlock() + } + + if page == 0 { + entry, found, err := lookup.Lookup(ctx, sourcecache.RowKindEntitlements, scope) + if err != nil { + return nil, err + } + if found && lastTok != "" && entry.CacheValidator == lastTok { + mc.mu.Lock() + mc.warmChunkRounds++ + mc.mu.Unlock() + rotate() + next := "" + if mc.unstampedExtra && chunk == 0 { + next = "chunk=0&extra" + } + return v2.EntitlementsServiceListEntitlementsResponse_builder{ + List: adds, + Annotations: annotations.New(v2.SourceCacheReplay_builder{ + ScopeKey: scope, + CacheValidator: newTok, + Overlay: true, + DeletedIds: deleted, + }.Build()), + NextPageToken: next, + }.Build(), nil + } + } + + ents := mc.chunkEntitlements(chunk) + half := len(ents) / 2 + if page == 0 { + mc.mu.Lock() + mc.coldChunkRounds++ + mc.mu.Unlock() + return v2.EntitlementsServiceListEntitlementsResponse_builder{ + List: ents[:half], + Annotations: annotations.New(v2.SourceCacheRecord_builder{ + ScopeKey: scope, + }.Build()), + NextPageToken: fmt.Sprintf("chunk=%d&page=1", chunk), + }.Build(), nil + } + rotate() + next := "" + if mc.unstampedExtra && chunk == 0 { + next = "chunk=0&extra" + } + return v2.EntitlementsServiceListEntitlementsResponse_builder{ + List: ents[half:], + Annotations: annotations.New(v2.SourceCacheRecord_builder{ + ScopeKey: scope, + CacheValidator: newTok, + }.Build()), + NextPageToken: next, + }.Build(), nil +} + +func (mc *chunkedEntitlementsMockConnector) ListGrants(ctx context.Context, in *v2.GrantsServiceListGrantsRequest, _ ...grpc.CallOption) (*v2.GrantsServiceListGrantsResponse, error) { + // Type-scoped entitlements types still get a per-resource grants fan-out + // unless also annotated TypeScopedGrants; return empty so grants don't + // interfere with entitlement assertions. + return v2.GrantsServiceListGrantsResponse_builder{}.Build(), nil +} + +func (mc *chunkedEntitlementsMockConnector) unstampedExtraEntitlements() []*v2.Entitlement { + g := mc.resByID[mc.groupIDs[0]] + return []*v2.Entitlement{ + et.NewAssignmentEntitlement(g, "extra-a", et.WithGrantableTo(userResourceType)), + et.NewAssignmentEntitlement(g, "extra-b", et.WithGrantableTo(userResourceType)), + } +} + +func requireSameEntitlementIDs(t *testing.T, want, got map[string]*v2.Entitlement, msg string) { + t.Helper() + var missing, extra []string + for id := range want { + if _, ok := got[id]; !ok { + missing = append(missing, id) + } + } + for id := range got { + if _, ok := want[id]; !ok { + extra = append(extra, id) + } + } + sort.Strings(missing) + sort.Strings(extra) + require.Emptyf(t, missing, "%s: entitlements missing: %s", msg, strings.Join(missing, ", ")) + require.Emptyf(t, extra, "%s: unexpected entitlements: %s", msg, strings.Join(extra, ", ")) +} + +func listEntitlementsInFile(ctx context.Context, t *testing.T, path string) map[string]*v2.Entitlement { + t.Helper() + reopen, err := dotc1z.NewStore(ctx, path, + dotc1z.WithEngine(c1zstore.EnginePebble), + dotc1z.WithReadOnly(true), + ) + require.NoError(t, err) + defer func() { _ = reopen.Close(ctx) }() + + syncMeta, err := reopen.SyncMeta().LatestFullSync(ctx) + require.NoError(t, err) + require.NotNil(t, syncMeta) + require.NoError(t, reopen.SetCurrentSync(ctx, syncMeta.ID)) + + out := map[string]*v2.Entitlement{} + pageToken := "" + for { + resp, err := reopen.ListEntitlements(ctx, v2.EntitlementsServiceListEntitlementsRequest_builder{PageToken: pageToken}.Build()) + require.NoError(t, err) + for _, e := range resp.GetList() { + out[e.GetId()] = e + } + pageToken = resp.GetNextPageToken() + if pageToken == "" { + break + } + } + return out +} + +func runChunkedEntitlementsSync(ctx context.Context, t *testing.T, mc *chunkedEntitlementsMockConnector, path, prevPath, tmpDir string, extraOpts ...SyncOpt) { + t.Helper() + store, err := dotc1z.NewStore(ctx, path, + dotc1z.WithEngine(c1zstore.EnginePebble), + dotc1z.WithTmpDir(tmpDir), + ) + require.NoError(t, err) + opts := []SyncOpt{WithConnectorStore(store), WithTmpDir(tmpDir)} + if prevPath != "" { + opts = append(opts, WithPreviousSyncC1ZPath(prevPath)) + } + opts = append(opts, extraOpts...) + syncer, err := NewSyncer(ctx, mc, opts...) + require.NoError(t, err) + require.NoError(t, syncer.Sync(ctx)) + require.NoError(t, syncer.Close(ctx)) +} + +func TestTypeScopedEntitlementsMixedStampedUnstampedCursor(t *testing.T) { + ctx, err := logging.Init(context.Background()) + require.NoError(t, err) + tmpDir := t.TempDir() + + mc := newChunkedEntitlementsMockConnector(t, 10, 1) + mc.unstampedExtra = true + require.Equal(t, 1, mc.chunkCount()) + + sync1 := filepath.Join(tmpDir, "sync1.c1z") + runChunkedEntitlementsSync(ctx, t, mc, sync1, "", tmpDir) + require.Equal(t, 1, mc.coldChunkRounds) + require.Equal(t, 1, mc.unstampedPages) + + ents1 := listEntitlementsInFile(ctx, t, sync1) + require.Len(t, ents1, 12) // 10 member + 2 unstamped + + mc.generation = 1 + sync2 := filepath.Join(tmpDir, "sync2.c1z") + runChunkedEntitlementsSync(ctx, t, mc, sync2, sync1, tmpDir) + require.Equal(t, 1, mc.warmChunkRounds) + require.Equal(t, 1, mc.coldChunkRounds) + require.Equal(t, 2, mc.unstampedPages) + + requireSameEntitlementIDs(t, ents1, listEntitlementsInFile(ctx, t, sync2), "warm sync with mixed pages vs cold") + + mc.unstampedExtra = false + mc.generation = 2 + sync3 := filepath.Join(tmpDir, "sync3.c1z") + runChunkedEntitlementsSync(ctx, t, mc, sync3, sync2, tmpDir) + require.Equal(t, 2, mc.warmChunkRounds) + require.Equal(t, 2, mc.unstampedPages) + + ents3 := listEntitlementsInFile(ctx, t, sync3) + require.Len(t, ents3, 10, "unstamped rows must not ride replay") + for _, e := range mc.unstampedExtraEntitlements() { + require.NotContains(t, ents3, e.GetId()) + } +} + +func TestTypeScopedEntitlementsChunkedDelta(t *testing.T) { + for _, workers := range []int{0, 4} { + t.Run(fmt.Sprintf("workers=%d", workers), func(t *testing.T) { + runChunkedEntitlementsDeltaScenario(t, workers) + }) + } +} + +func runChunkedEntitlementsDeltaScenario(t *testing.T, workers int) { + ctx, err := logging.Init(context.Background()) + require.NoError(t, err) + tmpDir := t.TempDir() + + var workerOpts []SyncOpt + if workers > 0 { + workerOpts = append(workerOpts, WithWorkerCount(workers)) + } + + mc := newChunkedEntitlementsMockConnector(t, 120, 1) + require.Equal(t, 3, mc.chunkCount()) + + sync1 := filepath.Join(tmpDir, "sync1.c1z") + runChunkedEntitlementsSync(ctx, t, mc, sync1, "", tmpDir, workerOpts...) + + require.Equal(t, 1, mc.planningCalls) + require.Equal(t, 3, mc.coldChunkRounds) + require.Zero(t, mc.warmChunkRounds) + require.Zero(t, mc.perResourceGroupLE) + + ents1 := listEntitlementsInFile(ctx, t, sync1) + require.Len(t, ents1, 120) + + mc.generation = 1 + mc.mutateAddEntitlement("group-071", "admin") + mc.mutateRemoveEntitlement("group-115", "member") + + sync2 := filepath.Join(tmpDir, "sync2.c1z") + runChunkedEntitlementsSync(ctx, t, mc, sync2, sync1, tmpDir, workerOpts...) + + require.Equal(t, 2, mc.planningCalls) + require.Equal(t, 3, mc.coldChunkRounds) + require.Equal(t, 3, mc.warmChunkRounds) + require.Zero(t, mc.perResourceGroupLE) + + ents2 := listEntitlementsInFile(ctx, t, sync2) + require.Len(t, ents2, 120, "one add and one remove cancel out") + require.Contains(t, ents2, mc.groupEnt("group-071", "admin").GetId(), "overlay add must land") + + mc.clearDeltas() + control := filepath.Join(tmpDir, "control.c1z") + runChunkedEntitlementsSync(ctx, t, mc, control, "", tmpDir, workerOpts...) + requireSameEntitlementIDs(t, listEntitlementsInFile(ctx, t, control), ents2, "warm vs cold control") +} + +func TestTypeScopedEntitlementsTargetedSyncSkipsPerResource(t *testing.T) { + ctx, err := logging.Init(context.Background()) + require.NoError(t, err) + tmpDir := t.TempDir() + + mc := newChunkedEntitlementsMockConnector(t, 5, 1) + target := mc.resByID["group-000"] + + // Full sync first so the resource type (with annotation) is stored. + full := filepath.Join(tmpDir, "full.c1z") + runChunkedEntitlementsSync(ctx, t, mc, full, "", tmpDir) + require.Zero(t, mc.perResourceGroupLE) + + mc.planningCalls = 0 + mc.perResourceGroupLE = 0 + mc.coldChunkRounds = 0 + mc.warmChunkRounds = 0 + + targeted := filepath.Join(tmpDir, "targeted.c1z") + store, err := dotc1z.NewStore(ctx, targeted, + dotc1z.WithEngine(c1zstore.EnginePebble), + dotc1z.WithTmpDir(tmpDir), + ) + require.NoError(t, err) + syncer, err := NewSyncer(ctx, mc, + WithConnectorStore(store), + WithTmpDir(tmpDir), + WithPreviousSyncC1ZPath(full), + WithTargetedSyncResources([]*v2.Resource{target}), + ) + require.NoError(t, err) + require.NoError(t, syncer.Sync(ctx)) + require.NoError(t, syncer.Close(ctx)) + + require.Zero(t, mc.perResourceGroupLE, "targeted sync must not fan out per-resource ListEntitlements for a type-scoped type") + require.Zero(t, mc.planningCalls, "targeted sync must not run the type-scoped entitlements planner either") +} diff --git a/pkg/sync/type_scoped_grants_test.go b/pkg/sync/type_scoped_grants_test.go new file mode 100644 index 000000000..dcc7a920c --- /dev/null +++ b/pkg/sync/type_scoped_grants_test.go @@ -0,0 +1,551 @@ +package sync //nolint:revive,nolintlint // we can't change the package name for backwards compatibility + +// End-to-end harness for TYPE-SCOPED grants (v2.TypeScopedGrants + +// EnqueuePageTokens), modeled on Microsoft Graph's chunked groups/delta: +// membership grants for a whole resource type are served through +// connector-defined 50-id chunk cursors instead of one ListGrants call per +// group. Each chunk is its own source-cache scope, so warm syncs replay +// unchanged chunks and overlay/tombstone the changed ones. +// +// The syncer-side behaviors under test: +// - the grants planner excludes the annotated type from the per-resource +// fan-out and enqueues exactly one type-scoped action, +// - the planning call's EnqueuePageTokens annotation enqueues one action per +// chunk, each checkpointed/paginated independently, +// - chunk scopes replay warm with overlay adds and canonical-id +// tombstones, and equivalence against a cold control sync holds. + +import ( + "context" + "fmt" + "path/filepath" + "sort" + "strings" + "sync" + "testing" + + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + "github.com/conductorone/baton-sdk/pkg/annotations" + "github.com/conductorone/baton-sdk/pkg/dotc1z" + "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" + "github.com/conductorone/baton-sdk/pkg/logging" + "github.com/conductorone/baton-sdk/pkg/sourcecache" + et "github.com/conductorone/baton-sdk/pkg/types/entitlement" + gt "github.com/conductorone/baton-sdk/pkg/types/grant" + rs "github.com/conductorone/baton-sdk/pkg/types/resource" +) + +const chunkSize = 50 + +// chunkedGrantsMockConnector serves group-membership grants exclusively +// through type-scoped cursors, mimicking chunked Graph delta: +// +// - planning call (empty page token): no rows; EnqueuePageTokens with one +// token per 50-group chunk. +// - chunk cursor, lookup miss (cold): enumerate the chunk's grants, +// paginated, final page carrying the chunk's validator. +// - chunk cursor, lookup hit (warm): replay + overlay adds + canonical +// grant-id tombstones + rotated validator. +type chunkedGrantsMockConnector struct { + *mockConnector + + mu sync.Mutex + lookup sourcecache.Lookup + + groupIDs []string // sorted; chunk i = groupIDs[i*50:(i+1)*50] + members map[string][]string // groupID -> userIDs + entByID map[string]*v2.Entitlement + resByID map[string]*v2.Resource + + tokenByChunk map[int]string // last validator issued per chunk + generation int + + // per-sync delta state, applied on warm rounds + addsByChunk map[int][]*v2.Grant + deletedIDsByChunk map[int][]string + + // unstampedExtra, when set, chains one UNSTAMPED page after chunk 0's + // scoped pages — cold and warm alike. Models the Okta shape: a + // type-scoped cursor whose members pages are scope-stamped, followed + // by fresh pages with no change signal (group role assignments) that + // carry no source-cache annotations and must be re-emitted every sync. + unstampedExtra bool + + // observability for assertions + planningCalls int + coldChunkRounds int + warmChunkRounds int + perResourceGroupLG int + unstampedPages int +} + +var typeScopedGroupRT = v2.ResourceType_builder{ + Id: "group", + DisplayName: "Group", + Traits: []v2.ResourceType_Trait{v2.ResourceType_TRAIT_GROUP}, + Annotations: annotations.New(&v2.TypeScopedGrants{}), +}.Build() + +func newChunkedGrantsMockConnector(t *testing.T, groupCount int, membersPerGroup int) *chunkedGrantsMockConnector { + t.Helper() + mc := &chunkedGrantsMockConnector{ + mockConnector: newMockConnector(), + members: map[string][]string{}, + entByID: map[string]*v2.Entitlement{}, + resByID: map[string]*v2.Resource{}, + tokenByChunk: map[int]string{}, + addsByChunk: map[int][]*v2.Grant{}, + deletedIDsByChunk: map[int][]string{}, + } + mc.rtDB = append(mc.rtDB, typeScopedGroupRT, userResourceType) + + // A pool of users, each carrying SkipEntitlementsAndGrants so only the + // group type participates in the grants phase. + userIDs := make([]string, 0, 10) + for i := 0; i < 10; i++ { + uid := fmt.Sprintf("user-%02d", i) + u, err := rs.NewUserResource(uid, userResourceType, uid, nil, rs.WithAnnotation(&v2.SkipEntitlementsAndGrants{})) + require.NoError(t, err) + mc.AddResource(context.Background(), u) + mc.resByID[uid] = u + userIDs = append(userIDs, uid) + } + + for i := 0; i < groupCount; i++ { + gid := fmt.Sprintf("group-%03d", i) + g, err := rs.NewGroupResource(gid, typeScopedGroupRT, gid, nil) + require.NoError(t, err) + mc.AddResource(context.Background(), g) + mc.resByID[gid] = g + mc.groupIDs = append(mc.groupIDs, gid) + + ent := et.NewAssignmentEntitlement(g, "member", et.WithGrantableTo(userResourceType)) + ent.SetSlug("member") + mc.entByID[gid] = ent + mc.entDB[gid] = []*v2.Entitlement{ent} + + for m := 0; m < membersPerGroup; m++ { + mc.members[gid] = append(mc.members[gid], userIDs[(i+m)%len(userIDs)]) + } + } + sort.Strings(mc.groupIDs) + return mc +} + +func (mc *chunkedGrantsMockConnector) SetSourceCache(_ context.Context, lookup sourcecache.Lookup) { + mc.mu.Lock() + defer mc.mu.Unlock() + if lookup == nil { + lookup = sourcecache.NoopLookup{} + } + mc.lookup = lookup +} + +func (mc *chunkedGrantsMockConnector) Validate(context.Context, *v2.ConnectorServiceValidateRequest, ...grpc.CallOption) (*v2.ConnectorServiceValidateResponse, error) { + return v2.ConnectorServiceValidateResponse_builder{ + Annotations: annotations.New(v2.SourceCacheCapability_builder{ + Mode: v2.SourceCacheCapability_MODE_READ_WRITE, + }.Build()), + }.Build(), nil +} + +func (mc *chunkedGrantsMockConnector) chunkCount() int { + return (len(mc.groupIDs) + chunkSize - 1) / chunkSize +} + +func (mc *chunkedGrantsMockConnector) chunkGroups(chunk int) []string { + lo := chunk * chunkSize + hi := lo + chunkSize + if hi > len(mc.groupIDs) { + hi = len(mc.groupIDs) + } + return mc.groupIDs[lo:hi] +} + +func chunkScope(chunk int) string { + return fmt.Sprintf("groups/delta/chunk/%03d?sig=testsig", chunk) +} + +func (mc *chunkedGrantsMockConnector) memberGrant(gid, uid string) *v2.Grant { + return gt.NewGrant(mc.resByID[gid], "member", mc.resByID[uid].GetId()) +} + +func (mc *chunkedGrantsMockConnector) chunkGrants(chunk int) []*v2.Grant { + var out []*v2.Grant + for _, gid := range mc.chunkGroups(chunk) { + for _, uid := range mc.members[gid] { + out = append(out, mc.memberGrant(gid, uid)) + } + } + return out +} + +// mutateAddMember journals a warm-round overlay add and updates live state +// (so a cold control sync sees the same tenant). +func (mc *chunkedGrantsMockConnector) mutateAddMember(gid, uid string) { + mc.mu.Lock() + defer mc.mu.Unlock() + mc.members[gid] = append(mc.members[gid], uid) + chunk := sort.SearchStrings(mc.groupIDs, gid) / chunkSize + mc.addsByChunk[chunk] = append(mc.addsByChunk[chunk], mc.memberGrant(gid, uid)) +} + +// mutateRemoveMember journals a warm-round tombstone and updates live state. +func (mc *chunkedGrantsMockConnector) mutateRemoveMember(gid, uid string) { + mc.mu.Lock() + defer mc.mu.Unlock() + kept := mc.members[gid][:0] + for _, m := range mc.members[gid] { + if m != uid { + kept = append(kept, m) + } + } + mc.members[gid] = kept + chunk := sort.SearchStrings(mc.groupIDs, gid) / chunkSize + mc.deletedIDsByChunk[chunk] = append(mc.deletedIDsByChunk[chunk], mc.memberGrant(gid, uid).GetId()) +} + +func (mc *chunkedGrantsMockConnector) clearDeltas() { + mc.mu.Lock() + defer mc.mu.Unlock() + mc.addsByChunk = map[int][]*v2.Grant{} + mc.deletedIDsByChunk = map[int][]string{} +} + +func (mc *chunkedGrantsMockConnector) ListGrants(ctx context.Context, in *v2.GrantsServiceListGrantsRequest, _ ...grpc.CallOption) (*v2.GrantsServiceListGrantsResponse, error) { + rid := in.GetResource().GetId() + if rid.GetResourceType() != typeScopedGroupRT.GetId() { + // Users are annotated SkipEntitlementsAndGrants; nothing else + // should arrive here. + return v2.GrantsServiceListGrantsResponse_builder{}.Build(), nil + } + reqAnnos := annotations.Annotations(in.GetAnnotations()) + if !reqAnnos.Contains(&v2.TypeScopedGrants{}) { + // The planner must never fan out per-resource for the annotated + // type; recorded and asserted in the test. + mc.mu.Lock() + mc.perResourceGroupLG++ + mc.mu.Unlock() + return v2.GrantsServiceListGrantsResponse_builder{}.Build(), nil + } + + // --- type-scoped call --- + tok := in.GetPageToken() + if tok == "" { + // Planning call: spawn one cursor per chunk, no rows. + mc.mu.Lock() + mc.planningCalls++ + n := mc.chunkCount() + mc.mu.Unlock() + tokens := make([]string, 0, n) + for i := 0; i < n; i++ { + tokens = append(tokens, fmt.Sprintf("chunk=%d", i)) + } + return v2.GrantsServiceListGrantsResponse_builder{ + Annotations: annotations.New(v2.EnqueuePageTokens_builder{ + PageTokens: tokens, + EstimatedTotal: int64(len(mc.groupIDs)), + }.Build()), + }.Build(), nil + } + + if tok == "chunk=0&extra" { + // The unstamped trailing page: fresh rows, NO source-cache + // annotations, emitted on every sync (there is no validator for + // them, so there is no legal replay). + mc.mu.Lock() + mc.unstampedPages++ + mc.mu.Unlock() + return v2.GrantsServiceListGrantsResponse_builder{ + List: mc.unstampedExtraGrants(), + }.Build(), nil + } + + var chunk, page int + if n, err := fmt.Sscanf(tok, "chunk=%d&page=%d", &chunk, &page); err != nil || n != 2 { + if _, err := fmt.Sscanf(tok, "chunk=%d", &chunk); err != nil { + return nil, fmt.Errorf("bad page token %q", tok) + } + page = 0 + } + scope := chunkScope(chunk) + + mc.mu.Lock() + lookup := mc.lookup + lastTok := mc.tokenByChunk[chunk] + adds := mc.addsByChunk[chunk] + deleted := mc.deletedIDsByChunk[chunk] + gen := mc.generation + mc.mu.Unlock() + if lookup == nil { + lookup = sourcecache.NoopLookup{} + } + + newTok := fmt.Sprintf("tok-c%d-g%d", chunk, gen) + rotate := func() { + mc.mu.Lock() + mc.tokenByChunk[chunk] = newTok + mc.mu.Unlock() + } + + if page == 0 { + entry, found, err := lookup.Lookup(ctx, sourcecache.RowKindGrants, scope) + if err != nil { + return nil, err + } + if found && lastTok != "" && entry.CacheValidator == lastTok { + // Warm round: one page — replay, overlay adds, tombstones, + // rotated validator. With unstampedExtra set, chunk 0 chains + // its unstamped trailing page after the replay page: scoped + // and unscoped pages coexisting in one cursor's stream. + mc.mu.Lock() + mc.warmChunkRounds++ + mc.mu.Unlock() + rotate() + next := "" + if mc.unstampedExtra && chunk == 0 { + next = "chunk=0&extra" + } + return v2.GrantsServiceListGrantsResponse_builder{ + List: adds, + Annotations: annotations.New(v2.SourceCacheReplay_builder{ + ScopeKey: scope, + CacheValidator: newTok, + Overlay: true, + DeletedIds: deleted, + }.Build()), + NextPageToken: next, + }.Build(), nil + } + } + + // Cold enumeration for this chunk, two pages to exercise cursor + // checkpointing: page 0 = first half + interim scope (no etag), + // page 1 = rest + validator. + grants := mc.chunkGrants(chunk) + half := len(grants) / 2 + if page == 0 { + mc.mu.Lock() + mc.coldChunkRounds++ + mc.mu.Unlock() + return v2.GrantsServiceListGrantsResponse_builder{ + List: grants[:half], + Annotations: annotations.New(v2.SourceCacheRecord_builder{ + ScopeKey: scope, + }.Build()), + NextPageToken: fmt.Sprintf("chunk=%d&page=1", chunk), + }.Build(), nil + } + rotate() + next := "" + if mc.unstampedExtra && chunk == 0 { + next = "chunk=0&extra" + } + return v2.GrantsServiceListGrantsResponse_builder{ + List: grants[half:], + Annotations: annotations.New(v2.SourceCacheRecord_builder{ + ScopeKey: scope, + CacheValidator: newTok, + }.Build()), + NextPageToken: next, + }.Build(), nil +} + +// enableUnstampedExtra turns on the unstamped trailing page AND registers +// the "admin" entitlement its grants reference: ingest invariant I8 drops +// grants whose entitlement has no row at seal, so the fixture must stay +// referentially honest (as the real connector this scenario pins is). +func (mc *chunkedGrantsMockConnector) enableUnstampedExtra() { + mc.unstampedExtra = true + gid := mc.groupIDs[0] + for _, ent := range mc.entDB[gid] { + if ent.GetSlug() == "admin" { + return + } + } + adminEnt := et.NewAssignmentEntitlement(mc.resByID[gid], "admin", et.WithGrantableTo(userResourceType)) + adminEnt.SetSlug("admin") + mc.entDB[gid] = append(mc.entDB[gid], adminEnt) +} + +// unstampedExtraGrants are the rows served by the unstamped trailing page: +// "admin" grants on group-000 for two fixed users. They exist only while +// the connector emits them — nothing replays them. +func (mc *chunkedGrantsMockConnector) unstampedExtraGrants() []*v2.Grant { + g := mc.resByID[mc.groupIDs[0]] + return []*v2.Grant{ + gt.NewGrant(g, "admin", mc.resByID["user-08"].GetId()), + gt.NewGrant(g, "admin", mc.resByID["user-09"].GetId()), + } +} + +func requireSameGrantIDs(t *testing.T, want, got map[string]*v2.Grant, msg string) { + t.Helper() + var missing, extra []string + for id := range want { + if _, ok := got[id]; !ok { + missing = append(missing, id) + } + } + for id := range got { + if _, ok := want[id]; !ok { + extra = append(extra, id) + } + } + sort.Strings(missing) + sort.Strings(extra) + require.Emptyf(t, missing, "%s: grants missing: %s", msg, strings.Join(missing, ", ")) + require.Emptyf(t, extra, "%s: unexpected grants: %s", msg, strings.Join(extra, ", ")) +} + +func runChunkedSync(ctx context.Context, t *testing.T, mc *chunkedGrantsMockConnector, path, prevPath, tmpDir string, extraOpts ...SyncOpt) { + t.Helper() + store, err := dotc1z.NewStore(ctx, path, + dotc1z.WithEngine(c1zstore.EnginePebble), + dotc1z.WithTmpDir(tmpDir), + ) + require.NoError(t, err) + opts := []SyncOpt{WithConnectorStore(store), WithTmpDir(tmpDir)} + if prevPath != "" { + opts = append(opts, WithPreviousSyncC1ZPath(prevPath)) + } + opts = append(opts, extraOpts...) + syncer, err := NewSyncer(ctx, mc, opts...) + require.NoError(t, err) + require.NoError(t, syncer.Sync(ctx)) + require.NoError(t, syncer.Close(ctx)) +} + +// TestTypeScopedGrantsMixedStampedUnstampedCursor pins a contract point +// proven in the field by the baton-okta POC: within ONE type-scoped +// cursor's page stream, scope-stamped pages (members, replayable) and +// unstamped fresh pages (rows with no change signal, e.g. group role +// assignments) coexist. Stamping is per-page context, not per-cursor +// state, so: +// +// - warm syncs replay the stamped pages while the unstamped pages are +// re-emitted fresh by the connector, and the union matches a cold +// sync exactly; +// - unstamped rows never ride replay: the sync only contains them while +// the connector emits them. +func TestTypeScopedGrantsMixedStampedUnstampedCursor(t *testing.T) { + ctx, err := logging.Init(context.Background()) + require.NoError(t, err) + tmpDir := t.TempDir() + + // 10 groups => 1 chunk, 3 members each => 30 member grants, plus 2 + // unstamped "admin" grants from the trailing extra page. + mc := newChunkedGrantsMockConnector(t, 10, 3) + mc.enableUnstampedExtra() + require.Equal(t, 1, mc.chunkCount()) + + // --- Sync 1: cold — scoped member pages then the unstamped page ---- + sync1 := filepath.Join(tmpDir, "sync1.c1z") + runChunkedSync(ctx, t, mc, sync1, "", tmpDir) + require.Equal(t, 1, mc.coldChunkRounds) + require.Equal(t, 1, mc.unstampedPages, "cold sync serves the unstamped trailing page") + + grants1 := listGrantsInFile(ctx, t, sync1) + require.Len(t, grants1, 32) + + // --- Sync 2: warm — replay page chains into the unstamped page ----- + mc.generation = 1 + sync2 := filepath.Join(tmpDir, "sync2.c1z") + runChunkedSync(ctx, t, mc, sync2, sync1, tmpDir) + require.Equal(t, 1, mc.warmChunkRounds, "member pages replay warm") + require.Equal(t, 1, mc.coldChunkRounds, "no cold re-enumeration") + require.Equal(t, 2, mc.unstampedPages, "unstamped page is re-served on the warm sync — it never replays") + + grants2 := listGrantsInFile(ctx, t, sync2) + requireSameGrantIDs(t, grants1, grants2, "warm sync with mixed pages vs cold") + + // --- Sync 3: warm, connector stops emitting the unstamped page ----- + // The admin grants must vanish: they were never scope-stamped, so + // nothing replays them. If they survive, unstamped rows are leaking + // into replay. + mc.unstampedExtra = false + mc.generation = 2 + sync3 := filepath.Join(tmpDir, "sync3.c1z") + runChunkedSync(ctx, t, mc, sync3, sync2, tmpDir) + require.Equal(t, 2, mc.warmChunkRounds) + require.Equal(t, 2, mc.unstampedPages, "no unstamped page served on sync 3") + + grants3 := listGrantsInFile(ctx, t, sync3) + require.Len(t, grants3, 30, "unstamped rows must not ride replay") + for _, g := range mc.unstampedExtraGrants() { + require.NotContains(t, grants3, g.GetId()) + } +} + +func TestTypeScopedGrantsChunkedDelta(t *testing.T) { + for _, workers := range []int{0, 4} { + t.Run(fmt.Sprintf("workers=%d", workers), func(t *testing.T) { + runChunkedDeltaScenario(t, workers) + }) + } +} + +func runChunkedDeltaScenario(t *testing.T, workers int) { + ctx, err := logging.Init(context.Background()) + require.NoError(t, err) + tmpDir := t.TempDir() + + var workerOpts []SyncOpt + if workers > 0 { + workerOpts = append(workerOpts, WithWorkerCount(workers)) + } + + // 120 groups => 3 chunks (50/50/20), 3 members each => 360 grants. + mc := newChunkedGrantsMockConnector(t, 120, 3) + require.Equal(t, 3, mc.chunkCount()) + + // --- Sync 1: cold --------------------------------------------------- + sync1 := filepath.Join(tmpDir, "sync1.c1z") + runChunkedSync(ctx, t, mc, sync1, "", tmpDir, workerOpts...) + + require.Equal(t, 1, mc.planningCalls, "one planning call per sync") + require.Equal(t, 3, mc.coldChunkRounds, "every chunk enumerates cold on the first sync") + require.Zero(t, mc.warmChunkRounds) + require.Zero(t, mc.perResourceGroupLG, "planner must not fan out per-resource for the annotated type") + + grants1 := listGrantsInFile(ctx, t, sync1) + require.Len(t, grants1, 360) + + // --- Sync 2: warm with changes in two chunks ------------------------ + mc.generation = 1 + mc.mutateAddMember("group-071", "user-00") // chunk 1 + mc.mutateRemoveMember("group-115", mc.members["group-115"][0]) // chunk 2 + + sync2 := filepath.Join(tmpDir, "sync2.c1z") + runChunkedSync(ctx, t, mc, sync2, sync1, tmpDir, workerOpts...) + + require.Equal(t, 2, mc.planningCalls) + require.Equal(t, 3, mc.coldChunkRounds, "no chunk re-enumerates on the warm sync") + require.Equal(t, 3, mc.warmChunkRounds, "every chunk replays warm") + require.Zero(t, mc.perResourceGroupLG) + + grants2 := listGrantsInFile(ctx, t, sync2) + require.Len(t, grants2, 360, "one add and one remove cancel out") + require.Contains(t, grants2, mc.memberGrant("group-071", "user-00").GetId(), "overlay add must land") + + // Equivalence: a cold control sync against the same tenant state must + // produce the same grant set. + mc.clearDeltas() + control := filepath.Join(tmpDir, "control.c1z") + runChunkedSync(ctx, t, mc, control, "", tmpDir, workerOpts...) + require.Equal(t, 6, mc.coldChunkRounds, "control (no previous c1z) re-enumerates every chunk") + controlGrants := listGrantsInFile(ctx, t, control) + requireSameGrantIDs(t, controlGrants, grants2, "warm sync vs cold control") + + // --- Sync 3: no-op warm chained off sync 2 -------------------------- + mc.generation = 2 + sync3 := filepath.Join(tmpDir, "sync3.c1z") + runChunkedSync(ctx, t, mc, sync3, sync2, tmpDir, workerOpts...) + require.Equal(t, 6, mc.warmChunkRounds, "no-op round replays all chunks (validators from sync 2 rotated)") + require.Equal(t, 6, mc.coldChunkRounds, "no cold rounds on the no-op sync") + grants3 := listGrantsInFile(ctx, t, sync3) + requireSameGrantIDs(t, grants2, grants3, "no-op warm sync vs previous") +} diff --git a/pkg/sync/whale_invariants_cost_test.go b/pkg/sync/whale_invariants_cost_test.go new file mode 100644 index 000000000..0bb1ff5e0 --- /dev/null +++ b/pkg/sync/whale_invariants_cost_test.go @@ -0,0 +1,215 @@ +package sync //nolint:revive,nolintlint // we can't change the package name for backwards compatibility + +// Throwaway cost measurement for runIngestionInvariants against a real +// (whale-scale, post-expansion) pebble c1z. Times each invariant's +// underlying store work exactly as the syncer performs it. Gated on +// BATON_WHALE_C1Z pointing at the artifact. +// +// TEMPORARY VERIFICATION FILE — safe to delete; nothing references it. + +import ( + "os" + "testing" + "time" + + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + reader_v2 "github.com/conductorone/baton-sdk/pb/c1/reader/v2" + "github.com/conductorone/baton-sdk/pkg/dotc1z" + "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" + "github.com/conductorone/baton-sdk/pkg/logging" + + "github.com/stretchr/testify/require" +) + +func TestWhaleInvariantCost(t *testing.T) { + path := os.Getenv("BATON_WHALE_C1Z") + if path == "" { + t.Skip("set BATON_WHALE_C1Z to the artifact path") + } + ctx, err := logging.Init(t.Context(), logging.WithLogLevel("error")) + require.NoError(t, err) + + store, err := dotc1z.NewStore(ctx, path, + dotc1z.WithEngine(c1zstore.EnginePebble), + dotc1z.WithReadOnly(true), + dotc1z.WithTmpDir(t.TempDir()), + ) + require.NoError(t, err) + defer func() { _ = store.Close(ctx) }() + + run, err := store.SyncMeta().LatestFullSync(ctx) + require.NoError(t, err) + require.NotNil(t, run) + require.NoError(t, store.SetCurrentSync(ctx, run.ID)) + + facts, ok := store.(dotc1z.IngestInvariantStore) + require.True(t, ok) + sc, isSC := store.(dotc1z.SourceCacheStore) + + // --- I3: grant→resource referential check, exactly as + // checkGrantResourceReferences performs it (skip-scan + point Get per + // distinct entitlement resource; the insert-fact probe only on + // dangling refs). + start := time.Now() + distinct, dangling, annotated := 0, 0, 0 + require.NoError(t, facts.ForEachDistinctGrantEntitlementResource(ctx, func(rt, rid string) error { + distinct++ + exists, err := facts.HasResourceRecord(ctx, rt, rid) + if err != nil { + return err + } + if exists { + return nil + } + dangling++ + carry, err := facts.GrantsForEntResourceCarryInsertFact(ctx, rt, rid) + if err != nil { + return err + } + if carry { + annotated++ + } + return nil + })) + i3 := time.Since(start) + t.Logf("I3 grant→resource: %v (distinct ent-resources=%d dangling=%d insert-annotated=%d)", i3, distinct, dangling, annotated) + + // --- I5: stored-keyspace exclusion-group validation, exactly as + // validateStoredExclusionGroups performs it (full entitlement listing + // + tracker.record per row). + start = time.Now() + tracker := &exclusionGroupTracker{} + ents := 0 + pageToken := "" + for { + resp, err := store.ListEntitlements(ctx, v2.EntitlementsServiceListEntitlementsRequest_builder{PageToken: pageToken}.Build()) + require.NoError(t, err) + for _, ent := range resp.GetList() { + ents++ + require.NoError(t, tracker.record(ent)) + } + if pageToken = resp.GetNextPageToken(); pageToken == "" { + break + } + } + i5 := time.Since(start) + t.Logf("I5 exclusion groups: %v (entitlements=%d)", i5, ents) + + // --- I6: scope-index vs manifest consistency. + if isSC { + start = time.Now() + orphans, err := sc.SourceCacheOrphanScopes(ctx) + require.NoError(t, err) + i6 := time.Since(start) + t.Logf("I6 orphan scopes: %v (orphans=%d kinds)", i6, len(orphans)) + } + + // --- I7: entitlement→resource referential check, exactly as + // checkEntitlementResourceReferences performs it (skip-scan + point + // Get per distinct entitlement resource). + start = time.Now() + i7Distinct, i7Dangling := 0, 0 + require.NoError(t, facts.ForEachDistinctEntitlementResource(ctx, func(rt, rid string) error { + i7Distinct++ + exists, err := facts.HasResourceRecord(ctx, rt, rid) + if err != nil { + return err + } + if !exists { + i7Dangling++ + } + return nil + })) + i7 := time.Since(start) + t.Logf("I7 entitlement→resource: %v (distinct resources=%d dangling=%d)", i7, i7Distinct, i7Dangling) + + // --- I8: grant→entitlement referential check, exactly as + // checkGrantEntitlementReferences performs it (skip-scan at + // entitlement-identity granularity + point probe per distinct + // entitlement; the engine only surfaces dangling ones). + typeKnown := map[string]bool{} + typeExists := func(rt string) bool { + if v, ok := typeKnown[rt]; ok { + return v + } + _, err := store.GetResourceType(ctx, reader_v2.ResourceTypesReaderServiceGetResourceTypeRequest_builder{ + ResourceTypeId: rt, + }.Build()) + typeKnown[rt] = err == nil + return typeKnown[rt] + } + + start = time.Now() + i8Dangling, i8UnsyncedType := 0, 0 + require.NoError(t, facts.ForEachDanglingGrantEntitlement(ctx, func(entID, rt, rid string) error { + i8Dangling++ + if !typeExists(rt) { + i8UnsyncedType++ + } + return nil + })) + i8 := time.Since(start) + t.Logf("I8 grant→entitlement: %v (dangling=%d of which unsynced-type=%d)", i8, i8Dangling, i8UnsyncedType) + + // --- I9: grant→principal dangling scan over by_principal. On a + // sealed artifact the deferred build already ran, so + // EnsureGrantIndexes no-ops and the scan is one seek + probe per + // distinct principal. + start = time.Now() + i9Dangling, i9MatchOnly, i9UnsyncedType := 0, 0, 0 + require.NoError(t, facts.EnsureGrantIndexes(ctx)) + require.NoError(t, facts.ForEachDanglingGrantPrincipal(ctx, func(rt, rid string, matchOnly bool, _ int64) error { + i9Dangling++ + if matchOnly { + i9MatchOnly++ + } + if !typeExists(rt) { + i9UnsyncedType++ + } + return nil + })) + i9 := time.Since(start) + t.Logf("I9 grant→principal: %v (dangling principals=%d match-carrier-only=%d unsynced-type=%d)", i9, i9Dangling, i9MatchOnly, i9UnsyncedType) + + // --- Contrast: a full O(grants) pass over the grant keyspace — what + // I3 would cost if it were per-grant instead of per-distinct-resource. + // Uses the raw listing read path (value decode per row, like any + // full-table invariant would pay). + start = time.Now() + grants := 0 + pageToken = "" + for { + resp, err := store.ListGrants(ctx, v2.GrantsServiceListGrantsRequest_builder{PageToken: pageToken, PageSize: 1000}.Build()) + require.NoError(t, err) + grants += len(resp.GetList()) + if pageToken = resp.GetNextPageToken(); pageToken == "" { + break + } + } + scan := time.Since(start) + t.Logf("CONTRAST full grant scan (O(grants)): %v (grants=%d) — vs I3's %v", scan, grants, i3) + + // --- I4 (STRICT-ONLY in production; measured for reference): full + // resource listing + annotation walk, as checkChildScheduling would. + start = time.Now() + resources, withCRT := 0, 0 + pageToken = "" + for { + resp, err := store.ListResources(ctx, v2.ResourcesServiceListResourcesRequest_builder{PageToken: pageToken}.Build()) + require.NoError(t, err) + for _, r := range resp.GetList() { + resources++ + for _, a := range r.GetAnnotations() { + if a.MessageIs((*v2.ChildResourceType)(nil)) { + withCRT++ + break + } + } + } + if pageToken = resp.GetNextPageToken(); pageToken == "" { + break + } + } + i4 := time.Since(start) + t.Logf("I4 (strict-only) child scheduling scan: %v (resources=%d with-child-annotations=%d)", i4, resources, withCRT) +} diff --git a/pkg/synccompactor/attached/attached.go b/pkg/synccompactor/attached/attached.go index 97c1e50b7..a953012db 100644 --- a/pkg/synccompactor/attached/attached.go +++ b/pkg/synccompactor/attached/attached.go @@ -119,6 +119,15 @@ func (c *Compactor) Compact(ctx context.Context) error { return fmt.Errorf("failed to process records: %w", err) } + // NOTE: unlike the pebble compactors, this path does not mark the + // output sync Compacted or clear source-cache state. That is safe, + // not an oversight: attached compaction is SQLite-only, SQLite + // stores never implement dotc1z.SourceCacheStore, and the syncer's + // replay gate type-asserts for that capability before it ever + // consults sync metadata — a SQLite artifact can't be a replay + // source regardless of its flags. If SQLite ever grows source-cache + // support, this path must adopt the pebble compactors' contract + // (SetCompacted + ClearSourceCacheEntries). return nil } diff --git a/pkg/synccompactor/compactor_pebble.go b/pkg/synccompactor/compactor_pebble.go index 9983ba315..7bedbeafc 100644 --- a/pkg/synccompactor/compactor_pebble.go +++ b/pkg/synccompactor/compactor_pebble.go @@ -647,6 +647,18 @@ func (c *Compactor) compactPebbleFold(ctx context.Context) (string, error) { l.Info("compactPebbleFold: no grant writes; base grant digest state left untouched") } + // Drop the source-cache manifest inherited from the base copy. The + // fold is a keep-newer UPSERT merge — base rows the partials deleted + // survive — so the output is not a faithful snapshot of any input + // sync and NO input's validator may be allowed to 304-validate it. + // With the manifest empty, a syncer using this artifact as its + // previous sync misses every lookup and fetches fresh: cold-correct. + // (The rebuild path is already clean: its bucket specs never carry + // source-cache entries into the fresh output.) + if err := destEng.ClearSourceCacheEntries(ctx); err != nil { + return "", fmt.Errorf("compactPebbleFold: clear source-cache manifest: %w", err) + } + // Optionally compact the folded LSM before save. Off by default: // a compaction rewrites the SSTs that overlap the partials' writes // — for scattered overrides that is most of the base — which both @@ -680,6 +692,10 @@ func (c *Compactor) compactPebbleFold(ctx context.Context) (string, error) { baseRec.SetSyncId(newSyncID) baseRec.SetParentSyncId("") baseRec.SetType(unionType) + // Compaction provenance: this artifact is a keep-newer merge, not a + // connector run — no input's source-cache validators describe it, so + // it must never serve as a replay source (see SyncRunRecord.compacted). + baseRec.SetCompacted(true) if !maxEnded.IsZero() { baseRec.SetEndedAt(timestamppb.New(maxEnded)) } @@ -1186,6 +1202,9 @@ func (c *Compactor) compactPebble(ctx context.Context, newSyncId string) error { return fmt.Errorf("compactPebble: load dest sync_run: %w", err) } rec.SetType(unionType) + // Compaction provenance: rebuild outputs are keep-newer merges too — + // never replay sources (see SyncRunRecord.compacted). + rec.SetCompacted(true) if !maxEnded.IsZero() { rec.SetEndedAt(timestamppb.New(maxEnded)) } diff --git a/pkg/synccompactor/pebble/bucket_plans.go b/pkg/synccompactor/pebble/bucket_plans.go index 33d9be541..7b63bdac9 100644 --- a/pkg/synccompactor/pebble/bucket_plans.go +++ b/pkg/synccompactor/pebble/bucket_plans.go @@ -60,6 +60,34 @@ func buildBucketPlans() []bucketPlan { lower: enginepkg.GrantByNeedsExpansionLowerBound(), upper: enginepkg.GrantByNeedsExpansionUpperBound(), }, + // Source-cache state MUST fold with its rows. Excising these + // ranges alongside the record buckets replaces the base sync's + // manifest/index entries with the applied sync's: scopes the + // applied sync touched keep a consistent (entry, rows) pair, and + // scopes it never touched become clean lookup misses. Leaving + // them out would strand the BASE sync's validators and index + // entries on top of the applied sync's rows — a stale manifest + // hit could then replay rows that no longer belong to the scope. + { + name: "grant_by_source_scope", + lower: enginepkg.GrantBySourceScopeLowerBound(), + upper: enginepkg.GrantBySourceScopeUpperBound(), + }, + { + name: "entitlement_by_source_scope", + lower: enginepkg.EntitlementBySourceScopeLowerBound(), + upper: enginepkg.EntitlementBySourceScopeUpperBound(), + }, + { + name: "resource_by_source_scope", + lower: enginepkg.ResourceBySourceScopeLowerBound(), + upper: enginepkg.ResourceBySourceScopeUpperBound(), + }, + { + name: "source_cache_entry", + lower: enginepkg.SourceCacheEntryLowerBound(), + upper: enginepkg.SourceCacheEntryUpperBound(), + }, { name: "grant_by_entitlement_principal_hash", lower: enginepkg.GrantByEntPrincHashLowerBound(), diff --git a/pkg/synccompactor/source_cache_replay_test.go b/pkg/synccompactor/source_cache_replay_test.go new file mode 100644 index 000000000..a4bb74eac --- /dev/null +++ b/pkg/synccompactor/source_cache_replay_test.go @@ -0,0 +1,189 @@ +package synccompactor + +// Compacted artifacts and source-cache replay. +// +// The production compaction paths (fold's keep-newer MergeInto, the +// overlay/kway rebuild) are UPSERT-merges: base rows the newer sync +// didn't touch survive, and deletions don't propagate. A compacted +// artifact is therefore NOT a faithful snapshot of its newest input +// sync. Carrying the newest sync's source-cache manifest entries into +// such an artifact would be dangerous: a future sync could 304-validate +// a scope against the carried etag and replay a row set that silently +// includes resurrected base rows — stale data presented as current. +// +// The safe semantic — and the one these tests PIN — is that compacted +// artifacts carry NO source-cache manifest entries: every lookup against +// a compacted replay source misses, the connector fetches fresh, and the +// sync is cold-correct. (The IngestAndExcise compactor in +// pkg/synccompactor/pebble replaces whole key ranges and could preserve +// entries soundly — see its bucket_plans.go — but it is not the +// production path.) +// +// Consequence worth knowing operationally: if the previous-sync c1z fed +// to a syncer is a compaction artifact, ETag replay silently never +// engages. Feed raw sync files to WithPreviousSyncC1ZPath when replay +// matters. + +import ( + "context" + "fmt" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + "github.com/conductorone/baton-sdk/pkg/connectorstore" + "github.com/conductorone/baton-sdk/pkg/dotc1z" + "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" + pebbleengine "github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble" + "github.com/conductorone/baton-sdk/pkg/sourcecache" + et "github.com/conductorone/baton-sdk/pkg/types/entitlement" + gt "github.com/conductorone/baton-sdk/pkg/types/grant" + rs "github.com/conductorone/baton-sdk/pkg/types/resource" +) + +var ( + scGroupRT = v2.ResourceType_builder{Id: "group", DisplayName: "Group", Traits: []v2.ResourceType_Trait{v2.ResourceType_TRAIT_GROUP}}.Build() + scUserRT = v2.ResourceType_builder{Id: "user", DisplayName: "User", Traits: []v2.ResourceType_Trait{v2.ResourceType_TRAIT_USER}}.Build() +) + +// writeScopedSyncFile writes one finished Pebble sync whose grants are +// stamped with scope and whose manifest carries (grants, scope, etag). +func writeScopedSyncFile(ctx context.Context, t *testing.T, path, tmpDir, scope, etag string, memberIDs []string) string { + t.Helper() + store, err := dotc1z.NewStore(ctx, path, + dotc1z.WithEngine(c1zstore.EnginePebble), + dotc1z.WithTmpDir(tmpDir), + ) + require.NoError(t, err) + + syncID, isNew, err := store.StartOrResumeSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + require.True(t, isNew) + + group, err := rs.NewGroupResource("g1", scGroupRT, "g1", nil) + require.NoError(t, err) + require.NoError(t, store.PutResourceTypes(ctx, scGroupRT, scUserRT)) + require.NoError(t, store.PutResources(ctx, group)) + // Referential honesty: the grants below reference the member + // entitlement, and compaction's expand-grants pass runs the + // ingestion invariants over the merged output — grants whose + // entitlement has no row would be dropped (I8), as they would be + // for a real connector with this bug. + memberEnt := et.NewAssignmentEntitlement(group, "member") + memberEnt.SetSlug("member") + require.NoError(t, store.PutEntitlements(ctx, memberEnt)) + + grants := make([]*v2.Grant, 0, len(memberIDs)) + for _, uid := range memberIDs { + user, err := rs.NewUserResource(uid, scUserRT, uid, nil) + require.NoError(t, err) + require.NoError(t, store.PutResources(ctx, user)) + grants = append(grants, gt.NewGrant(group, "member", user)) + } + putCtx := sourcecache.WithScope(ctx, scope) + require.NoError(t, store.PutGrants(putCtx, grants...)) + + scStore, ok := any(store).(dotc1z.SourceCacheStore) + require.True(t, ok) + require.NoError(t, scStore.PutSourceCacheEntry(ctx, sourcecache.RowKindGrants, scope, etag)) + + require.NoError(t, store.EndSync(ctx)) + require.NoError(t, store.Close(ctx)) + return syncID +} + +// TestCompactedFileIsColdReplaySource pins the compaction/replay safety +// contract for both production Pebble modes: the compacted output must +// carry NO source-cache manifest entries, so a syncer replaying from it +// misses every lookup and goes cold — never a 304 against a merged row +// set that upsert-merge semantics cannot guarantee matches the etag. +func TestCompactedFileIsColdReplaySource(t *testing.T) { + for _, mode := range []PebbleCompactorMode{PebbleCompactorModeFold, PebbleCompactorModeOverlay} { + t.Run(string(mode), func(t *testing.T) { + ctx := t.Context() + tmpDir := t.TempDir() + scope := sourcecache.HashScope("mock://grants/g1") + + olderPath := filepath.Join(tmpDir, "older.c1z") + newerPath := filepath.Join(tmpDir, "newer.c1z") + olderID := writeScopedSyncFile(ctx, t, olderPath, tmpDir, scope, "v1", []string{"u1", "u2"}) + newerID := writeScopedSyncFile(ctx, t, newerPath, tmpDir, scope, "v2", []string{"u1", "u3"}) + + // Sanity: the inputs DO carry manifest entries, or the + // empty-output assertion below tests nothing. + for _, p := range []string{olderPath, newerPath} { + in, err := dotc1z.NewStore(ctx, p, dotc1z.WithReadOnly(true), dotc1z.WithTmpDir(tmpDir)) + require.NoError(t, err) + holder, ok := any(in).(interface{ PebbleEngine() *pebbleengine.Engine }) + require.True(t, ok) + manifest, err := holder.PebbleEngine().SourceCacheManifestSnapshot(ctx) + require.NoError(t, err) + require.Len(t, manifest, 1, "sanity: input %s must carry its manifest entry", p) + require.NoError(t, in.Close(ctx)) + } + + outDir := filepath.Join(tmpDir, "out") + require.NoError(t, os.MkdirAll(outDir, 0o755)) + compactor, cleanup, err := NewCompactor(ctx, outDir, []*CompactableSync{ + {FilePath: olderPath, SyncID: olderID}, + {FilePath: newerPath, SyncID: newerID}, + }, WithTmpDir(tmpDir), WithPebbleCompactorMode(mode)) + require.NoError(t, err) + defer func() { require.NoError(t, cleanup()) }() + out, err := compactor.Compact(ctx) + require.NoError(t, err) + require.NotNil(t, out) + + compacted, err := dotc1z.NewStore(ctx, out.FilePath, + dotc1z.WithReadOnly(true), + dotc1z.WithTmpDir(tmpDir), + ) + require.NoError(t, err) + defer func() { _ = compacted.Close(ctx) }() + run, err := compacted.SyncMeta().LatestFinishedSyncOfAnyType(ctx) + require.NoError(t, err) + require.NotNil(t, run) + require.NoError(t, compacted.SetCurrentSync(ctx, run.ID)) + + // Provenance metadata: the artifact must SAY it was + // compacted, so orchestrators (and the syncer's replay gate) + // can answer "can this be used for replay?" without + // inspecting keyspaces. The answer is no. + require.True(t, run.Compacted, + "compaction must stamp the output sync run's compacted flag") + + // The safety property: NO manifest entries survive, so a + // syncer using this artifact as its previous sync misses + // every lookup (cold-correct), never replays a merged row + // set against a carried validator. + holder, ok := any(compacted).(interface{ PebbleEngine() *pebbleengine.Engine }) + require.True(t, ok, "compacted output must be a pebble store") + manifest, err := holder.PebbleEngine().SourceCacheManifestSnapshot(ctx) + require.NoError(t, err) + require.Empty(t, manifest, + "compacted artifacts must carry no source-cache manifest entries: upsert-merge output cannot be safely validated by any input's etag") + + scStore, ok := any(compacted).(dotc1z.SourceCacheStore) + require.True(t, ok) + _, found, err := scStore.LookupSourceCacheEntry(ctx, sourcecache.RowKindGrants, scope) + require.NoError(t, err) + require.False(t, found, "a lookup against a compacted replay source must miss") + + // The merged DATA is still there (upsert semantics: newest + // wins per key; base-only rows survive). + grants, err := compacted.ListGrants(ctx, v2.GrantsServiceListGrantsRequest_builder{}.Build()) + require.NoError(t, err) + ids := map[string]bool{} + for _, g := range grants.GetList() { + ids[g.GetId()] = true + } + for _, uid := range []string{"u1", "u3"} { + require.True(t, ids[fmt.Sprintf("group:g1:member:user:%s", uid)], + "newest sync's row for %s must be present", uid) + } + }) + } +} diff --git a/pkg/tasks/c1api/full_sync.go b/pkg/tasks/c1api/full_sync.go index ed2fe9fca..7a8e66518 100644 --- a/pkg/tasks/c1api/full_sync.go +++ b/pkg/tasks/c1api/full_sync.go @@ -8,6 +8,8 @@ import ( "io" "os" "path/filepath" + "slices" + stdsync "sync" "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" "go.uber.org/zap" @@ -44,7 +46,7 @@ type fullSyncTaskHandler struct { workerCount int storageEngine c1zstore.Engine - // previousSyncSparePath is the connector's ETag-replay opt-in: when + // previousSyncSparePath is the connector's source-cache-replay opt-in: when // non-empty, the handler retains one spare c1z (the last successfully // uploaded sync) at exactly this path and feeds it to the next sync // as the replay source. Empty (the default) disables the feature @@ -52,6 +54,41 @@ type fullSyncTaskHandler struct { // host a full c1z of disk. Computed once by the task manager via // previousSyncSparePath(). previousSyncSparePath string + + // previousSyncC1ZPath is an OPERATOR-supplied previous-sync c1z + // (--previous-sync-c1z), taking precedence over the SDK-owned spare. + // Unlike the spare it is never rotated or retired: deleting an + // operator's file is not ours to do (same contract as the local task + // manager), so a replay-integrity retry re-runs cold but leaves the + // file in place. + previousSyncC1ZPath string + + // spareMu serializes every mutation of the spare slot (promotion and + // retirement) across this manager's CONCURRENT full-sync handlers, + // which share one deterministic spare path. Without it, a failing + // task's replay-integrity retirement can race another task's + // promotion and delete a fresh, uninvolved spare. Shared from the + // task manager; nil in tests that don't exercise rotation. + spareMu *stdsync.Mutex + + // openedSpare reports that THIS task selected the SDK spare as its + // replay source. Retirement is gated on it: a replay-integrity + // failure may only retire the spare when the spare was actually + // opened — an operator-supplied --previous-sync-c1z takes precedence + // over the spare, and a failure on the operator's file must never + // delete an SDK spare that sat unopened on disk. + openedSpare bool + // openedSpareSyncID records the sync id of the spare THIS task + // actually opened as its replay source (read at open time, before + // any concurrent rotation can swap the file). Retirement compares it + // against the CURRENT spare's id and only removes on a match — the + // failing sync may only retire the artifact that failed it, never a + // newer spare promoted underneath it. + openedSpareSyncID string + // openedSparePrivatePath is the task-private hardlink pinning the + // exact inode this sync replays from (see openSpareForReplay). + // Removed when the sync finishes. + openedSparePrivatePath string } // previousSyncSparePrefix is the fixed filename prefix of the retained @@ -96,7 +133,7 @@ func previousSyncSparePath(tempDir, clientID string) string { // payload decode) so the spare a future task replays from is positively // identifiable in the logs, and both failure paths warn — a connector // that opted in but whose rotation keeps failing must be visible, not -// silently etag-less forever. +// silently replay-less forever. func promoteOrRemoveC1Z(ctx context.Context, currentPath, sparePath string, keep bool) { l := ctxzap.Extract(ctx) if keep { @@ -107,7 +144,7 @@ func promoteOrRemoveC1Z(ctx context.Context, currentPath, sparePath string, keep zap.String("spare_sync_id", c1zSyncIDBestEffort(sparePath))) return } - l.Warn("failed to retain uploaded c1z as previous-sync spare; removing it instead (etag replay will be inactive next sync)", + l.Warn("failed to retain uploaded c1z as previous-sync spare; removing it instead (source-cache replay will be inactive next sync)", zap.String("current_path", currentPath), zap.String("spare_path", sparePath), zap.Error(err)) @@ -143,6 +180,144 @@ func fileExists(path string) bool { return err == nil && fi.Mode().IsRegular() } +// retireSpareIfStillImplicated removes the spare after a +// replay-integrity failure, but ONLY when it is still the artifact the +// failing sync replayed from. Concurrent full-sync tasks share one +// deterministic spare path: while this task was syncing (minutes to +// hours), another task may have finished and promoted a FRESH spare into +// the slot — that artifact is uninvolved in this failure and must +// survive, or a slow failing task permanently blinds replay that a +// successful sibling just re-armed. +// +// Identity is the sync id read from the c1z manifest header at OPEN time +// vs now. An unreadable id on either side degrades to removal (the old, +// conservative behavior): an extra cold sync is safe; replaying from an +// implicated spare is not. Callers must hold the manager's spare mutex +// so the check-then-remove cannot interleave with a promotion. +func retireSpareIfStillImplicated(ctx context.Context, sparePath, openedSyncID string) { + l := ctxzap.Extract(ctx) + if openedSyncID != "" { + if currentID := c1zSyncIDBestEffort(sparePath); currentID != "" && currentID != openedSyncID { + l.Info("previous-sync spare was rotated by a concurrent task since this sync opened it; leaving the fresh spare in place", + zap.String("spare_path", sparePath), + zap.String("implicated_sync_id", openedSyncID), + zap.String("current_spare_sync_id", currentID)) + return + } + } + if rmErr := os.Remove(sparePath); rmErr != nil && !os.IsNotExist(rmErr) { + l.Warn("failed to remove implicated previous-sync spare", zap.Error(rmErr)) + } +} + +// withSpareLock runs fn holding the manager-wide spare mutex (no-op +// when unset — tests that don't exercise rotation). +func (c *fullSyncTaskHandler) withSpareLock(fn func()) { + if c.spareMu != nil { + c.spareMu.Lock() + defer c.spareMu.Unlock() + } + fn() +} + +// openSpareForReplay selects the SDK spare as this task's replay source +// and returns the path the syncer should open, or "" when no spare is +// on disk. Under the spare lock it captures the spare's sync id AND pins +// the exact inode with a task-private hardlink; the syncer opens the +// link, so a concurrent promotion (an os.Rename onto the shared slot) +// can neither swap the file between identity capture and open nor +// desync the recorded id from what the sync actually read. Without the +// pin, a promotion in that gap would make a later replay-integrity +// retirement compare the WRONG id and preserve the implicated artifact. +// +// The link lives next to the spare (same directory, task-digest suffix +// — constant shape, never caller-influenced) and is removed by +// releaseSpareSnapshot when the sync finishes. A failed link degrades +// to opening the shared path directly: the tiny selection/open race +// returns, costing at worst one preserved-implicated-spare cycle that +// the next successful rotation heals. +func (c *fullSyncTaskHandler) openSpareForReplay(ctx context.Context) string { + l := ctxzap.Extract(ctx) + openPath := "" + c.withSpareLock(func() { + if !fileExists(c.previousSyncSparePath) { + return + } + taskDigest := sha256.Sum256([]byte(c.task.GetId())) + private := fmt.Sprintf("%s.open-%x", c.previousSyncSparePath, taskDigest[:8]) + _ = os.Remove(private) // stale link from a crashed run of this task id + if linkErr := os.Link(c.previousSyncSparePath, private); linkErr == nil { + c.openedSparePrivatePath = private + openPath = private + } else { + l.Warn("failed to pin previous-sync spare with a task-private link; opening the shared path directly", + zap.String("spare_path", c.previousSyncSparePath), zap.Error(linkErr)) + openPath = c.previousSyncSparePath + } + // Read the id from the path the sync will actually open: with + // the pin this is race-free; without it, the same best-effort + // semantics as before. + c.openedSpareSyncID = c1zSyncIDBestEffort(openPath) + c.openedSpare = true + }) + return openPath +} + +// releaseSpareSnapshot removes the task-private hardlink created by +// openSpareForReplay. Safe to call when none was created. +func (c *fullSyncTaskHandler) releaseSpareSnapshot(ctx context.Context) { + if c.openedSparePrivatePath == "" { + return + } + if err := os.Remove(c.openedSparePrivatePath); err != nil && !errors.Is(err, os.ErrNotExist) { + ctxzap.Extract(ctx).Warn("failed to remove task-private spare link", zap.Error(err), + zap.String("path", c.openedSparePrivatePath)) + } + c.openedSparePrivatePath = "" +} + +// retireImplicatedSpare handles the spare side of a replay-integrity +// failure. Gated on openedSpare: only a sync that actually replayed +// from the spare may retire it — a failure on an operator-supplied +// previous file (which wins selection over the spare) must leave the +// unopened spare in place, and a sync that ran cold has nothing to +// retire. +func (c *fullSyncTaskHandler) retireImplicatedSpare(ctx context.Context) { + if !c.openedSpare || c.previousSyncSparePath == "" { + return + } + // Under the spare lock, and only the exact artifact this sync + // opened — a concurrent task's freshly promoted spare is not + // implicated and must survive. + c.withSpareLock(func() { + retireSpareIfStillImplicated(ctx, c.previousSyncSparePath, c.openedSpareSyncID) + }) +} + +// resolveStorageEngine returns the engine this task's sync writes: +// runner config wins, then the task's own engine, then "" (the SDK +// default — SQLite). +func (c *fullSyncTaskHandler) resolveStorageEngine() c1zstore.Engine { + if c.storageEngine != "" { + return c.storageEngine + } + if taskEngine := c.task.GetSyncFull().GetStorageEngine(); taskEngine != "" { + return c1zstore.Engine(taskEngine) + } + return "" +} + +// spareRetentionEligible reports whether an artifact written with engine +// can ever serve source-cache replay. Only Pebble implements +// dotc1z.SourceCacheStore (SQLite replay is an explicit non-goal), so +// retaining a non-Pebble spare costs a full c1z of disk per rotation for +// a file every future sync would refuse — the retention is skipped, with +// a warning so an opted-in operator learns WHY replay never engages +// (set --storage-engine pebble) instead of silently paying for nothing. +func spareRetentionEligible(engine c1zstore.Engine) bool { + return engine == c1zstore.EnginePebble +} + func (c *fullSyncTaskHandler) sync(ctx context.Context, c1zPath string) error { ctx, span := tracer.Start(ctx, "fullSyncTaskHandler.sync") var err error @@ -158,11 +333,7 @@ func (c *fullSyncTaskHandler) sync(ctx context.Context, c1zPath string) error { sdkSync.WithTmpDir(c.helpers.TempDir()), sdkSync.WithWorkerCount(c.workerCount), } - engine := c.storageEngine - if engine == "" && c.task.GetSyncFull().GetStorageEngine() != "" { - engine = c1zstore.Engine(c.task.GetSyncFull().GetStorageEngine()) - } - if engine != "" { + if engine := c.resolveStorageEngine(); engine != "" { syncOpts = append(syncOpts, sdkSync.WithStorageEngine(engine)) } @@ -188,22 +359,31 @@ func (c *fullSyncTaskHandler) sync(ctx context.Context, c1zPath string) error { syncOpts = append(syncOpts, sdkSync.WithExternalResourceC1ZPath(c.externalResourceC1ZPath)) } - // ETag replay (opt-in): feed the spare retained from the last - // successful upload as the previous-sync replay source. Optional - // semantics — a missing/corrupt/stale-format spare degrades to a - // plain sync and is replaced by this task's rotation on success, - // so a bad spare self-heals and can never fail the sync. Both - // branches log: a persistently-absent spare on an opted-in - // connector means rotation is broken, and that must be visible - // rather than silently disabling etag replay forever. - if c.previousSyncSparePath != "" { - if fileExists(c.previousSyncSparePath) { - l.Info("previous-sync spare found; etag replay enabled for this sync", + // source-cache replay (opt-in): an operator-supplied + // --previous-sync-c1z path wins over the SDK-owned spare retained + // from the last successful upload. The spare uses optional semantics + // — a missing/corrupt/stale-format spare degrades to a plain sync + // and is replaced by this task's rotation on success, so a bad spare + // self-heals and can never fail the sync. Both spare branches log: a + // persistently-absent spare on an opted-in connector means rotation + // is broken, and that must be visible rather than silently disabling + // source-cache replay forever. + switch { + case c.previousSyncC1ZPath != "": + l.Info("using operator-supplied previous-sync c1z as the source-cache replay source", + zap.String("previous_sync_c1z", c.previousSyncC1ZPath), + zap.String("previous_sync_id", c1zSyncIDBestEffort(c.previousSyncC1ZPath))) + syncOpts = append(syncOpts, sdkSync.WithPreviousSyncC1ZPath(c.previousSyncC1ZPath)) + case c.previousSyncSparePath != "": + if openPath := c.openSpareForReplay(ctx); openPath != "" { + defer c.releaseSpareSnapshot(ctx) + l.Info("previous-sync spare found; source-cache replay enabled for this sync", zap.String("spare_path", c.previousSyncSparePath), - zap.String("spare_sync_id", c1zSyncIDBestEffort(c.previousSyncSparePath))) - syncOpts = append(syncOpts, sdkSync.WithOptionalPreviousSyncC1ZPath(c.previousSyncSparePath)) + zap.String("open_path", openPath), + zap.String("spare_sync_id", c.openedSpareSyncID)) + syncOpts = append(syncOpts, sdkSync.WithOptionalPreviousSyncC1ZPath(openPath)) } else { - l.Info("no previous-sync spare on disk; etag replay inactive for this sync (expected on the first sync after enabling, or after a failed rotation)", + l.Info("no previous-sync spare on disk; source-cache replay inactive for this sync (expected on the first sync after enabling, or after a failed rotation)", zap.String("spare_path", c.previousSyncSparePath)) } } @@ -235,28 +415,45 @@ func (c *fullSyncTaskHandler) sync(ctx context.Context, c1zPath string) error { syncOpts = append(syncOpts, sdkSync.WithSessionStore(setSessionStore)) } - syncer, err := sdkSync.NewSyncer(ctx, c.helpers.ConnectorClient(), syncOpts...) - if err != nil { - l.Error("failed to create syncer", zap.Error(err)) - return err + runSync := func(opts []sdkSync.SyncOpt) error { + syncer, err := sdkSync.NewSyncer(ctx, c.helpers.ConnectorClient(), opts...) + if err != nil { + l.Error("failed to create syncer", zap.Error(err)) + return err + } + err = syncer.Sync(ctx) + if err != nil { + // We don't defer syncer.Close() in order to capture the error without named return values. + if closeErr := syncer.Close(ctx); closeErr != nil { + l.Error("failed to close syncer after sync error", zap.Error(closeErr)) + err = errors.Join(err, closeErr) + } + return err + } + return syncer.Close(ctx) } // TODO(jirwin): Should we attempt to retry at all before failing the task? - err = syncer.Sync(ctx) - if err != nil { - l.Error("failed to sync", zap.Error(err)) - - // We don't defer syncer.Close() in order to capture the error without named return values. - if closeErr := syncer.Close(ctx); closeErr != nil { - l.Error("failed to close syncer after sync error", zap.Error(err)) - err = errors.Join(err, closeErr) + err = runSync(syncOpts) + if errors.Is(err, sdkSync.ErrReplayIntegrity) { + // Replay is a pure optimization: a replay-integrity failure means + // the warm sync's state cannot be trusted, and the safe automatic + // remediation is a cold re-run — discard the partial output (its + // replayed rows may be wrong; resuming would keep them), retire + // the spare (it is implicated until a fresh rotation replaces + // it), and sync again without a previous source. One retry: a + // cold sync cannot fail this way again. + l.Error("source-cache replay integrity failure; discarding replay state and re-running the sync cold", + zap.Error(err), + zap.String("spare_path", c.previousSyncSparePath)) + if rmErr := os.Remove(c1zPath); rmErr != nil && !os.IsNotExist(rmErr) { + return errors.Join(err, rmErr) } - - return err + c.retireImplicatedSpare(ctx) + err = runSync(append(slices.Clone(syncOpts), sdkSync.WithoutPreviousSync())) } - - if err := syncer.Close(ctx); err != nil { - l.Error("failed to close syncer", zap.Error(err)) + if err != nil { + l.Error("failed to sync", zap.Error(err)) return err } @@ -326,7 +523,21 @@ func (c *fullSyncTaskHandler) HandleTask(ctx context.Context) error { // retained previous sync always corresponds to an artifact C1 has. uploaded := false defer func() { - promoteOrRemoveC1Z(ctx, c1zPath, c.previousSyncSparePath, c.previousSyncSparePath != "" && uploaded) + keep := c.previousSyncSparePath != "" && uploaded + if keep && !spareRetentionEligible(c.resolveStorageEngine()) { + // Retaining a spare that no future sync can replay from is + // pure disk cost; see spareRetentionEligible. + l.Warn("keep-previous-sync-c1z is enabled but this sync's storage engine cannot serve source-cache replay; not retaining a spare "+ + "(set --storage-engine pebble to activate replay)", + zap.String("storage_engine", string(c.resolveStorageEngine()))) + keep = false + } + // Under the spare lock: promotion renames onto the shared slot + // and must not interleave with another task's identity-checked + // retirement. + c.withSpareLock(func() { + promoteOrRemoveC1Z(ctx, c1zPath, c.previousSyncSparePath, keep) + }) }() defer func(f *os.File) { if closeErr := f.Close(); closeErr != nil { @@ -361,6 +572,8 @@ func newFullSyncTaskHandler( workerCount int, storageEngine c1zstore.Engine, previousSyncSparePath string, + previousSyncC1ZPath string, + spareMu *stdsync.Mutex, ) tasks.TaskHandler { return &fullSyncTaskHandler{ task: task, @@ -373,6 +586,8 @@ func newFullSyncTaskHandler( workerCount: workerCount, storageEngine: storageEngine, previousSyncSparePath: previousSyncSparePath, + previousSyncC1ZPath: previousSyncC1ZPath, + spareMu: spareMu, } } diff --git a/pkg/tasks/c1api/full_sync_spare_race_test.go b/pkg/tasks/c1api/full_sync_spare_race_test.go new file mode 100644 index 000000000..0a4c4a4ca --- /dev/null +++ b/pkg/tasks/c1api/full_sync_spare_race_test.go @@ -0,0 +1,196 @@ +package c1api + +// Regression tests for the spare-slot lifecycle race: concurrent +// full-sync handlers share ONE deterministic spare path, so a failing +// task's replay-integrity retirement can interleave with a successful +// sibling's promotion. Retirement must therefore be identity-checked — +// it may only remove the exact artifact the failing sync replayed from, +// never a fresh spare promoted underneath it — and both mutations are +// serialized by the manager's spare mutex. + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + + v1 "github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1" + "github.com/conductorone/baton-sdk/pkg/connectorstore" + "github.com/conductorone/baton-sdk/pkg/dotc1z" + "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" + "github.com/conductorone/baton-sdk/pkg/logging" +) + +// writeSpareC1Z writes a minimal finished pebble c1z at path and returns +// its sync id (readable via c1zSyncIDBestEffort — the same identity the +// production retirement compares). +func writeSpareC1Z(ctx context.Context, t *testing.T, path, tmpDir string) string { + t.Helper() + store, err := dotc1z.NewStore(ctx, path, + dotc1z.WithEngine(c1zstore.EnginePebble), + dotc1z.WithTmpDir(tmpDir), + ) + require.NoError(t, err) + syncID, err := store.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + require.NoError(t, store.EndSync(ctx)) + require.NoError(t, store.Close(ctx)) + require.Equal(t, syncID, c1zSyncIDBestEffort(path), + "fixture: the spare's manifest header must carry the sync id the identity check reads") + return syncID +} + +func TestRetireSpareIfStillImplicated(t *testing.T) { + ctx, err := logging.Init(context.Background()) + require.NoError(t, err) + tmpDir := t.TempDir() + sparePath := filepath.Join(tmpDir, "baton-previous-sync-test.c1z") + + t.Run("rotated spare survives a stale task's retirement", func(t *testing.T) { + // Task A opens spare X, syncs for a long time. Meanwhile task B + // finishes and promotes a FRESH spare Y onto the same path. A's + // replay-integrity failure implicates X — not Y. + implicatedID := writeSpareC1Z(ctx, t, sparePath, tmpDir) + + freshPath := filepath.Join(tmpDir, "fresh.c1z") + freshID := writeSpareC1Z(ctx, t, freshPath, tmpDir) + require.NotEqual(t, implicatedID, freshID) + require.NoError(t, os.Rename(freshPath, sparePath)) // B's promotion + + retireSpareIfStillImplicated(ctx, sparePath, implicatedID) + + require.FileExists(t, sparePath, + "a fresh spare promoted by a concurrent task is not implicated and must survive") + require.Equal(t, freshID, c1zSyncIDBestEffort(sparePath)) + }) + + t.Run("unrotated spare is removed", func(t *testing.T) { + implicatedID := writeSpareC1Z(ctx, t, sparePath, tmpDir) + retireSpareIfStillImplicated(ctx, sparePath, implicatedID) + require.NoFileExists(t, sparePath, + "the artifact the failing sync replayed from must be retired") + }) + + t.Run("unknown opened identity degrades to removal", func(t *testing.T) { + // The failing sync could not identify what it opened (unreadable + // header): conservative direction is retirement — an extra cold + // sync is safe, replaying from an implicated spare is not. + writeSpareC1Z(ctx, t, sparePath, tmpDir) + retireSpareIfStillImplicated(ctx, sparePath, "") + require.NoFileExists(t, sparePath) + }) + + t.Run("unreadable current spare degrades to removal", func(t *testing.T) { + // The slot holds something the identity check can't read (e.g. a + // truncated file): it can't be positively identified as a fresh + // promotion, so it is retired. + require.NoError(t, os.WriteFile(sparePath, []byte("not a c1z"), 0o600)) + retireSpareIfStillImplicated(ctx, sparePath, "some-sync-id") + require.NoFileExists(t, sparePath) + }) + + t.Run("missing spare is a no-op", func(t *testing.T) { + retireSpareIfStillImplicated(ctx, filepath.Join(tmpDir, "ghost.c1z"), "some-sync-id") + }) +} + +// TestRetireImplicatedSpare_RequiresOpenedSpare pins the retirement +// gate: only a sync that actually opened the spare may retire it. An +// operator-supplied --previous-sync-c1z wins selection over the spare, +// so a replay-integrity failure on the operator's file must leave the +// never-opened spare on disk — before the gate, any warm failure with a +// spare path configured deleted the spare unconditionally (and, with no +// opened identity recorded, would also delete a concurrent promotion). +func TestRetireImplicatedSpare_RequiresOpenedSpare(t *testing.T) { + ctx, err := logging.Init(context.Background()) + require.NoError(t, err) + tmpDir := t.TempDir() + sparePath := filepath.Join(tmpDir, "baton-previous-sync-test.c1z") + writeSpareC1Z(ctx, t, sparePath, tmpDir) + + h := &fullSyncTaskHandler{ + task: fullSyncTask(t, "pebble"), + previousSyncSparePath: sparePath, + previousSyncC1ZPath: filepath.Join(tmpDir, "operator.c1z"), // operator path configured too + // openedSpare deliberately false: the operator path won selection. + } + h.retireImplicatedSpare(ctx) + require.FileExists(t, sparePath, + "a warm failure on the operator's previous file must not retire the SDK spare it never opened") + + // The same handler after actually opening the spare retires it. + h2 := &fullSyncTaskHandler{task: fullSyncTask(t, "pebble"), previousSyncSparePath: sparePath} + openPath := h2.openSpareForReplay(ctx) + require.NotEmpty(t, openPath) + h2.retireImplicatedSpare(ctx) + h2.releaseSpareSnapshot(ctx) + require.NoFileExists(t, sparePath, "an opened, unrotated spare is implicated and must be retired") +} + +// TestOpenSpareForReplay_PinSurvivesConcurrentPromotion pins the +// selection/open race fix: the task-private hardlink freezes the exact +// inode the sync replays from, so a promotion landing between selection +// and open can neither swap the file under the sync nor desync the +// recorded identity — retirement then correctly preserves the fresh +// promotion while the sync read the implicated artifact throughout. +func TestOpenSpareForReplay_PinSurvivesConcurrentPromotion(t *testing.T) { + ctx, err := logging.Init(context.Background()) + require.NoError(t, err) + tmpDir := t.TempDir() + sparePath := filepath.Join(tmpDir, "baton-previous-sync-test.c1z") + implicatedID := writeSpareC1Z(ctx, t, sparePath, tmpDir) + + h := &fullSyncTaskHandler{task: fullSyncTask(t, "pebble"), previousSyncSparePath: sparePath} + openPath := h.openSpareForReplay(ctx) + require.NotEmpty(t, openPath) + require.NotEqual(t, sparePath, openPath, "selection must pin a task-private link, not the shared slot") + require.Equal(t, implicatedID, h.openedSpareSyncID) + + // Concurrent task promotes a FRESH spare onto the shared slot after + // selection but before/during this sync's reads. + freshPath := filepath.Join(tmpDir, "fresh.c1z") + freshID := writeSpareC1Z(ctx, t, freshPath, tmpDir) + require.NoError(t, os.Rename(freshPath, sparePath)) + + // The pinned inode still serves the artifact this sync selected. + require.Equal(t, implicatedID, c1zSyncIDBestEffort(openPath), + "the pinned link must keep serving the selected artifact across a promotion") + + // Warm failure: retirement preserves the fresh promotion (identity + // mismatch) — and the implicated artifact dies with the pin release. + h.retireImplicatedSpare(ctx) + require.FileExists(t, sparePath) + require.Equal(t, freshID, c1zSyncIDBestEffort(sparePath), + "the concurrently promoted spare is not implicated and must survive") + h.releaseSpareSnapshot(ctx) + require.NoFileExists(t, openPath, "the task-private pin must be cleaned up") +} + +// TestSpareRetentionEligibility pins the engine gate on spare rotation: +// only Pebble artifacts can serve source-cache replay (SQLite replay is +// an explicit non-goal), so a keep-previous-sync-c1z opt-in running on +// the DEFAULT engine (empty = SQLite) must not retain a spare — a full +// c1z of disk per rotation for a file every future sync would refuse. +func TestSpareRetentionEligibility(t *testing.T) { + require.True(t, spareRetentionEligible(c1zstore.EnginePebble)) + require.False(t, spareRetentionEligible(c1zstore.EngineSQLite)) + require.False(t, spareRetentionEligible(""), "the empty engine resolves to the SQLite default and must not retain") + + // Engine resolution precedence: runner config > task > default. + h := &fullSyncTaskHandler{task: fullSyncTask(t, "")} + require.Equal(t, c1zstore.Engine(""), h.resolveStorageEngine(), "no config, no task engine: SDK default (SQLite)") + h = &fullSyncTaskHandler{task: fullSyncTask(t, "pebble")} + require.Equal(t, c1zstore.EnginePebble, h.resolveStorageEngine(), "task-supplied engine applies") + h = &fullSyncTaskHandler{task: fullSyncTask(t, "sqlite"), storageEngine: c1zstore.EnginePebble} + require.Equal(t, c1zstore.EnginePebble, h.resolveStorageEngine(), "runner config wins over the task") +} + +// fullSyncTask builds a minimal full-sync task carrying storageEngine. +func fullSyncTask(t *testing.T, storageEngine string) *v1.Task { + t.Helper() + syncFull := &v1.Task_SyncFullTask{} + syncFull.SetStorageEngine(storageEngine) + return v1.Task_builder{SyncFull: syncFull}.Build() +} diff --git a/pkg/tasks/c1api/manager.go b/pkg/tasks/c1api/manager.go index c0df2348a..032c9a956 100644 --- a/pkg/tasks/c1api/manager.go +++ b/pkg/tasks/c1api/manager.go @@ -6,6 +6,7 @@ import ( "errors" "os" "strconv" + stdsync "sync" "sync/atomic" "time" @@ -71,11 +72,22 @@ type c1ApiTaskManager struct { storageEngine c1zstore.Engine // previousSyncSparePath is non-empty when the connector opted into - // ETag replay (keepPreviousSyncC1Z): the fixed, client-id-namespaced + // source-cache replay (keepPreviousSyncC1Z): the fixed, client-id-namespaced // location of the retained previous-sync c1z. Computed once at // construction so every full-sync task agrees on the same spare. previousSyncSparePath string + // previousSyncC1ZPath is an operator-supplied previous-sync c1z + // (--previous-sync-c1z); takes precedence over the spare and is + // never rotated or deleted by the SDK. + previousSyncC1ZPath string + + // spareLifecycleMu serializes spare-slot mutations (promotion, + // replay-integrity retirement) across concurrent full-sync handlers, + // which all share previousSyncSparePath. See + // retireSpareIfStillImplicated. + spareLifecycleMu stdsync.Mutex + // runnerShouldDebug is flipped by the StartDebugging task handler (which // runs on a task-processing goroutine) and read by the runner loop via // ShouldDebug(). It is atomic to avoid a data race between those two. @@ -434,6 +446,8 @@ func (c *c1ApiTaskManager) Process(ctx context.Context, task *v1.Task, cc types. c.workerCount, c.storageEngine, c.previousSyncSparePath, + c.previousSyncC1ZPath, + &c.spareLifecycleMu, ) case taskTypes.HelloType: handler = newHelloTaskHandler(task, tHelpers) @@ -503,6 +517,7 @@ func NewC1TaskManager( storageEngine c1zstore.Engine, taskConcurrency int, keepPreviousSyncC1Z bool, + previousSyncC1Z string, ) (BootstrappingTaskManager, error) { serviceClient, err := newServiceClient(ctx, clientID, clientSecret) if err != nil { @@ -530,5 +545,6 @@ func NewC1TaskManager( workerCount: workerCount, storageEngine: storageEngine, previousSyncSparePath: sparePath, + previousSyncC1ZPath: previousSyncC1Z, }, nil } diff --git a/pkg/tasks/local/syncer.go b/pkg/tasks/local/syncer.go index 5d469ca50..1b0a12b7b 100644 --- a/pkg/tasks/local/syncer.go +++ b/pkg/tasks/local/syncer.go @@ -3,10 +3,13 @@ package local import ( "context" "errors" + "os" + "slices" "sync" "time" "go.opentelemetry.io/otel/trace" + "go.uber.org/zap" v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" v1 "github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1" @@ -17,6 +20,7 @@ import ( "github.com/conductorone/baton-sdk/pkg/types" "github.com/conductorone/baton-sdk/pkg/uotel" "github.com/conductorone/baton-sdk/pkg/uotel/uotelzap" + "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" ) type localSyncer struct { @@ -24,6 +28,7 @@ type localSyncer struct { o sync.Once tmpDir string externalResourceC1Z string + previousSyncC1Z string externalResourceEntitlementIdFilter string targetedSyncResources []*v2.Resource skipEntitlementsAndGrants bool @@ -47,6 +52,12 @@ func WithExternalResourceC1Z(externalResourceC1Z string) Option { } } +func WithPreviousSyncC1Z(previousSyncC1Z string) Option { + return func(m *localSyncer) { + m.previousSyncC1Z = previousSyncC1Z + } +} + func WithExternalResourceEntitlementIdFilter(entitlementId string) Option { return func(m *localSyncer) { m.externalResourceEntitlementIdFilter = entitlementId @@ -122,6 +133,7 @@ func (m *localSyncer) Process(ctx context.Context, task *v1.Task, cc types.Conne sdkSync.WithC1ZPath(m.dbPath), sdkSync.WithTmpDir(m.tmpDir), sdkSync.WithExternalResourceC1ZPath(m.externalResourceC1Z), + sdkSync.WithPreviousSyncC1ZPath(m.previousSyncC1Z), sdkSync.WithExternalResourceEntitlementIdFilter(m.externalResourceEntitlementIdFilter), sdkSync.WithTargetedSyncResources(m.targetedSyncResources), sdkSync.WithSkipEntitlementsAndGrants(m.skipEntitlementsAndGrants), @@ -134,24 +146,43 @@ func (m *localSyncer) Process(ctx context.Context, task *v1.Task, cc types.Conne syncOpts = append(syncOpts, sdkSync.WithStorageEngine(m.storageEngine)) } - syncer, err := sdkSync.NewSyncer(ctx, cc, syncOpts...) - if err != nil { - return err - } - - err = syncer.Sync(ctx) - if err != nil { - if closeErr := syncer.Close(ctx); closeErr != nil { - err = errors.Join(err, closeErr) + runSync := func(opts []sdkSync.SyncOpt) error { + syncer, err := sdkSync.NewSyncer(ctx, cc, opts...) + if err != nil { + return err } - return err - } - - if err := syncer.Close(ctx); err != nil { - return err + if err := syncer.Sync(ctx); err != nil { + if closeErr := syncer.Close(ctx); closeErr != nil { + err = errors.Join(err, closeErr) + } + return err + } + return syncer.Close(ctx) + } + + err = runSync(syncOpts) + if errors.Is(err, sdkSync.ErrReplayIntegrity) { + // Replay is a pure optimization: discard the partial output (its + // replayed rows may be wrong; resuming would keep them) and + // re-run cold, without the previous-sync source. One retry: a + // cold sync cannot fail this way again. See ErrReplayIntegrity. + // + // Unlike the service-mode runner (c1api), which retires its + // SDK-owned previous-sync spare on this failure, the local + // previous-sync file is USER-SUPPLIED and is deliberately left + // in place — deleting an operator's file is not ours to do. The + // consequence: if the file itself is bad, every future run that + // passes it will warm-fail and cold-retry again, so say so. + ctxzap.Extract(ctx).Error("source-cache replay integrity failure; discarding replay state and re-running the sync cold. "+ + "The previous-sync c1z is retained; if this recurs across runs, the file is implicated — replace it with a fresh sync's output or stop passing it", + zap.Error(err), + zap.String("previous_sync_c1z", m.previousSyncC1Z)) + if rmErr := os.Remove(m.dbPath); rmErr != nil && !os.IsNotExist(rmErr) { + return errors.Join(err, rmErr) + } + err = runSync(append(slices.Clone(syncOpts), sdkSync.WithoutPreviousSync())) } - - return nil + return err } // NewSyncer returns a task manager that queues a sync task. diff --git a/pkg/types/resource/resource.go b/pkg/types/resource/resource.go index 16585ed3f..4b246a0eb 100644 --- a/pkg/types/resource/resource.go +++ b/pkg/types/resource/resource.go @@ -8,6 +8,7 @@ import ( v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" "github.com/conductorone/baton-sdk/pkg/annotations" "github.com/conductorone/baton-sdk/pkg/pagination" + "github.com/conductorone/baton-sdk/pkg/sourcecache" "github.com/conductorone/baton-sdk/pkg/types/sessions" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -524,9 +525,14 @@ func NewManagedDeviceResource( } type SyncOpAttrs struct { - Session sessions.SessionStore - SyncID string - PageToken pagination.Token + Session sessions.SessionStore + // SourceCache resolves a scope's previous-sync validator (HTTP ETag / + // delta token) for source-cache replay. Never nil for framework-built + // connectors: when source cache is disabled or degraded the SDK + // supplies sourcecache.NoopLookup, which always misses. + SourceCache sourcecache.Lookup + SyncID string + PageToken pagination.Token } type SyncOpResults struct { diff --git a/proto/c1/connector/v2/annotation_source_cache.proto b/proto/c1/connector/v2/annotation_source_cache.proto new file mode 100644 index 000000000..d4fcce1f7 --- /dev/null +++ b/proto/c1/connector/v2/annotation_source_cache.proto @@ -0,0 +1,291 @@ +syntax = "proto3"; + +package c1.connector.v2; + +import "validate/validate.proto"; + +option go_package = "github.com/conductorone/baton-sdk/pb/c1/connector/v2"; + +// Source-cache replay annotations. +// +// A connector that talks to an upstream API with cheap change detection +// (HTTP conditional requests, delta/watermark queries) can opt in to +// source-cache replay: the SDK persists which rows each upstream +// request/scope produced, and on the next sync the connector can tell the +// SDK to replay a scope's previous rows instead of re-emitting them. +// +// The connector owns scope computation. A scope is any stable partition of +// upstream data the connector can cheaply revalidate as a unit: a page URL +// for GitHub-style conditional GETs, a whole collection for Microsoft +// Graph-style delta queries. The SDK never computes provider scopes; it +// only keys storage by the connector-supplied scope_key. +// +// The cache_validator is an opaque validator persisted per scope: a literal HTTP ETag +// for conditional requests, a delta/skip token for delta queries. The SDK +// hands it back via the source-cache lookup on the next sync and never +// interprets it. +// +// Flow per sync, per scope: +// +// 1. Connector looks up the previous validator for (row_kind, scope_key) via +// the SDK-provided lookup. Miss => fetch fresh, emit rows, attach +// SourceCacheRecord. Hit => revalidate upstream with the validator. +// 2. Unchanged (HTTP 304): attach SourceCacheReplay with overlay=false and +// the (unchanged) validator; emit no rows. +// 3. Changed-with-diff (delta query): attach SourceCacheReplay with +// overlay=true and any removed ids; emit ONLY the changed rows (tagged +// with SourceCacheRecord); the final page's SourceCacheRecord carries the +// new token as cache_validator. +// 4. Changed-without-diff (plain conditional GET): fetch fresh, emit rows, +// attach SourceCacheRecord with the new validator. Nothing is replayed. +// +// Replay is only honored when the connector declared +// SourceCacheCapability MODE_READ_WRITE on its Validate response AND the +// sync is running on a storage engine that supports source-cache replay +// with a usable previous sync. When either condition fails the SDK +// installs a lookup that always misses, so a well-behaved connector +// (invariant: only emit SourceCacheReplay for a scope whose validator came +// from this sync's lookup) naturally degrades to full fetch. +// +// "This sync's lookup" means the lookup happened during THIS sync — not +// necessarily during the same call that emits the replay. A planning call +// may batch-resolve validators for many scopes and hand the verdicts to +// sibling cursors via EnqueuePageTokens page tokens; a replay emitted by such a +// cursor satisfies the invariant. What the invariant forbids is validators +// that outlive the sync (connector-side caches, config, upstream echoes): +// the lookup result must originate from the sync that consumes it. + +// SourceCacheCapability is attached to ConnectorServiceValidateResponse +// annotations to opt in to source-cache replay. Absent or any mode other +// than MODE_READ_WRITE means all source-cache annotations are ignored. +message SourceCacheCapability { + enum Mode { + MODE_UNSPECIFIED = 0; + MODE_DISABLED = 1; + MODE_READ_WRITE = 2; + } + Mode mode = 1; + + // cache_generation is the connector's replay-compatibility generation: + // bump it whenever the connector changes how it computes scopes, + // validators, or rows in a way that makes a PRIOR artifact's cached + // rows unsafe to replay (schema of emitted rows, scope partitioning, + // id construction). The SDK never interprets the value — replay is + // permitted only when the previous artifact recorded a byte-identical + // generation (any mismatch runs the sync cold). Never derive this from + // a semver: compatibility is an explicit declaration, not an + // inference. Empty is a valid (constant) generation. + string cache_generation = 2 [(validate.rules).string = { + ignore_empty: true + max_bytes: 256 + }]; + + // config_fingerprint is an opaque connector-computed digest of every + // configuration and permission input that changes what upstream data + // the connector CAN see (credentials scope, tenant/org selection, + // API-permission grants, include/exclude config). Replay is permitted + // only when the previous artifact recorded a byte-identical + // fingerprint: a config change means cached scopes may cover a + // different universe, and replaying them would resurrect rows the new + // configuration can no longer observe (or hide rows it now can). + // Empty means "connector declares no config sensitivity" and matches + // only empty. + string config_fingerprint = 3 [(validate.rules).string = { + ignore_empty: true + max_bytes: 256 + }]; +} + +// SourceCacheRecord is attached to a list-response page whose rows were +// freshly fetched from upstream. The SDK stamps the page's rows with +// scope_key so a future sync can replay them as a unit. +message SourceCacheRecord { + // Connector-computed stable identifier for the canonical scope. Must be + // byte-stable across syncs for the same logical scope. Prefer an ORDERED + // natural identifier (e.g. the request URL, or "groups/{id}/members") + // over a random hash: scope-index writes lead with this value, so + // identifiers that correlate with fetch order keep index writes nearly + // append-ordered at scale. sourcecache.HashScope is the fallback when no + // compact natural form exists. + string scope_key = 1; + + // Opaque validator to persist for this scope. May be empty on interim + // pages of a multi-page scope (e.g. Graph @odata.nextLink pages); the + // SDK writes the scope's manifest entry when a non-empty cache_validator arrives. + // A 200 response with zero rows still persists the entry. + string cache_validator = 2; + + // Tombstones applied after this page's rows commit. Lets every page of + // a multi-page delta round carry its own deletions as the provider + // delivers them, instead of buffering a whole round onto the first + // (replay-annotated) page. Same formats as SourceCacheReplay. + // + // PRECONDITION for tombstones anywhere in a round: the provider's delta + // must be coalesced — at most one add-or-tombstone per object per round + // (Microsoft Graph guarantees this by returning final object state). + // With interleaved add/remove events for one object, per-page ordering + // is deterministic (a page's rows upsert before its deletions apply) + // but cross-page re-adds after a tombstone are the connector's + // responsibility to order. + repeated string deleted_ids = 3; + + // Principal-scoped grant tombstones: for RowKindGrants pages, each + // entry deletes EVERY grant row stamped with this scope whose principal + // id equals the entry — no principal resource type and no canonical + // grant-id reconstruction required (delta tombstones usually carry only + // a bare object id, and the object may no longer exist to look up). + // + // PRECONDITION: the scope must be partitioned so that "principal + // removed from scope" means every grant they have in the scope is gone + // — one scope per navigation with independent removal semantics (e.g. + // members and owners of a group are separate scopes, or membership + // removal would take the owner grant with it). + // + // For RowKindResources pages, each entry deletes the resource row(s) + // stamped with this scope whose resource id equals the entry, any + // resource type. + repeated string deleted_principal_ids = 4; +} + +// SourceCacheReplay is attached to a list-response page to tell the SDK to +// copy the previous sync's rows for scope_key into the current sync. +// +// The row kind (resources, entitlements, grants) is determined by which +// RPC the annotation arrived on, never by the annotation itself. +// +// The connector must only emit this for a scope whose validator it received +// from the SDK's source-cache lookup during this same sync. A replay for +// an unknown scope fails the sync: the connector has already skipped row +// generation, so there is nothing to fall back to. +message SourceCacheReplay { + string scope_key = 1; + + // Validator to persist for this scope in the current sync. For an HTTP + // 304 this is the unchanged validator. For a delta query this is the NEW + // token; it may instead be supplied by the final overlay page's + // SourceCacheRecord.cache_validator, in which case this field may be left empty. + string cache_validator = 2; + + // When true, the response (and subsequent pages carrying + // SourceCacheRecord with the same scope_key) contains changed rows to + // upsert on top of the replayed base. When false the response must + // contain no rows for this scope. + // + // TRANSITIONAL: the SDK currently tolerates rows on a non-overlay + // replay page — it warns and upserts them with overlay semantics — + // while the model is validated against real providers. Do not rely on + // this: the tolerance will become a hard error, because a connector + // that fetched fresh rows and still attached a replay annotation keeps + // resurrecting the replayed base every sync (upstream deletions never + // propagate). + bool overlay = 3; + + // Public canonical IDs (grant/entitlement IDs, or resource BIDs for + // RowKindResources) to delete from the current sync after the replay + // copy and this page's upserts. Used for delta-query tombstones (e.g. + // Microsoft Graph @removed entries). Subsequent pages of the round + // carry their tombstones on SourceCacheRecord.deleted_ids. + repeated string deleted_ids = 4; + + // Principal-scoped tombstones for this page; see + // SourceCacheRecord.deleted_principal_ids for semantics and + // preconditions. + repeated string deleted_principal_ids = 5; +} + +// ----------------------------------------------------------------------- +// Lookup continuation (ask/answer) — the source-cache lookup for runtimes +// that cannot call back to the syncer (single-shot request/response +// transports, e.g. gRPC-over-Lambda). See +// docs/tasks/source-cache-lambda-lookup.md. +// +// syncer connector +// ── list req + LookupOffer ───────▶ +// lookup deferred: collect scopes +// ◀── resp{SourceCacheLookupAsk} ─── +// resolve locally (previous c1z) +// ── SAME req + LookupOffer +// + SourceCacheLookupAnswers ─▶ +// lookup served from answers +// ◀── rows + Scope / Replay ──────── (normal page handling) +// +// The connector must only defer (emit an ask) when the request carried +// SourceCacheLookupOffer — old syncers never send it, so version skew +// degrades to a cold sync instead of a misinterpreted response. +// ----------------------------------------------------------------------- + +// SourceCacheLookupOffer is attached by the SDK to list REQUESTS when the +// syncer can answer source-cache lookup asks: the connector declared +// SourceCacheCapability and a warm previous-sync lookup is installed. +// Its presence is the connector's permission to answer with +// SourceCacheLookupAsk instead of rows. Connectors with a direct lookup +// (in-process, subprocess) never need to defer and may ignore it. +message SourceCacheLookupOffer {} + +// SourceCacheLookupAsk is attached to a list RESPONSE in place of rows: +// the connector needs previous-sync validators before it can serve the +// page. An ask response must carry NO rows, NO next page token, and no +// other source-cache annotations; the syncer consumes it (it never +// reaches page handling), resolves every query, and re-invokes the same +// request with SourceCacheLookupAnswers attached. +// +// Only legal on responses to requests that carried +// SourceCacheLookupOffer. Bounces are capped per REQUEST (a page-token +// advance is a new request and resets the cap, so a multi-page action +// may legally ask once per page); a connector that keeps asking without +// progressing past the cap fails the sync loudly. +message SourceCacheLookupAsk { + message Query { + // One of the sourcecache.RowKind values: "resources", + // "entitlements", "grants". + string row_kind = 1 [(validate.rules).string = { + min_bytes: 1 + max_bytes: 64 + }]; + string scope_key = 2 [(validate.rules).string = { + min_bytes: 1 + max_bytes: 256 + }]; + } + repeated Query queries = 1 [(validate.rules).repeated = { + min_items: 1 + max_items: 4096 + }]; +} + +// SourceCacheLookupAnswers is attached by the SDK to the re-invoked +// REQUEST, carrying the resolution of a prior ask's queries. +// +// Every query of the prior ask is answered — found=true with the +// previous sync's validator, or found=false meaning fetch fresh. A found +// answer whose validator does not fit the per-request answer size budget is +// delivered as found=false: the scope degrades to a cold fetch +// (correct, just slower) instead of staying unresolved, because answers +// accumulate across bounces and a validator that did not fit once can never +// fit a later re-invoke. An ABSENT query (a phase-2 lookup for a scope +// the connector did not ask before) means unresolved: the connector may +// ask again, subject to the bounce cap. +message SourceCacheLookupAnswers { + message Answer { + string row_kind = 1 [(validate.rules).string = { + min_bytes: 1 + max_bytes: 64 + }]; + string scope_key = 2 [(validate.rules).string = { + min_bytes: 1 + max_bytes: 256 + }]; + bool found = 3; + // The previous sync's validator; empty when found is false. Cap + // matches the lookup RPC (Graph delta tokens run long). + string cache_validator = 4 [(validate.rules).string = { + ignore_empty: true + max_bytes: 65536 + }]; + } + // Cap = ask max_items (4096) × the per-request bounce cap (4): answers + // accumulate across bounces, so a request may legally carry every + // answer its asks produced. Keep in sync with + // sourcecache.MaxLookupBouncesPerRequest / maxAnswersPerMessage. + repeated Answer answers = 1 [(validate.rules).repeated = {max_items: 16384}]; +} diff --git a/proto/c1/connector/v2/annotation_type_scoped_entitlements.proto b/proto/c1/connector/v2/annotation_type_scoped_entitlements.proto new file mode 100644 index 000000000..cd2ddb416 --- /dev/null +++ b/proto/c1/connector/v2/annotation_type_scoped_entitlements.proto @@ -0,0 +1,61 @@ +syntax = "proto3"; + +package c1.connector.v2; + +option go_package = "github.com/conductorone/baton-sdk/pb/c1/connector/v2"; + +// Type-scoped entitlement ingestion. +// +// Some providers expose change/enumeration APIs whose natural unit is a +// whole collection (or a connector-defined shard of one), not a single +// resource — e.g. Microsoft Graph delta queries, where one stream yields +// entitlement definitions for up to 50 groups. Forcing that shape through +// the per-resource ListEntitlements fan-out costs one request per resource +// even when nothing changed. +// +// A resource type annotated with TypeScopedEntitlements is EXCLUDED from +// the per-resource entitlements fan-out. Instead the syncer issues +// ListEntitlements calls carrying this annotation on the REQUEST as the +// routing marker; the request's resource is a self-referential stub +// ({type, type} — wire validation requires a non-empty resource id and +// the stub identifies the type, never a real resource). The connector +// answers with entitlements for the whole type, across as many paginated +// cursors as it chooses to spawn (see EnqueuePageTokens in +// annotation_type_scoped_grants.proto). +// +// This is the entitlements-phase analogue of TypeScopedGrants: the +// connector takes over enumeration for the type, and every downstream +// behavior (storage, source-cache scopes/replay/tombstones, exclusion- +// group validation, rate limiting, page-token checkpointing) is unchanged +// because entitlements are self-describing rows. +// +// COMPLETENESS CONTRACT: a full sync of an annotated type must emit (or +// replay, via source-cache annotations) EVERY entitlement of that type. +// The syncer no longer visits each resource, so completeness rests +// entirely on the connector's enumeration. +// +// SKIP ANNOTATIONS: for an annotated type the syncer does not consult +// shouldSkipEntitlements on the type-scoped path. The connector owns skip +// semantics for resources of that type (omit rows; do not rely on +// per-resource SkipEntitlements* annotations to suppress the type-scoped +// call). +// +// ZERO-RESOURCE TYPES: the planner still enqueues the type-scoped action +// when the store holds no resources of the type. The connector must +// return an empty page (or EnqueuePageTokens that resolve empty), not an +// error — an empty type is a valid outcome. +// +// TARGETED SYNC: SyncTargetedResource must not enqueue a per-resource +// SyncEntitlementsOp for a type-scoped type. The type-scoped planner +// action is the only legal entitlements path; a targeted sync of one +// resource of an annotated type syncs the resource (and children) but +// leaves type-scoped entitlement enumeration to a full entitlements +// phase. +// +// ROUTING MARKER: TypeScopedEntitlements routes the ListEntitlements +// call; it implies no repair machinery of its own. It IS listed in the +// ingestion-invariant side-effect coverage map, covered by invariant I7 +// (entitlement→resource referential integrity): type-granularity scopes +// can carry entitlement rows for resources that vanished between +// enumerations, and the post-collection check is what catches that. +message TypeScopedEntitlements {} diff --git a/proto/c1/connector/v2/annotation_type_scoped_grants.proto b/proto/c1/connector/v2/annotation_type_scoped_grants.proto new file mode 100644 index 000000000..3b8a19c74 --- /dev/null +++ b/proto/c1/connector/v2/annotation_type_scoped_grants.proto @@ -0,0 +1,111 @@ +syntax = "proto3"; + +package c1.connector.v2; + +import "validate/validate.proto"; + +option go_package = "github.com/conductorone/baton-sdk/pb/c1/connector/v2"; + +// Type-scoped grant ingestion. +// +// Some providers expose change/enumeration APIs whose natural unit is a +// whole collection (or a connector-defined shard of one), not a single +// resource — e.g. Microsoft Graph delta queries, where one stream yields +// membership changes for up to 50 groups. Forcing that shape through the +// per-resource ListGrants fan-out costs one request per resource even when +// nothing changed. +// +// A resource type annotated with TypeScopedGrants is EXCLUDED from the +// per-resource grants fan-out. Instead the syncer issues ListGrants calls +// carrying this annotation on the REQUEST as the routing marker; the +// request's resource is a self-referential stub ({type, type} — wire +// validation requires a non-empty resource id and the stub identifies +// the type, never a real resource). The connector answers with grants +// for the whole type, across as many paginated cursors as it chooses to +// spawn (see EnqueuePageTokens). +// +// The connector takes over enumeration for the type (the same contract +// as TypeScopedEntitlements in +// annotation_type_scoped_entitlements.proto), and every downstream behavior +// (storage, source-cache scopes/replay/tombstones, grant expansion, rate +// limiting, page-token checkpointing) is unchanged because grants are +// self-describing rows. +// +// COMPLETENESS CONTRACT: a full sync of an annotated type must emit (or +// replay, via source-cache annotations) EVERY grant of that type. The +// syncer no longer visits each resource, so completeness rests entirely on +// the connector's enumeration. +// +// SKIP ANNOTATIONS: for an annotated type the syncer does not consult +// shouldSkipGrants on the type-scoped path. The connector owns skip +// semantics for resources of that type. +// +// ZERO-RESOURCE TYPES: the planner still enqueues the type-scoped action +// when the store holds no resources of the type. The connector must +// return an empty page (or EnqueuePageTokens that resolve empty), not an error. +// +// TARGETED SYNC: SyncTargetedResource must not enqueue a per-resource +// SyncGrantsOp for a type-scoped type. Type-scoped grant enumeration is +// left to a full grants phase. +// +// ROUTING MARKER: TypeScopedGrants routes ListGrants; it implies no +// repair machinery of its own. It IS listed in the ingestion-invariant +// side-effect coverage map, covered by invariant I8 (grant→entitlement +// referential integrity): independently-fresh type scopes can strand +// grant references against a refreshed entitlement set, and the +// post-collection check is what catches that. +message TypeScopedGrants {} + +// EnqueuePageTokens is attached to a ListGrants or ListEntitlements response to +// enqueue additional independent sibling cursors. Each token is delivered +// back to the connector as the page token of its own action — scheduled by +// the syncer's worker pool, rate-limited, and checkpointed like any other +// pagination. Honored on BOTH type-scoped and per-resource ListGrants / +// ListEntitlements responses; the spawned actions inherit the response's +// resource identity (type only for type-scoped calls, type+resource for +// per-resource calls). +// +// Typical uses: +// +// - Type-scoped planning: the first call computes shard assignments — +// e.g. one 50-id delta filter chunk per cursor — returns no rows, and +// spawns one cursor per shard. Cold enumeration then parallelizes +// across shards instead of serializing through one stream. +// - Per-resource parallel warm revalidation (page-numbered APIs): on the +// first page of a collection the connector already knows every other +// page's URL and stored validator, so it answers page one and spawns +// pages 2..N; the worker pool revalidates them concurrently instead +// of paying one round trip per page serially. +// +// Spawned cursors are ORDINARY pages that happen to be enqueued eagerly. +// The SDK assumes nothing about how they resolve: a spawned page may hit +// its source-cache lookup and replay, miss and fetch cold (e.g. a page +// boundary shifted since the last sync), and may continue a chain via its +// response's next page token. Any page of a fan-out may itself spawn. +// +// Progress accounting counts a resource as covered when its ORIGIN +// action's chain ends; spawned siblings never double-count it. +// +// Tokens are opaque to the SDK. They must be self-contained: a cursor may +// execute after a suspend/resume, on a different worker, so anything the +// connector needs to serve the cursor must be inside the token (or +// re-derivable from it) — for per-resource spawns the resource identity +// rides the action, so tokens only need the page coordinate. +message EnqueuePageTokens { + // Page tokens for the sibling cursors to enqueue, one action each. + // Capped per response (spawned actions persist in the checkpointed + // state token); chain additional spawns across pages to fan out wider. + repeated string page_tokens = 1 [(validate.rules).repeated = { + max_items: 1024 + items: { + string: { + min_len: 1 + max_len: 1048576 + } + } + }]; + + // Optional connector estimate of total rows across all cursors of this + // type, for progress reporting. Zero means unknown. + int64 estimated_total = 2; +} diff --git a/proto/c1/connector/v2/resource.proto b/proto/c1/connector/v2/resource.proto index 4a4cecd33..2b68838e4 100644 --- a/proto/c1/connector/v2/resource.proto +++ b/proto/c1/connector/v2/resource.proto @@ -41,7 +41,9 @@ message ResourceType { } repeated Trait traits = 3 [(validate.rules).repeated = { unique: true - items: {enum: {defined_only: true}} + items: { + enum: {defined_only: true} + } }]; repeated google.protobuf.Any annotations = 4; string description = 5 [(validate.rules).string = { diff --git a/proto/c1/connectorapi/baton/v1/source_cache.proto b/proto/c1/connectorapi/baton/v1/source_cache.proto new file mode 100644 index 000000000..37a510b2c --- /dev/null +++ b/proto/c1/connectorapi/baton/v1/source_cache.proto @@ -0,0 +1,55 @@ +syntax = "proto3"; + +package c1.connectorapi.baton.v1; + +import "validate/validate.proto"; + +option go_package = "gitlab.com/ductone/c1/pkg/pb/c1/connectorapi/baton/v1"; + +// BatonSourceCacheService is the dedicated parent-side RPC the connector +// subprocess uses to ask "do I have a previous validator (HTTP ETag / delta +// token) for this scope?" before revalidating upstream. +// +// This is intentionally NOT routed through the session-store gRPC service: +// session data goes through the connector's local MemorySessionCache +// (otter), which would subject sync-scoped validator state to generic +// TTL/eviction policies and burn bounded cache weight on it. A dedicated +// service keeps the message shape explicit and the path uncached. +// +// The parent has exactly one active lookup registered at a time (set +// per-sync, cleared at sync end), so no sync_id travels on the wire. +service BatonSourceCacheService { + rpc Lookup(LookupRequest) returns (LookupResponse) {} +} + +message LookupRequest { + // Row kind: resources / entitlements / grants + // (pkg/sourcecache.RowKind values). Entries are partitioned by row + // kind, so one scope key can carry a different validator per kind. + string row_kind = 1 [(validate.rules).string = { + min_len: 1 + max_len: 64 + }]; + + // Connector-defined stable scope identifier (a stable identifier for the canonical scope + // (often a hex hash; see pkg/sourcecache.HashScope). Opaque to the + // parent; matched verbatim against the previous sync's source-cache + // entries. + string scope_key = 2 [(validate.rules).string = { + min_len: 1 + max_len: 256 + }]; +} + +message LookupResponse { + // False means no prior entry exists for (row_kind, scope_key): the + // connector must fetch fresh and must not emit SourceCacheReplay for + // this scope. + bool found = 1; + + // The opaque validator the previous sync recorded for this scope (HTTP + // ETag, delta token, ...). Empty when found is false. The cap is a + // sanity bound sized for Microsoft Graph delta tokens, which are known + // to run to thousands of characters; storage imposes no limit. + string cache_validator = 2 [(validate.rules).string = {max_len: 65536}]; +} diff --git a/proto/c1/storage/v3/records.proto b/proto/c1/storage/v3/records.proto index bdd167640..0f10bf5ee 100644 --- a/proto/c1/storage/v3/records.proto +++ b/proto/c1/storage/v3/records.proto @@ -103,6 +103,13 @@ message ResourceRecord { // into `annotations` as Any (the v2 connector contract — see // pkg/types/resource/user_trait.go). The annotations field above // round-trips them lossless; no extra field is needed. + + // Source-cache scope stamp (see annotation_source_cache.proto). + // Empty for rows not produced under a source-cache scope. + string source_scope_key = 9 [(index) = { + name: "by_source_scope" + partial: "source_scope_key != ''" + }]; } message EntitlementRecord { @@ -142,6 +149,13 @@ message EntitlementRecord { // resource_types table when needed. Consumed by // pkg/sync/syncer.go's principal-type narrowing. repeated string grantable_to_resource_type_ids = 10; + + // Source-cache scope stamp (see annotation_source_cache.proto). + // Empty for rows not produced under a source-cache scope. + string source_scope_key = 11 [(index) = { + name: "by_source_scope" + partial: "source_scope_key != ''" + }]; } message GrantRecord { @@ -188,6 +202,16 @@ message GrantRecord { // map — same wire shape as // c1.connector.v2.GrantSources.sources. map sources = 9; + + // Source-cache scope stamp (see annotation_source_cache.proto). + // Empty for rows not produced under a source-cache scope — notably + // expander-derived grants, which are recreated by expansion each sync + // and never replayed. StoreExpandedGrants preserves this field on + // rewrites of existing rows, exactly like expansion/needs_expansion. + string source_scope_key = 10 [(index) = { + name: "by_source_scope" + partial: "source_scope_key != ''" + }]; } message AssetRecord { @@ -220,6 +244,21 @@ message SyncRunRecord { string sync_token = 6; bool supports_diff = 7; string linked_sync_id = 8; + + // compacted marks a sync produced by compaction (fold or rebuild) + // rather than by a real connector run. Compacted artifacts are + // keep-newer UPSERT merges — base rows a newer input deleted survive — + // so no input sync's upstream validators (source-cache validators) describe + // their contents. + // + // "Can this sync be used as a source-cache replay source?" is the + // predicate c1zstore.SyncRun.UsableAsReplaySource: type == FULL and + // !compacted. The syncer enforces it — feeding a compacted or + // partial/derived sync to WithPreviousSyncC1ZPath degrades to a cold + // (full-fetch) sync. Orchestrators deciding which artifact to + // materialize as the previous sync should apply the same predicate + // instead of guessing from provenance. + bool compacted = 9; } // SyncStatsRecord is the engine-internal sidecar populated at @@ -250,10 +289,89 @@ message SyncStatsRecord { message SessionRecord { option (table) = { name: "sessions" - primary_key: ["sync_id", "key"] + primary_key: [ + "sync_id", + "key" + ] }; string sync_id = 1; string key = 2; bytes value = 3; } + +// SourceCacheEntryRecord is the per-scope manifest entry for source-cache +// replay (see c1/connector/v2/annotation_source_cache.proto). One entry +// per (row_kind, scope_key), written for every freshly fetched scope — +// including zero-row responses — and rewritten on replay with the scope's +// current validator. The previous sync's entries (read from the previous +// c1z) are the lookup surface connectors revalidate against. +message SourceCacheEntryRecord { + option (table) = { + name: "source_cache_entries" + primary_key: [ + "row_kind", + "scope_key" + ] + }; + + // Row kind partition: "resources", "entitlements", or "grants" + // (pkg/sourcecache.RowKind values). + string row_kind = 1; + + // Connector-computed stable scope identifier (conventionally lowercase + // hex when hashed via sourcecache.HashScope; any byte-stable string). + string scope_key = 2; + + // Opaque upstream validator: HTTP ETag, delta token, etc. + string cache_validator = 3; + + google.protobuf.Timestamp discovered_at = 4; + + // Set when the dangling-reference drops (ingestion invariants I7/I8/I9, + // pkg/sync/ingest_invariants.go) deleted rows stamped with this scope: + // the artifact no longer holds the full row set the validator vouches + // for, so a future sync must not 304-replay it. Lookups treat an + // invalidated entry as a miss (the scope re-fetches cold and converges + // with a cold sync); the entry itself is kept so the scope's surviving + // stamped rows do not read as an I6 orphan (lost manifest write). + bool invalidated = 5; +} + +// SourceCacheCompatRecord is the sync's replay-compatibility key: the +// exact conditions under which this artifact's source-cache manifest was +// recorded. A future sync may replay from this artifact ONLY when its +// own key matches byte-exactly on every component; an absent, unreadable, +// or mismatched record degrades that sync to cold (no-op lookup). +// Compatibility is never inferred from versions — each component is an +// explicit declaration by its owner. +// +// One record per sync, written when the write side of the source cache +// enables (before any manifest entry), cleared by compaction folds with +// the manifest. +message SourceCacheCompatRecord { + option (table) = { + name: "source_cache_compat" + primary_key: ["id"] + }; + + // Fixed singleton id ("compat"); present so the record satisfies the + // table shape. + string id = 1; + + // Connector-declared components (SourceCacheCapability annotations): + // the connector's cache generation and configuration/permission + // fingerprint. + string connector_cache_generation = 2; + string connector_config_fingerprint = 3; + + // SDK materialization-policy generation: bumped when the SDK changes + // how response rows or their side effects materialize into the store + // in a replay-visible way (sourcecache.MaterializationPolicyGeneration). + string sdk_materialization_generation = 4; + + // SDK sync-selection fingerprint: digest of the sync-shaping inputs + // (enabled resource types, skip flags) — a selection change means the + // prior artifact's scopes cover a different row universe. + string sync_selection_fingerprint = 5; +}