From ffe3faf72674bd3dad3e3b00c6854ef8ba0cc333 Mon Sep 17 00:00:00 2001 From: Matt Kaniaris Date: Tue, 14 Jul 2026 16:28:29 -0600 Subject: [PATCH 01/17] c1z source-cache replay framework, type-scoped grants, and sync stats Connector-driven source-cache replay for pebble c1z stores: scope manifests with ETag revalidation, engine-side row replay with overlay and delete tombstones, continuation (ask/answer) lookups for single-shot transports, type-scoped grants via SpawnCursors, sync run stats, and the replay-equivalence differential harness together with the replay side-effect bug fixes it drove out (child scheduling, external-match grants, related-resource rows, exclusion groups, continuation caps). Co-authored-by: Cursor --- internal/connector/connector.go | 58 +- .../v2/annotation_source_cache.pb.go | 919 +++++++++ .../v2/annotation_source_cache.pb.validate.go | 1003 +++++++++ .../annotation_source_cache_protoopaque.pb.go | 865 ++++++++ .../v2/annotation_type_scoped_grants.pb.go | 255 +++ ...notation_type_scoped_grants.pb.validate.go | 264 +++ ...ation_type_scoped_grants_protoopaque.pb.go | 250 +++ .../connectorapi/baton/v1/source_cache.pb.go | 249 +++ .../baton/v1/source_cache.pb.validate.go | 271 +++ .../baton/v1/source_cache_grpc.pb.go | 145 ++ .../baton/v1/source_cache_protoopaque.pb.go | 235 +++ pb/c1/storage/v3/records.pb.go | 380 +++- pb/c1/storage/v3/records.pb.validate.go | 145 ++ pb/c1/storage/v3/records_protoopaque.pb.go | 368 +++- pkg/cli/cli.go | 8 +- pkg/cli/commands.go | 144 +- pkg/cli/connector_default_test.go | 77 + pkg/config/config.go | 21 + pkg/connectorbuilder/connectorbuilder.go | 76 +- pkg/connectorbuilder/continuation_test.go | 222 ++ pkg/connectorbuilder/resource_syncer.go | 183 +- .../source_cache_slot_test.go | 56 + pkg/connectorrunner/runner.go | 38 +- pkg/dotc1z/c1zstore/grant_store.go | 9 + pkg/dotc1z/c1zstore/sync_meta.go | 11 +- pkg/dotc1z/engine/pebble/adapter.go | 48 +- .../engine/pebble/adapter_grants_store.go | 6 + pkg/dotc1z/engine/pebble/adapter_sync_meta.go | 2 + pkg/dotc1z/engine/pebble/bulk_import.go | 2 + pkg/dotc1z/engine/pebble/cleanup.go | 4 + pkg/dotc1z/engine/pebble/engine.go | 20 +- pkg/dotc1z/engine/pebble/entitlements.go | 56 +- pkg/dotc1z/engine/pebble/grants.go | 41 +- .../engine/pebble/grants_synth_encode.go | 8 + .../engine/pebble/grants_synth_encode_test.go | 21 +- .../engine/pebble/id_index_migration.go | 2 +- pkg/dotc1z/engine/pebble/if_newer.go | 14 + pkg/dotc1z/engine/pebble/inspect_diff_test.go | 2 +- pkg/dotc1z/engine/pebble/keys.go | 154 +- pkg/dotc1z/engine/pebble/lookup.go | 32 +- pkg/dotc1z/engine/pebble/raw_records.go | 306 ++- pkg/dotc1z/engine/pebble/resources.go | 23 +- pkg/dotc1z/engine/pebble/source_cache.go | 1173 +++++++++++ .../engine/pebble/source_cache_engine_test.go | 529 +++++ pkg/dotc1z/source_cache.go | 213 ++ pkg/field/defaults.go | 15 + pkg/field/fields.go | 28 + pkg/lambda/grpc/continuation_wire_test.go | 144 ++ pkg/lambda/grpc/wire.go | 7 +- pkg/sourcecache/context.go | 20 + pkg/sourcecache/continuation.go | 285 +++ pkg/sourcecache/grpc_lookup.go | 91 + pkg/sourcecache/grpc_lookup_test.go | 145 ++ pkg/sourcecache/grpc_server.go | 83 + pkg/sourcecache/sourcecache.go | 140 ++ pkg/sync/parallel_syncer.go | 18 + pkg/sync/progresslog/progresslog.go | 34 +- pkg/sync/source_cache.go | 537 +++++ pkg/sync/source_cache_continuation.go | 229 +++ .../source_cache_continuation_e2e_test.go | 257 +++ pkg/sync/source_cache_continuation_test.go | 780 +++++++ pkg/sync/source_cache_equivalence_test.go | 1827 +++++++++++++++++ pkg/sync/source_cache_sideeffects_test.go | 776 +++++++ pkg/sync/source_cache_stats.go | 143 ++ pkg/sync/source_cache_stats_test.go | 115 ++ pkg/sync/source_cache_test.go | 568 +++++ pkg/sync/spawn_cursors_test.go | 580 ++++++ pkg/sync/state.go | 21 + pkg/sync/syncer.go | 515 ++++- pkg/sync/type_scoped_grants_test.go | 534 +++++ pkg/synccompactor/compactor_pebble.go | 19 + pkg/synccompactor/pebble/bucket_plans.go | 28 + pkg/synccompactor/source_cache_replay_test.go | 180 ++ pkg/tasks/local/syncer.go | 8 + pkg/types/resource/resource.go | 12 +- .../v2/annotation_source_cache.proto | 250 +++ .../v2/annotation_type_scoped_grants.proto | 87 + proto/c1/connector/v2/resource.proto | 4 +- .../connectorapi/baton/v1/source_cache.proto | 55 + proto/c1/storage/v3/records.proto | 68 +- 80 files changed, 17132 insertions(+), 379 deletions(-) create mode 100644 pb/c1/connector/v2/annotation_source_cache.pb.go create mode 100644 pb/c1/connector/v2/annotation_source_cache.pb.validate.go create mode 100644 pb/c1/connector/v2/annotation_source_cache_protoopaque.pb.go create mode 100644 pb/c1/connector/v2/annotation_type_scoped_grants.pb.go create mode 100644 pb/c1/connector/v2/annotation_type_scoped_grants.pb.validate.go create mode 100644 pb/c1/connector/v2/annotation_type_scoped_grants_protoopaque.pb.go create mode 100644 pb/c1/connectorapi/baton/v1/source_cache.pb.go create mode 100644 pb/c1/connectorapi/baton/v1/source_cache.pb.validate.go create mode 100644 pb/c1/connectorapi/baton/v1/source_cache_grpc.pb.go create mode 100644 pb/c1/connectorapi/baton/v1/source_cache_protoopaque.pb.go create mode 100644 pkg/cli/connector_default_test.go create mode 100644 pkg/connectorbuilder/continuation_test.go create mode 100644 pkg/connectorbuilder/source_cache_slot_test.go create mode 100644 pkg/dotc1z/engine/pebble/source_cache.go create mode 100644 pkg/dotc1z/engine/pebble/source_cache_engine_test.go create mode 100644 pkg/dotc1z/source_cache.go create mode 100644 pkg/lambda/grpc/continuation_wire_test.go create mode 100644 pkg/sourcecache/context.go create mode 100644 pkg/sourcecache/continuation.go create mode 100644 pkg/sourcecache/grpc_lookup.go create mode 100644 pkg/sourcecache/grpc_lookup_test.go create mode 100644 pkg/sourcecache/grpc_server.go create mode 100644 pkg/sourcecache/sourcecache.go create mode 100644 pkg/sync/source_cache.go create mode 100644 pkg/sync/source_cache_continuation.go create mode 100644 pkg/sync/source_cache_continuation_e2e_test.go create mode 100644 pkg/sync/source_cache_continuation_test.go create mode 100644 pkg/sync/source_cache_equivalence_test.go create mode 100644 pkg/sync/source_cache_sideeffects_test.go create mode 100644 pkg/sync/source_cache_stats.go create mode 100644 pkg/sync/source_cache_stats_test.go create mode 100644 pkg/sync/source_cache_test.go create mode 100644 pkg/sync/spawn_cursors_test.go create mode 100644 pkg/sync/type_scoped_grants_test.go create mode 100644 pkg/synccompactor/source_cache_replay_test.go create mode 100644 proto/c1/connector/v2/annotation_source_cache.proto create mode 100644 proto/c1/connector/v2/annotation_type_scoped_grants.proto create mode 100644 proto/c1/connectorapi/baton/v1/source_cache.proto diff --git a/internal/connector/connector.go b/internal/connector/connector.go index 04170efd6..066fcff7d 100644 --- a/internal/connector/connector.go +++ b/internal/connector/connector.go @@ -23,11 +23,13 @@ import ( connectorV2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" connectorwrapperV1 "github.com/conductorone/baton-sdk/pb/c1/connector_wrapper/v1" + batonV1 "github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1" ratelimitV1 "github.com/conductorone/baton-sdk/pb/c1/ratelimit/v1" tlsV1 "github.com/conductorone/baton-sdk/pb/c1/utls/v1" "github.com/conductorone/baton-sdk/pkg/bid" ratelimit2 "github.com/conductorone/baton-sdk/pkg/ratelimit" "github.com/conductorone/baton-sdk/pkg/session" + "github.com/conductorone/baton-sdk/pkg/sourcecache" "github.com/conductorone/baton-sdk/pkg/types" "github.com/conductorone/baton-sdk/pkg/types/sessions" "github.com/conductorone/baton-sdk/pkg/ugrpc" @@ -55,19 +57,42 @@ type connectorClient struct { connectorV2.ActionServiceClient sessionStoreSetter sessions.SetSessionStore // this is the session store server + sourceCacheSetter sourcecache.SetLookup // this is the source-cache lookup server } var _ sessions.SetSessionStore = (*connectorClient)(nil) +var _ sourcecache.SetLookup = (*connectorClient)(nil) var _ SetSessionStoreSetter = (*connectorClient)(nil) +var _ SetSourceCacheSetter = (*connectorClient)(nil) type SetSessionStoreSetter interface { SetSessionStoreSetter(setsessionStoreSetter sessions.SetSessionStore) } +type SetSourceCacheSetter interface { + SetSourceCacheSetter(sourceCacheSetter sourcecache.SetLookup) +} + func (c *connectorClient) SetSessionStoreSetter(sessionStoreSetter sessions.SetSessionStore) { c.sessionStoreSetter = sessionStoreSetter } +func (c *connectorClient) SetSourceCacheSetter(sourceCacheSetter sourcecache.SetLookup) { + c.sourceCacheSetter = sourceCacheSetter +} + +// SetSourceCache forwards the syncer's per-sync lookup to whatever +// receives it: the subprocess-mode BatonSourceCacheService server, or the +// in-process builder. A nil setter means the connector never opted in; +// the syncer calls this unconditionally, so stay quiet at debug level. +func (c *connectorClient) SetSourceCache(ctx context.Context, lookup sourcecache.Lookup) { + if c.sourceCacheSetter == nil { + ctxzap.Extract(ctx).Debug("connectorClient's source cache setter is nil — connector did not opt into source caching") + return + } + c.sourceCacheSetter.SetSourceCache(ctx, lookup) +} + func (c *connectorClient) SetSessionStore(ctx context.Context, store sessions.SessionStore) { if c.sessionStoreSetter == nil { // Demoted from Warn to Debug: this path is the normal case for @@ -109,7 +134,8 @@ type wrapper struct { now func() time.Time - SessionServer sessions.SetSessionStore + SessionServer sessions.SetSessionStore + SourceCacheServer sourcecache.SetLookup } type Option func(ctx context.Context, w *wrapper) error @@ -197,6 +223,12 @@ func NewWrapper(ctx context.Context, server interface{}, opts ...Option) (*wrapp server: connectorServer, now: time.Now, } + // In-process delivery: a connectorbuilder-based server implements + // sourcecache.SetLookup itself, so the syncer's per-sync lookup can be + // installed without the subprocess gRPC hop. + if sourceCacheServer, ok := connectorServer.(sourcecache.SetLookup); ok { + w.SourceCacheServer = sourceCacheServer + } for _, o := range opts { err := o(ctx, w) @@ -276,18 +308,29 @@ func (cw *wrapper) runServer(ctx context.Context, serverCred *tlsV1.Credential) return 0, fmt.Errorf("failed to create session listener config: %w", err) } - // TODO(kans): block until we send a request or something/error handling in general. + // One listener serves BatonSessionService (connector session data) + // and BatonSourceCacheService (source-cache scope lookups). Keeping + // them as separate RPCs keeps validator lookups out of the + // connector's local MemorySessionCache and its TTL/eviction rules. l.Info("starting session store server") - server := session.NewGRPCSessionServer() - cw.SessionServer = server + sessionServer := session.NewGRPCSessionServer() + sourceCacheServer := sourcecache.NewGRPCServer() + cw.SessionServer = sessionServer + cw.SourceCacheServer = sourceCacheServer go func() { defer sessionListener.Close() - serverErr := session.StartGRPCSessionServerWithOptions(ctx, sessionListener, server, + grpcServer := grpc.NewServer( grpc.Creds(credentials.NewTLS(tlsConfig)), grpc.ChainUnaryInterceptor(ugrpc.UnaryServerInterceptor(ctx)...), ) - if serverErr != nil { - l.Error("failed to create session store server", zap.Error(serverErr)) + batonV1.RegisterBatonSessionServiceServer(grpcServer, sessionServer) + batonV1.RegisterBatonSourceCacheServiceServer(grpcServer, sourceCacheServer) + go func() { + <-ctx.Done() + grpcServer.GracefulStop() + }() + if serveErr := grpcServer.Serve(sessionListener); serveErr != nil { + l.Error("session/source-cache server stopped", zap.Error(serveErr)) return } }() @@ -440,6 +483,7 @@ func (cw *wrapper) C(ctx context.Context) (types.ConnectorClient, error) { cw.conn = conn client := NewConnectorClient(ctx, cw.conn) client.SetSessionStoreSetter(cw.SessionServer) + client.SetSourceCacheSetter(cw.SourceCacheServer) cw.client = client return client, nil diff --git a/pb/c1/connector/v2/annotation_source_cache.pb.go b/pb/c1/connector/v2/annotation_source_cache.pb.go new file mode 100644 index 000000000..5e81b2954 --- /dev/null +++ b/pb/c1/connector/v2/annotation_source_cache.pb.go @@ -0,0 +1,919 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.10 +// protoc (unknown) +// source: c1/connector/v2/annotation_source_cache.proto + +//go:build !protoopaque + +package v2 + +import ( + _ "github.com/envoyproxy/protoc-gen-validate/validate" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type SourceCacheCapability_Mode int32 + +const ( + SourceCacheCapability_MODE_UNSPECIFIED SourceCacheCapability_Mode = 0 + SourceCacheCapability_MODE_DISABLED SourceCacheCapability_Mode = 1 + SourceCacheCapability_MODE_READ_WRITE SourceCacheCapability_Mode = 2 +) + +// Enum value maps for SourceCacheCapability_Mode. +var ( + SourceCacheCapability_Mode_name = map[int32]string{ + 0: "MODE_UNSPECIFIED", + 1: "MODE_DISABLED", + 2: "MODE_READ_WRITE", + } + SourceCacheCapability_Mode_value = map[string]int32{ + "MODE_UNSPECIFIED": 0, + "MODE_DISABLED": 1, + "MODE_READ_WRITE": 2, + } +) + +func (x SourceCacheCapability_Mode) Enum() *SourceCacheCapability_Mode { + p := new(SourceCacheCapability_Mode) + *p = x + return p +} + +func (x SourceCacheCapability_Mode) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (SourceCacheCapability_Mode) Descriptor() protoreflect.EnumDescriptor { + return file_c1_connector_v2_annotation_source_cache_proto_enumTypes[0].Descriptor() +} + +func (SourceCacheCapability_Mode) Type() protoreflect.EnumType { + return &file_c1_connector_v2_annotation_source_cache_proto_enumTypes[0] +} + +func (x SourceCacheCapability_Mode) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// SourceCacheCapability is attached to ConnectorServiceValidateResponse +// annotations to opt in to source-cache replay. Absent or any mode other +// than MODE_READ_WRITE means all source-cache annotations are ignored. +type SourceCacheCapability struct { + state protoimpl.MessageState `protogen:"hybrid.v1"` + Mode SourceCacheCapability_Mode `protobuf:"varint,1,opt,name=mode,proto3,enum=c1.connector.v2.SourceCacheCapability_Mode" json:"mode,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SourceCacheCapability) Reset() { + *x = SourceCacheCapability{} + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SourceCacheCapability) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SourceCacheCapability) ProtoMessage() {} + +func (x *SourceCacheCapability) ProtoReflect() protoreflect.Message { + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *SourceCacheCapability) GetMode() SourceCacheCapability_Mode { + if x != nil { + return x.Mode + } + return SourceCacheCapability_MODE_UNSPECIFIED +} + +func (x *SourceCacheCapability) SetMode(v SourceCacheCapability_Mode) { + x.Mode = v +} + +type SourceCacheCapability_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + Mode SourceCacheCapability_Mode +} + +func (b0 SourceCacheCapability_builder) Build() *SourceCacheCapability { + m0 := &SourceCacheCapability{} + b, x := &b0, m0 + _, _ = b, x + x.Mode = b.Mode + return m0 +} + +// SourceCacheScope is attached to a list-response page whose rows were +// freshly fetched from upstream. The SDK stamps the page's rows with +// scope_hash so a future sync can replay them as a unit. +type SourceCacheScope struct { + state protoimpl.MessageState `protogen:"hybrid.v1"` + // Connector-computed stable identifier for the canonical scope. Must be + // byte-stable across syncs for the same logical scope. Prefer an ORDERED + // natural identifier (e.g. the request URL, or "groups/{id}/members") + // over a random hash: scope-index writes lead with this value, so + // identifiers that correlate with fetch order keep index writes nearly + // append-ordered at scale. sourcecache.HashScope is the fallback when no + // compact natural form exists. + ScopeHash string `protobuf:"bytes,1,opt,name=scope_hash,json=scopeHash,proto3" json:"scope_hash,omitempty"` + // Opaque validator to persist for this scope. May be empty on interim + // pages of a multi-page scope (e.g. Graph @odata.nextLink pages); the + // SDK writes the scope's manifest entry when a non-empty etag arrives. + // A 200 response with zero rows still persists the entry. + Etag string `protobuf:"bytes,2,opt,name=etag,proto3" json:"etag,omitempty"` + // Tombstones applied after this page's rows commit. Lets every page of + // a multi-page delta round carry its own deletions as the provider + // delivers them, instead of buffering a whole round onto the first + // (replay-annotated) page. Same formats as SourceCacheReplay. + // + // PRECONDITION for tombstones anywhere in a round: the provider's delta + // must be coalesced — at most one add-or-tombstone per object per round + // (Microsoft Graph guarantees this by returning final object state). + // With interleaved add/remove events for one object, per-page ordering + // is deterministic (a page's rows upsert before its deletions apply) + // but cross-page re-adds after a tombstone are the connector's + // responsibility to order. + DeletedIds []string `protobuf:"bytes,3,rep,name=deleted_ids,json=deletedIds,proto3" json:"deleted_ids,omitempty"` + // Principal-scoped grant tombstones: for RowKindGrants pages, each + // entry deletes EVERY grant row stamped with this scope whose principal + // id equals the entry — no principal resource type and no canonical + // grant-id reconstruction required (delta tombstones usually carry only + // a bare object id, and the object may no longer exist to look up). + // + // PRECONDITION: the scope must be partitioned so that "principal + // removed from scope" means every grant they have in the scope is gone + // — one scope per navigation with independent removal semantics (e.g. + // members and owners of a group are separate scopes, or membership + // removal would take the owner grant with it). + // + // For RowKindResources pages, each entry deletes the resource row(s) + // stamped with this scope whose resource id equals the entry, any + // resource type. + DeletedPrincipalIds []string `protobuf:"bytes,4,rep,name=deleted_principal_ids,json=deletedPrincipalIds,proto3" json:"deleted_principal_ids,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SourceCacheScope) Reset() { + *x = SourceCacheScope{} + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SourceCacheScope) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SourceCacheScope) ProtoMessage() {} + +func (x *SourceCacheScope) ProtoReflect() protoreflect.Message { + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *SourceCacheScope) GetScopeHash() string { + if x != nil { + return x.ScopeHash + } + return "" +} + +func (x *SourceCacheScope) GetEtag() string { + if x != nil { + return x.Etag + } + return "" +} + +func (x *SourceCacheScope) GetDeletedIds() []string { + if x != nil { + return x.DeletedIds + } + return nil +} + +func (x *SourceCacheScope) GetDeletedPrincipalIds() []string { + if x != nil { + return x.DeletedPrincipalIds + } + return nil +} + +func (x *SourceCacheScope) SetScopeHash(v string) { + x.ScopeHash = v +} + +func (x *SourceCacheScope) SetEtag(v string) { + x.Etag = v +} + +func (x *SourceCacheScope) SetDeletedIds(v []string) { + x.DeletedIds = v +} + +func (x *SourceCacheScope) SetDeletedPrincipalIds(v []string) { + x.DeletedPrincipalIds = v +} + +type SourceCacheScope_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + // Connector-computed stable identifier for the canonical scope. Must be + // byte-stable across syncs for the same logical scope. Prefer an ORDERED + // natural identifier (e.g. the request URL, or "groups/{id}/members") + // over a random hash: scope-index writes lead with this value, so + // identifiers that correlate with fetch order keep index writes nearly + // append-ordered at scale. sourcecache.HashScope is the fallback when no + // compact natural form exists. + ScopeHash string + // Opaque validator to persist for this scope. May be empty on interim + // pages of a multi-page scope (e.g. Graph @odata.nextLink pages); the + // SDK writes the scope's manifest entry when a non-empty etag arrives. + // A 200 response with zero rows still persists the entry. + Etag string + // Tombstones applied after this page's rows commit. Lets every page of + // a multi-page delta round carry its own deletions as the provider + // delivers them, instead of buffering a whole round onto the first + // (replay-annotated) page. Same formats as SourceCacheReplay. + // + // PRECONDITION for tombstones anywhere in a round: the provider's delta + // must be coalesced — at most one add-or-tombstone per object per round + // (Microsoft Graph guarantees this by returning final object state). + // With interleaved add/remove events for one object, per-page ordering + // is deterministic (a page's rows upsert before its deletions apply) + // but cross-page re-adds after a tombstone are the connector's + // responsibility to order. + DeletedIds []string + // Principal-scoped grant tombstones: for RowKindGrants pages, each + // entry deletes EVERY grant row stamped with this scope whose principal + // id equals the entry — no principal resource type and no canonical + // grant-id reconstruction required (delta tombstones usually carry only + // a bare object id, and the object may no longer exist to look up). + // + // PRECONDITION: the scope must be partitioned so that "principal + // removed from scope" means every grant they have in the scope is gone + // — one scope per navigation with independent removal semantics (e.g. + // members and owners of a group are separate scopes, or membership + // removal would take the owner grant with it). + // + // For RowKindResources pages, each entry deletes the resource row(s) + // stamped with this scope whose resource id equals the entry, any + // resource type. + DeletedPrincipalIds []string +} + +func (b0 SourceCacheScope_builder) Build() *SourceCacheScope { + m0 := &SourceCacheScope{} + b, x := &b0, m0 + _, _ = b, x + x.ScopeHash = b.ScopeHash + x.Etag = b.Etag + x.DeletedIds = b.DeletedIds + x.DeletedPrincipalIds = b.DeletedPrincipalIds + return m0 +} + +// SourceCacheReplay is attached to a list-response page to tell the SDK to +// copy the previous sync's rows for scope_hash into the current sync. +// +// The row kind (resources, entitlements, grants) is determined by which +// RPC the annotation arrived on, never by the annotation itself. +// +// The connector must only emit this for a scope whose etag it received +// from the SDK's source-cache lookup during this same sync. A replay for +// an unknown scope fails the sync: the connector has already skipped row +// generation, so there is nothing to fall back to. +type SourceCacheReplay struct { + state protoimpl.MessageState `protogen:"hybrid.v1"` + ScopeHash string `protobuf:"bytes,1,opt,name=scope_hash,json=scopeHash,proto3" json:"scope_hash,omitempty"` + // Validator to persist for this scope in the current sync. For an HTTP + // 304 this is the unchanged etag. For a delta query this is the NEW + // token; it may instead be supplied by the final overlay page's + // SourceCacheScope.etag, in which case this field may be left empty. + Etag string `protobuf:"bytes,2,opt,name=etag,proto3" json:"etag,omitempty"` + // When true, the response (and subsequent pages carrying + // SourceCacheScope with the same scope_hash) contains changed rows to + // upsert on top of the replayed base. When false the response must + // contain no rows for this scope. + Overlay bool `protobuf:"varint,3,opt,name=overlay,proto3" json:"overlay,omitempty"` + // Public canonical IDs (grant/entitlement IDs, or resource BIDs for + // RowKindResources) to delete from the current sync after the replay + // copy and this page's upserts. Used for delta-query tombstones (e.g. + // Microsoft Graph @removed entries). Subsequent pages of the round + // carry their tombstones on SourceCacheScope.deleted_ids. + DeletedIds []string `protobuf:"bytes,4,rep,name=deleted_ids,json=deletedIds,proto3" json:"deleted_ids,omitempty"` + // Principal-scoped tombstones for this page; see + // SourceCacheScope.deleted_principal_ids for semantics and + // preconditions. + DeletedPrincipalIds []string `protobuf:"bytes,5,rep,name=deleted_principal_ids,json=deletedPrincipalIds,proto3" json:"deleted_principal_ids,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SourceCacheReplay) Reset() { + *x = SourceCacheReplay{} + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SourceCacheReplay) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SourceCacheReplay) ProtoMessage() {} + +func (x *SourceCacheReplay) ProtoReflect() protoreflect.Message { + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *SourceCacheReplay) GetScopeHash() string { + if x != nil { + return x.ScopeHash + } + return "" +} + +func (x *SourceCacheReplay) GetEtag() string { + if x != nil { + return x.Etag + } + return "" +} + +func (x *SourceCacheReplay) GetOverlay() bool { + if x != nil { + return x.Overlay + } + return false +} + +func (x *SourceCacheReplay) GetDeletedIds() []string { + if x != nil { + return x.DeletedIds + } + return nil +} + +func (x *SourceCacheReplay) GetDeletedPrincipalIds() []string { + if x != nil { + return x.DeletedPrincipalIds + } + return nil +} + +func (x *SourceCacheReplay) SetScopeHash(v string) { + x.ScopeHash = v +} + +func (x *SourceCacheReplay) SetEtag(v string) { + x.Etag = v +} + +func (x *SourceCacheReplay) SetOverlay(v bool) { + x.Overlay = v +} + +func (x *SourceCacheReplay) SetDeletedIds(v []string) { + x.DeletedIds = v +} + +func (x *SourceCacheReplay) SetDeletedPrincipalIds(v []string) { + x.DeletedPrincipalIds = v +} + +type SourceCacheReplay_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + ScopeHash string + // Validator to persist for this scope in the current sync. For an HTTP + // 304 this is the unchanged etag. For a delta query this is the NEW + // token; it may instead be supplied by the final overlay page's + // SourceCacheScope.etag, in which case this field may be left empty. + Etag string + // When true, the response (and subsequent pages carrying + // SourceCacheScope with the same scope_hash) contains changed rows to + // upsert on top of the replayed base. When false the response must + // contain no rows for this scope. + Overlay bool + // Public canonical IDs (grant/entitlement IDs, or resource BIDs for + // RowKindResources) to delete from the current sync after the replay + // copy and this page's upserts. Used for delta-query tombstones (e.g. + // Microsoft Graph @removed entries). Subsequent pages of the round + // carry their tombstones on SourceCacheScope.deleted_ids. + DeletedIds []string + // Principal-scoped tombstones for this page; see + // SourceCacheScope.deleted_principal_ids for semantics and + // preconditions. + DeletedPrincipalIds []string +} + +func (b0 SourceCacheReplay_builder) Build() *SourceCacheReplay { + m0 := &SourceCacheReplay{} + b, x := &b0, m0 + _, _ = b, x + x.ScopeHash = b.ScopeHash + x.Etag = b.Etag + x.Overlay = b.Overlay + x.DeletedIds = b.DeletedIds + x.DeletedPrincipalIds = b.DeletedPrincipalIds + return m0 +} + +// SourceCacheLookupOffer is attached by the SDK to list REQUESTS when the +// syncer can answer source-cache lookup asks: the connector declared +// SourceCacheCapability and a warm previous-sync lookup is installed. +// Its presence is the connector's permission to answer with +// SourceCacheLookupAsk instead of rows. Connectors with a direct lookup +// (in-process, subprocess) never need to defer and may ignore it. +type SourceCacheLookupOffer struct { + state protoimpl.MessageState `protogen:"hybrid.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SourceCacheLookupOffer) Reset() { + *x = SourceCacheLookupOffer{} + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SourceCacheLookupOffer) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SourceCacheLookupOffer) ProtoMessage() {} + +func (x *SourceCacheLookupOffer) ProtoReflect() protoreflect.Message { + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +type SourceCacheLookupOffer_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + +} + +func (b0 SourceCacheLookupOffer_builder) Build() *SourceCacheLookupOffer { + m0 := &SourceCacheLookupOffer{} + b, x := &b0, m0 + _, _ = b, x + return m0 +} + +// SourceCacheLookupAsk is attached to a list RESPONSE in place of rows: +// the connector needs previous-sync validators before it can serve the +// page. An ask response must carry NO rows, NO next page token, and no +// other source-cache annotations; the syncer consumes it (it never +// reaches page handling), resolves every query, and re-invokes the same +// request with SourceCacheLookupAnswers attached. +// +// Only legal on responses to requests that carried +// SourceCacheLookupOffer. Bounces are capped per REQUEST (a page-token +// advance is a new request and resets the cap, so a multi-page action +// may legally ask once per page); a connector that keeps asking without +// progressing past the cap fails the sync loudly. +type SourceCacheLookupAsk struct { + state protoimpl.MessageState `protogen:"hybrid.v1"` + Queries []*SourceCacheLookupAsk_Query `protobuf:"bytes,1,rep,name=queries,proto3" json:"queries,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SourceCacheLookupAsk) Reset() { + *x = SourceCacheLookupAsk{} + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SourceCacheLookupAsk) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SourceCacheLookupAsk) ProtoMessage() {} + +func (x *SourceCacheLookupAsk) ProtoReflect() protoreflect.Message { + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[4] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *SourceCacheLookupAsk) GetQueries() []*SourceCacheLookupAsk_Query { + if x != nil { + return x.Queries + } + return nil +} + +func (x *SourceCacheLookupAsk) SetQueries(v []*SourceCacheLookupAsk_Query) { + x.Queries = v +} + +type SourceCacheLookupAsk_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + Queries []*SourceCacheLookupAsk_Query +} + +func (b0 SourceCacheLookupAsk_builder) Build() *SourceCacheLookupAsk { + m0 := &SourceCacheLookupAsk{} + b, x := &b0, m0 + _, _ = b, x + x.Queries = b.Queries + return m0 +} + +// SourceCacheLookupAnswers is attached by the SDK to the re-invoked +// REQUEST, carrying the resolution of a prior ask's queries. +// +// Every query of the prior ask is answered — found=true with the +// previous sync's etag, or found=false meaning fetch fresh. A found +// answer whose etag does not fit the per-request answer size budget is +// delivered as found=false: the scope degrades to a cold fetch +// (correct, just slower) instead of staying unresolved, because answers +// accumulate across bounces and an etag that did not fit once can never +// fit a later re-invoke. An ABSENT query (a phase-2 lookup for a scope +// the connector did not ask before) means unresolved: the connector may +// ask again, subject to the bounce cap. +type SourceCacheLookupAnswers struct { + state protoimpl.MessageState `protogen:"hybrid.v1"` + Answers []*SourceCacheLookupAnswers_Answer `protobuf:"bytes,1,rep,name=answers,proto3" json:"answers,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SourceCacheLookupAnswers) Reset() { + *x = SourceCacheLookupAnswers{} + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SourceCacheLookupAnswers) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SourceCacheLookupAnswers) ProtoMessage() {} + +func (x *SourceCacheLookupAnswers) ProtoReflect() protoreflect.Message { + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[5] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *SourceCacheLookupAnswers) GetAnswers() []*SourceCacheLookupAnswers_Answer { + if x != nil { + return x.Answers + } + return nil +} + +func (x *SourceCacheLookupAnswers) SetAnswers(v []*SourceCacheLookupAnswers_Answer) { + x.Answers = v +} + +type SourceCacheLookupAnswers_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + Answers []*SourceCacheLookupAnswers_Answer +} + +func (b0 SourceCacheLookupAnswers_builder) Build() *SourceCacheLookupAnswers { + m0 := &SourceCacheLookupAnswers{} + b, x := &b0, m0 + _, _ = b, x + x.Answers = b.Answers + return m0 +} + +type SourceCacheLookupAsk_Query struct { + state protoimpl.MessageState `protogen:"hybrid.v1"` + // One of the sourcecache.RowKind values: "resources", + // "entitlements", "grants". + RowKind string `protobuf:"bytes,1,opt,name=row_kind,json=rowKind,proto3" json:"row_kind,omitempty"` + ScopeHash string `protobuf:"bytes,2,opt,name=scope_hash,json=scopeHash,proto3" json:"scope_hash,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SourceCacheLookupAsk_Query) Reset() { + *x = SourceCacheLookupAsk_Query{} + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SourceCacheLookupAsk_Query) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SourceCacheLookupAsk_Query) ProtoMessage() {} + +func (x *SourceCacheLookupAsk_Query) ProtoReflect() protoreflect.Message { + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[6] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *SourceCacheLookupAsk_Query) GetRowKind() string { + if x != nil { + return x.RowKind + } + return "" +} + +func (x *SourceCacheLookupAsk_Query) GetScopeHash() string { + if x != nil { + return x.ScopeHash + } + return "" +} + +func (x *SourceCacheLookupAsk_Query) SetRowKind(v string) { + x.RowKind = v +} + +func (x *SourceCacheLookupAsk_Query) SetScopeHash(v string) { + x.ScopeHash = v +} + +type SourceCacheLookupAsk_Query_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + // One of the sourcecache.RowKind values: "resources", + // "entitlements", "grants". + RowKind string + ScopeHash string +} + +func (b0 SourceCacheLookupAsk_Query_builder) Build() *SourceCacheLookupAsk_Query { + m0 := &SourceCacheLookupAsk_Query{} + b, x := &b0, m0 + _, _ = b, x + x.RowKind = b.RowKind + x.ScopeHash = b.ScopeHash + return m0 +} + +type SourceCacheLookupAnswers_Answer struct { + state protoimpl.MessageState `protogen:"hybrid.v1"` + RowKind string `protobuf:"bytes,1,opt,name=row_kind,json=rowKind,proto3" json:"row_kind,omitempty"` + ScopeHash string `protobuf:"bytes,2,opt,name=scope_hash,json=scopeHash,proto3" json:"scope_hash,omitempty"` + Found bool `protobuf:"varint,3,opt,name=found,proto3" json:"found,omitempty"` + // The previous sync's validator; empty when found is false. Cap + // matches the lookup RPC (Graph delta tokens run long). + Etag string `protobuf:"bytes,4,opt,name=etag,proto3" json:"etag,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SourceCacheLookupAnswers_Answer) Reset() { + *x = SourceCacheLookupAnswers_Answer{} + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SourceCacheLookupAnswers_Answer) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SourceCacheLookupAnswers_Answer) ProtoMessage() {} + +func (x *SourceCacheLookupAnswers_Answer) ProtoReflect() protoreflect.Message { + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[7] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *SourceCacheLookupAnswers_Answer) GetRowKind() string { + if x != nil { + return x.RowKind + } + return "" +} + +func (x *SourceCacheLookupAnswers_Answer) GetScopeHash() string { + if x != nil { + return x.ScopeHash + } + return "" +} + +func (x *SourceCacheLookupAnswers_Answer) GetFound() bool { + if x != nil { + return x.Found + } + return false +} + +func (x *SourceCacheLookupAnswers_Answer) GetEtag() string { + if x != nil { + return x.Etag + } + return "" +} + +func (x *SourceCacheLookupAnswers_Answer) SetRowKind(v string) { + x.RowKind = v +} + +func (x *SourceCacheLookupAnswers_Answer) SetScopeHash(v string) { + x.ScopeHash = v +} + +func (x *SourceCacheLookupAnswers_Answer) SetFound(v bool) { + x.Found = v +} + +func (x *SourceCacheLookupAnswers_Answer) SetEtag(v string) { + x.Etag = v +} + +type SourceCacheLookupAnswers_Answer_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + RowKind string + ScopeHash string + Found bool + // The previous sync's validator; empty when found is false. Cap + // matches the lookup RPC (Graph delta tokens run long). + Etag string +} + +func (b0 SourceCacheLookupAnswers_Answer_builder) Build() *SourceCacheLookupAnswers_Answer { + m0 := &SourceCacheLookupAnswers_Answer{} + b, x := &b0, m0 + _, _ = b, x + x.RowKind = b.RowKind + x.ScopeHash = b.ScopeHash + x.Found = b.Found + x.Etag = b.Etag + return m0 +} + +var File_c1_connector_v2_annotation_source_cache_proto protoreflect.FileDescriptor + +const file_c1_connector_v2_annotation_source_cache_proto_rawDesc = "" + + "\n" + + "-c1/connector/v2/annotation_source_cache.proto\x12\x0fc1.connector.v2\x1a\x17validate/validate.proto\"\x9e\x01\n" + + "\x15SourceCacheCapability\x12?\n" + + "\x04mode\x18\x01 \x01(\x0e2+.c1.connector.v2.SourceCacheCapability.ModeR\x04mode\"D\n" + + "\x04Mode\x12\x14\n" + + "\x10MODE_UNSPECIFIED\x10\x00\x12\x11\n" + + "\rMODE_DISABLED\x10\x01\x12\x13\n" + + "\x0fMODE_READ_WRITE\x10\x02\"\x9a\x01\n" + + "\x10SourceCacheScope\x12\x1d\n" + + "\n" + + "scope_hash\x18\x01 \x01(\tR\tscopeHash\x12\x12\n" + + "\x04etag\x18\x02 \x01(\tR\x04etag\x12\x1f\n" + + "\vdeleted_ids\x18\x03 \x03(\tR\n" + + "deletedIds\x122\n" + + "\x15deleted_principal_ids\x18\x04 \x03(\tR\x13deletedPrincipalIds\"\xb5\x01\n" + + "\x11SourceCacheReplay\x12\x1d\n" + + "\n" + + "scope_hash\x18\x01 \x01(\tR\tscopeHash\x12\x12\n" + + "\x04etag\x18\x02 \x01(\tR\x04etag\x12\x18\n" + + "\aoverlay\x18\x03 \x01(\bR\aoverlay\x12\x1f\n" + + "\vdeleted_ids\x18\x04 \x03(\tR\n" + + "deletedIds\x122\n" + + "\x15deleted_principal_ids\x18\x05 \x03(\tR\x13deletedPrincipalIds\"\x18\n" + + "\x16SourceCacheLookupOffer\"\xc4\x01\n" + + "\x14SourceCacheLookupAsk\x12R\n" + + "\aqueries\x18\x01 \x03(\v2+.c1.connector.v2.SourceCacheLookupAsk.QueryB\v\xfaB\b\x92\x01\x05\b\x01\x10\x80 R\aqueries\x1aX\n" + + "\x05Query\x12$\n" + + "\brow_kind\x18\x01 \x01(\tB\t\xfaB\x06r\x04 \x01(@R\arowKind\x12)\n" + + "\n" + + "scope_hash\x18\x02 \x01(\tB\n" + + "\xfaB\ar\x05 \x01(\x80\x02R\tscopeHash\"\xfa\x01\n" + + "\x18SourceCacheLookupAnswers\x12J\n" + + "\aanswers\x18\x01 \x03(\v20.c1.connector.v2.SourceCacheLookupAnswers.AnswerR\aanswers\x1a\x91\x01\n" + + "\x06Answer\x12$\n" + + "\brow_kind\x18\x01 \x01(\tB\t\xfaB\x06r\x04 \x01(@R\arowKind\x12)\n" + + "\n" + + "scope_hash\x18\x02 \x01(\tB\n" + + "\xfaB\ar\x05 \x01(\x80\x02R\tscopeHash\x12\x14\n" + + "\x05found\x18\x03 \x01(\bR\x05found\x12 \n" + + "\x04etag\x18\x04 \x01(\tB\f\xfaB\tr\a(\x80\x80\x04\xd0\x01\x01R\x04etagB6Z4github.com/conductorone/baton-sdk/pb/c1/connector/v2b\x06proto3" + +var file_c1_connector_v2_annotation_source_cache_proto_enumTypes = make([]protoimpl.EnumInfo, 1) +var file_c1_connector_v2_annotation_source_cache_proto_msgTypes = make([]protoimpl.MessageInfo, 8) +var file_c1_connector_v2_annotation_source_cache_proto_goTypes = []any{ + (SourceCacheCapability_Mode)(0), // 0: c1.connector.v2.SourceCacheCapability.Mode + (*SourceCacheCapability)(nil), // 1: c1.connector.v2.SourceCacheCapability + (*SourceCacheScope)(nil), // 2: c1.connector.v2.SourceCacheScope + (*SourceCacheReplay)(nil), // 3: c1.connector.v2.SourceCacheReplay + (*SourceCacheLookupOffer)(nil), // 4: c1.connector.v2.SourceCacheLookupOffer + (*SourceCacheLookupAsk)(nil), // 5: c1.connector.v2.SourceCacheLookupAsk + (*SourceCacheLookupAnswers)(nil), // 6: c1.connector.v2.SourceCacheLookupAnswers + (*SourceCacheLookupAsk_Query)(nil), // 7: c1.connector.v2.SourceCacheLookupAsk.Query + (*SourceCacheLookupAnswers_Answer)(nil), // 8: c1.connector.v2.SourceCacheLookupAnswers.Answer +} +var file_c1_connector_v2_annotation_source_cache_proto_depIdxs = []int32{ + 0, // 0: c1.connector.v2.SourceCacheCapability.mode:type_name -> c1.connector.v2.SourceCacheCapability.Mode + 7, // 1: c1.connector.v2.SourceCacheLookupAsk.queries:type_name -> c1.connector.v2.SourceCacheLookupAsk.Query + 8, // 2: c1.connector.v2.SourceCacheLookupAnswers.answers:type_name -> c1.connector.v2.SourceCacheLookupAnswers.Answer + 3, // [3:3] is the sub-list for method output_type + 3, // [3:3] is the sub-list for method input_type + 3, // [3:3] is the sub-list for extension type_name + 3, // [3:3] is the sub-list for extension extendee + 0, // [0:3] is the sub-list for field type_name +} + +func init() { file_c1_connector_v2_annotation_source_cache_proto_init() } +func file_c1_connector_v2_annotation_source_cache_proto_init() { + if File_c1_connector_v2_annotation_source_cache_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_c1_connector_v2_annotation_source_cache_proto_rawDesc), len(file_c1_connector_v2_annotation_source_cache_proto_rawDesc)), + NumEnums: 1, + NumMessages: 8, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_c1_connector_v2_annotation_source_cache_proto_goTypes, + DependencyIndexes: file_c1_connector_v2_annotation_source_cache_proto_depIdxs, + EnumInfos: file_c1_connector_v2_annotation_source_cache_proto_enumTypes, + MessageInfos: file_c1_connector_v2_annotation_source_cache_proto_msgTypes, + }.Build() + File_c1_connector_v2_annotation_source_cache_proto = out.File + file_c1_connector_v2_annotation_source_cache_proto_goTypes = nil + file_c1_connector_v2_annotation_source_cache_proto_depIdxs = nil +} diff --git a/pb/c1/connector/v2/annotation_source_cache.pb.validate.go b/pb/c1/connector/v2/annotation_source_cache.pb.validate.go new file mode 100644 index 000000000..a301fbcbe --- /dev/null +++ b/pb/c1/connector/v2/annotation_source_cache.pb.validate.go @@ -0,0 +1,1003 @@ +// Code generated by protoc-gen-validate. DO NOT EDIT. +// source: c1/connector/v2/annotation_source_cache.proto + +package v2 + +import ( + "bytes" + "errors" + "fmt" + "net" + "net/mail" + "net/url" + "regexp" + "sort" + "strings" + "time" + "unicode/utf8" + + "google.golang.org/protobuf/types/known/anypb" +) + +// ensure the imports are used +var ( + _ = bytes.MinRead + _ = errors.New("") + _ = fmt.Print + _ = utf8.UTFMax + _ = (*regexp.Regexp)(nil) + _ = (*strings.Reader)(nil) + _ = net.IPv4len + _ = time.Duration(0) + _ = (*url.URL)(nil) + _ = (*mail.Address)(nil) + _ = anypb.Any{} + _ = sort.Sort +) + +// Validate checks the field values on SourceCacheCapability with the rules +// defined in the proto definition for this message. If any rules are +// violated, the first error encountered is returned, or nil if there are no violations. +func (m *SourceCacheCapability) Validate() error { + return m.validate(false) +} + +// ValidateAll checks the field values on SourceCacheCapability with the rules +// defined in the proto definition for this message. If any rules are +// violated, the result is a list of violation errors wrapped in +// SourceCacheCapabilityMultiError, or nil if none found. +func (m *SourceCacheCapability) ValidateAll() error { + return m.validate(true) +} + +func (m *SourceCacheCapability) validate(all bool) error { + if m == nil { + return nil + } + + var errors []error + + // no validation rules for Mode + + if len(errors) > 0 { + return SourceCacheCapabilityMultiError(errors) + } + + return nil +} + +// SourceCacheCapabilityMultiError is an error wrapping multiple validation +// errors returned by SourceCacheCapability.ValidateAll() if the designated +// constraints aren't met. +type SourceCacheCapabilityMultiError []error + +// Error returns a concatenation of all the error messages it wraps. +func (m SourceCacheCapabilityMultiError) Error() string { + msgs := make([]string, 0, len(m)) + for _, err := range m { + msgs = append(msgs, err.Error()) + } + return strings.Join(msgs, "; ") +} + +// AllErrors returns a list of validation violation errors. +func (m SourceCacheCapabilityMultiError) AllErrors() []error { return m } + +// SourceCacheCapabilityValidationError is the validation error returned by +// SourceCacheCapability.Validate if the designated constraints aren't met. +type SourceCacheCapabilityValidationError struct { + field string + reason string + cause error + key bool +} + +// Field function returns field value. +func (e SourceCacheCapabilityValidationError) Field() string { return e.field } + +// Reason function returns reason value. +func (e SourceCacheCapabilityValidationError) Reason() string { return e.reason } + +// Cause function returns cause value. +func (e SourceCacheCapabilityValidationError) Cause() error { return e.cause } + +// Key function returns key value. +func (e SourceCacheCapabilityValidationError) Key() bool { return e.key } + +// ErrorName returns error name. +func (e SourceCacheCapabilityValidationError) ErrorName() string { + return "SourceCacheCapabilityValidationError" +} + +// Error satisfies the builtin error interface +func (e SourceCacheCapabilityValidationError) Error() string { + cause := "" + if e.cause != nil { + cause = fmt.Sprintf(" | caused by: %v", e.cause) + } + + key := "" + if e.key { + key = "key for " + } + + return fmt.Sprintf( + "invalid %sSourceCacheCapability.%s: %s%s", + key, + e.field, + e.reason, + cause) +} + +var _ error = SourceCacheCapabilityValidationError{} + +var _ interface { + Field() string + Reason() string + Key() bool + Cause() error + ErrorName() string +} = SourceCacheCapabilityValidationError{} + +// Validate checks the field values on SourceCacheScope with the rules defined +// in the proto definition for this message. If any rules are violated, the +// first error encountered is returned, or nil if there are no violations. +func (m *SourceCacheScope) Validate() error { + return m.validate(false) +} + +// ValidateAll checks the field values on SourceCacheScope with the rules +// defined in the proto definition for this message. If any rules are +// violated, the result is a list of violation errors wrapped in +// SourceCacheScopeMultiError, or nil if none found. +func (m *SourceCacheScope) ValidateAll() error { + return m.validate(true) +} + +func (m *SourceCacheScope) validate(all bool) error { + if m == nil { + return nil + } + + var errors []error + + // no validation rules for ScopeHash + + // no validation rules for Etag + + if len(errors) > 0 { + return SourceCacheScopeMultiError(errors) + } + + return nil +} + +// SourceCacheScopeMultiError is an error wrapping multiple validation errors +// returned by SourceCacheScope.ValidateAll() if the designated constraints +// aren't met. +type SourceCacheScopeMultiError []error + +// Error returns a concatenation of all the error messages it wraps. +func (m SourceCacheScopeMultiError) Error() string { + msgs := make([]string, 0, len(m)) + for _, err := range m { + msgs = append(msgs, err.Error()) + } + return strings.Join(msgs, "; ") +} + +// AllErrors returns a list of validation violation errors. +func (m SourceCacheScopeMultiError) AllErrors() []error { return m } + +// SourceCacheScopeValidationError is the validation error returned by +// SourceCacheScope.Validate if the designated constraints aren't met. +type SourceCacheScopeValidationError struct { + field string + reason string + cause error + key bool +} + +// Field function returns field value. +func (e SourceCacheScopeValidationError) Field() string { return e.field } + +// Reason function returns reason value. +func (e SourceCacheScopeValidationError) Reason() string { return e.reason } + +// Cause function returns cause value. +func (e SourceCacheScopeValidationError) Cause() error { return e.cause } + +// Key function returns key value. +func (e SourceCacheScopeValidationError) Key() bool { return e.key } + +// ErrorName returns error name. +func (e SourceCacheScopeValidationError) ErrorName() string { return "SourceCacheScopeValidationError" } + +// Error satisfies the builtin error interface +func (e SourceCacheScopeValidationError) Error() string { + cause := "" + if e.cause != nil { + cause = fmt.Sprintf(" | caused by: %v", e.cause) + } + + key := "" + if e.key { + key = "key for " + } + + return fmt.Sprintf( + "invalid %sSourceCacheScope.%s: %s%s", + key, + e.field, + e.reason, + cause) +} + +var _ error = SourceCacheScopeValidationError{} + +var _ interface { + Field() string + Reason() string + Key() bool + Cause() error + ErrorName() string +} = SourceCacheScopeValidationError{} + +// Validate checks the field values on SourceCacheReplay with the rules defined +// in the proto definition for this message. If any rules are violated, the +// first error encountered is returned, or nil if there are no violations. +func (m *SourceCacheReplay) Validate() error { + return m.validate(false) +} + +// ValidateAll checks the field values on SourceCacheReplay with the rules +// defined in the proto definition for this message. If any rules are +// violated, the result is a list of violation errors wrapped in +// SourceCacheReplayMultiError, or nil if none found. +func (m *SourceCacheReplay) ValidateAll() error { + return m.validate(true) +} + +func (m *SourceCacheReplay) validate(all bool) error { + if m == nil { + return nil + } + + var errors []error + + // no validation rules for ScopeHash + + // no validation rules for Etag + + // no validation rules for Overlay + + if len(errors) > 0 { + return SourceCacheReplayMultiError(errors) + } + + return nil +} + +// SourceCacheReplayMultiError is an error wrapping multiple validation errors +// returned by SourceCacheReplay.ValidateAll() if the designated constraints +// aren't met. +type SourceCacheReplayMultiError []error + +// Error returns a concatenation of all the error messages it wraps. +func (m SourceCacheReplayMultiError) Error() string { + msgs := make([]string, 0, len(m)) + for _, err := range m { + msgs = append(msgs, err.Error()) + } + return strings.Join(msgs, "; ") +} + +// AllErrors returns a list of validation violation errors. +func (m SourceCacheReplayMultiError) AllErrors() []error { return m } + +// SourceCacheReplayValidationError is the validation error returned by +// SourceCacheReplay.Validate if the designated constraints aren't met. +type SourceCacheReplayValidationError struct { + field string + reason string + cause error + key bool +} + +// Field function returns field value. +func (e SourceCacheReplayValidationError) Field() string { return e.field } + +// Reason function returns reason value. +func (e SourceCacheReplayValidationError) Reason() string { return e.reason } + +// Cause function returns cause value. +func (e SourceCacheReplayValidationError) Cause() error { return e.cause } + +// Key function returns key value. +func (e SourceCacheReplayValidationError) Key() bool { return e.key } + +// ErrorName returns error name. +func (e SourceCacheReplayValidationError) ErrorName() string { + return "SourceCacheReplayValidationError" +} + +// Error satisfies the builtin error interface +func (e SourceCacheReplayValidationError) Error() string { + cause := "" + if e.cause != nil { + cause = fmt.Sprintf(" | caused by: %v", e.cause) + } + + key := "" + if e.key { + key = "key for " + } + + return fmt.Sprintf( + "invalid %sSourceCacheReplay.%s: %s%s", + key, + e.field, + e.reason, + cause) +} + +var _ error = SourceCacheReplayValidationError{} + +var _ interface { + Field() string + Reason() string + Key() bool + Cause() error + ErrorName() string +} = SourceCacheReplayValidationError{} + +// Validate checks the field values on SourceCacheLookupOffer with the rules +// defined in the proto definition for this message. If any rules are +// violated, the first error encountered is returned, or nil if there are no violations. +func (m *SourceCacheLookupOffer) Validate() error { + return m.validate(false) +} + +// ValidateAll checks the field values on SourceCacheLookupOffer with the rules +// defined in the proto definition for this message. If any rules are +// violated, the result is a list of violation errors wrapped in +// SourceCacheLookupOfferMultiError, or nil if none found. +func (m *SourceCacheLookupOffer) ValidateAll() error { + return m.validate(true) +} + +func (m *SourceCacheLookupOffer) validate(all bool) error { + if m == nil { + return nil + } + + var errors []error + + if len(errors) > 0 { + return SourceCacheLookupOfferMultiError(errors) + } + + return nil +} + +// SourceCacheLookupOfferMultiError is an error wrapping multiple validation +// errors returned by SourceCacheLookupOffer.ValidateAll() if the designated +// constraints aren't met. +type SourceCacheLookupOfferMultiError []error + +// Error returns a concatenation of all the error messages it wraps. +func (m SourceCacheLookupOfferMultiError) Error() string { + msgs := make([]string, 0, len(m)) + for _, err := range m { + msgs = append(msgs, err.Error()) + } + return strings.Join(msgs, "; ") +} + +// AllErrors returns a list of validation violation errors. +func (m SourceCacheLookupOfferMultiError) AllErrors() []error { return m } + +// SourceCacheLookupOfferValidationError is the validation error returned by +// SourceCacheLookupOffer.Validate if the designated constraints aren't met. +type SourceCacheLookupOfferValidationError struct { + field string + reason string + cause error + key bool +} + +// Field function returns field value. +func (e SourceCacheLookupOfferValidationError) Field() string { return e.field } + +// Reason function returns reason value. +func (e SourceCacheLookupOfferValidationError) Reason() string { return e.reason } + +// Cause function returns cause value. +func (e SourceCacheLookupOfferValidationError) Cause() error { return e.cause } + +// Key function returns key value. +func (e SourceCacheLookupOfferValidationError) Key() bool { return e.key } + +// ErrorName returns error name. +func (e SourceCacheLookupOfferValidationError) ErrorName() string { + return "SourceCacheLookupOfferValidationError" +} + +// Error satisfies the builtin error interface +func (e SourceCacheLookupOfferValidationError) Error() string { + cause := "" + if e.cause != nil { + cause = fmt.Sprintf(" | caused by: %v", e.cause) + } + + key := "" + if e.key { + key = "key for " + } + + return fmt.Sprintf( + "invalid %sSourceCacheLookupOffer.%s: %s%s", + key, + e.field, + e.reason, + cause) +} + +var _ error = SourceCacheLookupOfferValidationError{} + +var _ interface { + Field() string + Reason() string + Key() bool + Cause() error + ErrorName() string +} = SourceCacheLookupOfferValidationError{} + +// Validate checks the field values on SourceCacheLookupAsk with the rules +// defined in the proto definition for this message. If any rules are +// violated, the first error encountered is returned, or nil if there are no violations. +func (m *SourceCacheLookupAsk) Validate() error { + return m.validate(false) +} + +// ValidateAll checks the field values on SourceCacheLookupAsk with the rules +// defined in the proto definition for this message. If any rules are +// violated, the result is a list of violation errors wrapped in +// SourceCacheLookupAskMultiError, or nil if none found. +func (m *SourceCacheLookupAsk) ValidateAll() error { + return m.validate(true) +} + +func (m *SourceCacheLookupAsk) validate(all bool) error { + if m == nil { + return nil + } + + var errors []error + + if l := len(m.GetQueries()); l < 1 || l > 4096 { + err := SourceCacheLookupAskValidationError{ + field: "Queries", + reason: "value must contain between 1 and 4096 items, inclusive", + } + if !all { + return err + } + errors = append(errors, err) + } + + for idx, item := range m.GetQueries() { + _, _ = idx, item + + if all { + switch v := interface{}(item).(type) { + case interface{ ValidateAll() error }: + if err := v.ValidateAll(); err != nil { + errors = append(errors, SourceCacheLookupAskValidationError{ + field: fmt.Sprintf("Queries[%v]", idx), + reason: "embedded message failed validation", + cause: err, + }) + } + case interface{ Validate() error }: + if err := v.Validate(); err != nil { + errors = append(errors, SourceCacheLookupAskValidationError{ + field: fmt.Sprintf("Queries[%v]", idx), + reason: "embedded message failed validation", + cause: err, + }) + } + } + } else if v, ok := interface{}(item).(interface{ Validate() error }); ok { + if err := v.Validate(); err != nil { + return SourceCacheLookupAskValidationError{ + field: fmt.Sprintf("Queries[%v]", idx), + reason: "embedded message failed validation", + cause: err, + } + } + } + + } + + if len(errors) > 0 { + return SourceCacheLookupAskMultiError(errors) + } + + return nil +} + +// SourceCacheLookupAskMultiError is an error wrapping multiple validation +// errors returned by SourceCacheLookupAsk.ValidateAll() if the designated +// constraints aren't met. +type SourceCacheLookupAskMultiError []error + +// Error returns a concatenation of all the error messages it wraps. +func (m SourceCacheLookupAskMultiError) Error() string { + msgs := make([]string, 0, len(m)) + for _, err := range m { + msgs = append(msgs, err.Error()) + } + return strings.Join(msgs, "; ") +} + +// AllErrors returns a list of validation violation errors. +func (m SourceCacheLookupAskMultiError) AllErrors() []error { return m } + +// SourceCacheLookupAskValidationError is the validation error returned by +// SourceCacheLookupAsk.Validate if the designated constraints aren't met. +type SourceCacheLookupAskValidationError struct { + field string + reason string + cause error + key bool +} + +// Field function returns field value. +func (e SourceCacheLookupAskValidationError) Field() string { return e.field } + +// Reason function returns reason value. +func (e SourceCacheLookupAskValidationError) Reason() string { return e.reason } + +// Cause function returns cause value. +func (e SourceCacheLookupAskValidationError) Cause() error { return e.cause } + +// Key function returns key value. +func (e SourceCacheLookupAskValidationError) Key() bool { return e.key } + +// ErrorName returns error name. +func (e SourceCacheLookupAskValidationError) ErrorName() string { + return "SourceCacheLookupAskValidationError" +} + +// Error satisfies the builtin error interface +func (e SourceCacheLookupAskValidationError) Error() string { + cause := "" + if e.cause != nil { + cause = fmt.Sprintf(" | caused by: %v", e.cause) + } + + key := "" + if e.key { + key = "key for " + } + + return fmt.Sprintf( + "invalid %sSourceCacheLookupAsk.%s: %s%s", + key, + e.field, + e.reason, + cause) +} + +var _ error = SourceCacheLookupAskValidationError{} + +var _ interface { + Field() string + Reason() string + Key() bool + Cause() error + ErrorName() string +} = SourceCacheLookupAskValidationError{} + +// Validate checks the field values on SourceCacheLookupAnswers with the rules +// defined in the proto definition for this message. If any rules are +// violated, the first error encountered is returned, or nil if there are no violations. +func (m *SourceCacheLookupAnswers) Validate() error { + return m.validate(false) +} + +// ValidateAll checks the field values on SourceCacheLookupAnswers with the +// rules defined in the proto definition for this message. If any rules are +// violated, the result is a list of violation errors wrapped in +// SourceCacheLookupAnswersMultiError, or nil if none found. +func (m *SourceCacheLookupAnswers) ValidateAll() error { + return m.validate(true) +} + +func (m *SourceCacheLookupAnswers) validate(all bool) error { + if m == nil { + return nil + } + + var errors []error + + for idx, item := range m.GetAnswers() { + _, _ = idx, item + + if all { + switch v := interface{}(item).(type) { + case interface{ ValidateAll() error }: + if err := v.ValidateAll(); err != nil { + errors = append(errors, SourceCacheLookupAnswersValidationError{ + field: fmt.Sprintf("Answers[%v]", idx), + reason: "embedded message failed validation", + cause: err, + }) + } + case interface{ Validate() error }: + if err := v.Validate(); err != nil { + errors = append(errors, SourceCacheLookupAnswersValidationError{ + field: fmt.Sprintf("Answers[%v]", idx), + reason: "embedded message failed validation", + cause: err, + }) + } + } + } else if v, ok := interface{}(item).(interface{ Validate() error }); ok { + if err := v.Validate(); err != nil { + return SourceCacheLookupAnswersValidationError{ + field: fmt.Sprintf("Answers[%v]", idx), + reason: "embedded message failed validation", + cause: err, + } + } + } + + } + + if len(errors) > 0 { + return SourceCacheLookupAnswersMultiError(errors) + } + + return nil +} + +// SourceCacheLookupAnswersMultiError is an error wrapping multiple validation +// errors returned by SourceCacheLookupAnswers.ValidateAll() if the designated +// constraints aren't met. +type SourceCacheLookupAnswersMultiError []error + +// Error returns a concatenation of all the error messages it wraps. +func (m SourceCacheLookupAnswersMultiError) Error() string { + msgs := make([]string, 0, len(m)) + for _, err := range m { + msgs = append(msgs, err.Error()) + } + return strings.Join(msgs, "; ") +} + +// AllErrors returns a list of validation violation errors. +func (m SourceCacheLookupAnswersMultiError) AllErrors() []error { return m } + +// SourceCacheLookupAnswersValidationError is the validation error returned by +// SourceCacheLookupAnswers.Validate if the designated constraints aren't met. +type SourceCacheLookupAnswersValidationError struct { + field string + reason string + cause error + key bool +} + +// Field function returns field value. +func (e SourceCacheLookupAnswersValidationError) Field() string { return e.field } + +// Reason function returns reason value. +func (e SourceCacheLookupAnswersValidationError) Reason() string { return e.reason } + +// Cause function returns cause value. +func (e SourceCacheLookupAnswersValidationError) Cause() error { return e.cause } + +// Key function returns key value. +func (e SourceCacheLookupAnswersValidationError) Key() bool { return e.key } + +// ErrorName returns error name. +func (e SourceCacheLookupAnswersValidationError) ErrorName() string { + return "SourceCacheLookupAnswersValidationError" +} + +// Error satisfies the builtin error interface +func (e SourceCacheLookupAnswersValidationError) Error() string { + cause := "" + if e.cause != nil { + cause = fmt.Sprintf(" | caused by: %v", e.cause) + } + + key := "" + if e.key { + key = "key for " + } + + return fmt.Sprintf( + "invalid %sSourceCacheLookupAnswers.%s: %s%s", + key, + e.field, + e.reason, + cause) +} + +var _ error = SourceCacheLookupAnswersValidationError{} + +var _ interface { + Field() string + Reason() string + Key() bool + Cause() error + ErrorName() string +} = SourceCacheLookupAnswersValidationError{} + +// Validate checks the field values on SourceCacheLookupAsk_Query with the +// rules defined in the proto definition for this message. If any rules are +// violated, the first error encountered is returned, or nil if there are no violations. +func (m *SourceCacheLookupAsk_Query) Validate() error { + return m.validate(false) +} + +// ValidateAll checks the field values on SourceCacheLookupAsk_Query with the +// rules defined in the proto definition for this message. If any rules are +// violated, the result is a list of violation errors wrapped in +// SourceCacheLookupAsk_QueryMultiError, or nil if none found. +func (m *SourceCacheLookupAsk_Query) ValidateAll() error { + return m.validate(true) +} + +func (m *SourceCacheLookupAsk_Query) validate(all bool) error { + if m == nil { + return nil + } + + var errors []error + + if l := len(m.GetRowKind()); l < 1 || l > 64 { + err := SourceCacheLookupAsk_QueryValidationError{ + field: "RowKind", + reason: "value length must be between 1 and 64 bytes, inclusive", + } + if !all { + return err + } + errors = append(errors, err) + } + + if l := len(m.GetScopeHash()); l < 1 || l > 256 { + err := SourceCacheLookupAsk_QueryValidationError{ + field: "ScopeHash", + reason: "value length must be between 1 and 256 bytes, inclusive", + } + if !all { + return err + } + errors = append(errors, err) + } + + if len(errors) > 0 { + return SourceCacheLookupAsk_QueryMultiError(errors) + } + + return nil +} + +// SourceCacheLookupAsk_QueryMultiError is an error wrapping multiple +// validation errors returned by SourceCacheLookupAsk_Query.ValidateAll() if +// the designated constraints aren't met. +type SourceCacheLookupAsk_QueryMultiError []error + +// Error returns a concatenation of all the error messages it wraps. +func (m SourceCacheLookupAsk_QueryMultiError) Error() string { + msgs := make([]string, 0, len(m)) + for _, err := range m { + msgs = append(msgs, err.Error()) + } + return strings.Join(msgs, "; ") +} + +// AllErrors returns a list of validation violation errors. +func (m SourceCacheLookupAsk_QueryMultiError) AllErrors() []error { return m } + +// SourceCacheLookupAsk_QueryValidationError is the validation error returned +// by SourceCacheLookupAsk_Query.Validate if the designated constraints aren't met. +type SourceCacheLookupAsk_QueryValidationError struct { + field string + reason string + cause error + key bool +} + +// Field function returns field value. +func (e SourceCacheLookupAsk_QueryValidationError) Field() string { return e.field } + +// Reason function returns reason value. +func (e SourceCacheLookupAsk_QueryValidationError) Reason() string { return e.reason } + +// Cause function returns cause value. +func (e SourceCacheLookupAsk_QueryValidationError) Cause() error { return e.cause } + +// Key function returns key value. +func (e SourceCacheLookupAsk_QueryValidationError) Key() bool { return e.key } + +// ErrorName returns error name. +func (e SourceCacheLookupAsk_QueryValidationError) ErrorName() string { + return "SourceCacheLookupAsk_QueryValidationError" +} + +// Error satisfies the builtin error interface +func (e SourceCacheLookupAsk_QueryValidationError) Error() string { + cause := "" + if e.cause != nil { + cause = fmt.Sprintf(" | caused by: %v", e.cause) + } + + key := "" + if e.key { + key = "key for " + } + + return fmt.Sprintf( + "invalid %sSourceCacheLookupAsk_Query.%s: %s%s", + key, + e.field, + e.reason, + cause) +} + +var _ error = SourceCacheLookupAsk_QueryValidationError{} + +var _ interface { + Field() string + Reason() string + Key() bool + Cause() error + ErrorName() string +} = SourceCacheLookupAsk_QueryValidationError{} + +// Validate checks the field values on SourceCacheLookupAnswers_Answer with the +// rules defined in the proto definition for this message. If any rules are +// violated, the first error encountered is returned, or nil if there are no violations. +func (m *SourceCacheLookupAnswers_Answer) Validate() error { + return m.validate(false) +} + +// ValidateAll checks the field values on SourceCacheLookupAnswers_Answer with +// the rules defined in the proto definition for this message. If any rules +// are violated, the result is a list of violation errors wrapped in +// SourceCacheLookupAnswers_AnswerMultiError, or nil if none found. +func (m *SourceCacheLookupAnswers_Answer) ValidateAll() error { + return m.validate(true) +} + +func (m *SourceCacheLookupAnswers_Answer) validate(all bool) error { + if m == nil { + return nil + } + + var errors []error + + if l := len(m.GetRowKind()); l < 1 || l > 64 { + err := SourceCacheLookupAnswers_AnswerValidationError{ + field: "RowKind", + reason: "value length must be between 1 and 64 bytes, inclusive", + } + if !all { + return err + } + errors = append(errors, err) + } + + if l := len(m.GetScopeHash()); l < 1 || l > 256 { + err := SourceCacheLookupAnswers_AnswerValidationError{ + field: "ScopeHash", + reason: "value length must be between 1 and 256 bytes, inclusive", + } + if !all { + return err + } + errors = append(errors, err) + } + + // no validation rules for Found + + if m.GetEtag() != "" { + + if len(m.GetEtag()) > 65536 { + err := SourceCacheLookupAnswers_AnswerValidationError{ + field: "Etag", + reason: "value length must be at most 65536 bytes", + } + if !all { + return err + } + errors = append(errors, err) + } + + } + + if len(errors) > 0 { + return SourceCacheLookupAnswers_AnswerMultiError(errors) + } + + return nil +} + +// SourceCacheLookupAnswers_AnswerMultiError is an error wrapping multiple +// validation errors returned by SourceCacheLookupAnswers_Answer.ValidateAll() +// if the designated constraints aren't met. +type SourceCacheLookupAnswers_AnswerMultiError []error + +// Error returns a concatenation of all the error messages it wraps. +func (m SourceCacheLookupAnswers_AnswerMultiError) Error() string { + msgs := make([]string, 0, len(m)) + for _, err := range m { + msgs = append(msgs, err.Error()) + } + return strings.Join(msgs, "; ") +} + +// AllErrors returns a list of validation violation errors. +func (m SourceCacheLookupAnswers_AnswerMultiError) AllErrors() []error { return m } + +// SourceCacheLookupAnswers_AnswerValidationError is the validation error +// returned by SourceCacheLookupAnswers_Answer.Validate if the designated +// constraints aren't met. +type SourceCacheLookupAnswers_AnswerValidationError struct { + field string + reason string + cause error + key bool +} + +// Field function returns field value. +func (e SourceCacheLookupAnswers_AnswerValidationError) Field() string { return e.field } + +// Reason function returns reason value. +func (e SourceCacheLookupAnswers_AnswerValidationError) Reason() string { return e.reason } + +// Cause function returns cause value. +func (e SourceCacheLookupAnswers_AnswerValidationError) Cause() error { return e.cause } + +// Key function returns key value. +func (e SourceCacheLookupAnswers_AnswerValidationError) Key() bool { return e.key } + +// ErrorName returns error name. +func (e SourceCacheLookupAnswers_AnswerValidationError) ErrorName() string { + return "SourceCacheLookupAnswers_AnswerValidationError" +} + +// Error satisfies the builtin error interface +func (e SourceCacheLookupAnswers_AnswerValidationError) Error() string { + cause := "" + if e.cause != nil { + cause = fmt.Sprintf(" | caused by: %v", e.cause) + } + + key := "" + if e.key { + key = "key for " + } + + return fmt.Sprintf( + "invalid %sSourceCacheLookupAnswers_Answer.%s: %s%s", + key, + e.field, + e.reason, + cause) +} + +var _ error = SourceCacheLookupAnswers_AnswerValidationError{} + +var _ interface { + Field() string + Reason() string + Key() bool + Cause() error + ErrorName() string +} = SourceCacheLookupAnswers_AnswerValidationError{} diff --git a/pb/c1/connector/v2/annotation_source_cache_protoopaque.pb.go b/pb/c1/connector/v2/annotation_source_cache_protoopaque.pb.go new file mode 100644 index 000000000..baf0e3b67 --- /dev/null +++ b/pb/c1/connector/v2/annotation_source_cache_protoopaque.pb.go @@ -0,0 +1,865 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.10 +// protoc (unknown) +// source: c1/connector/v2/annotation_source_cache.proto + +//go:build protoopaque + +package v2 + +import ( + _ "github.com/envoyproxy/protoc-gen-validate/validate" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type SourceCacheCapability_Mode int32 + +const ( + SourceCacheCapability_MODE_UNSPECIFIED SourceCacheCapability_Mode = 0 + SourceCacheCapability_MODE_DISABLED SourceCacheCapability_Mode = 1 + SourceCacheCapability_MODE_READ_WRITE SourceCacheCapability_Mode = 2 +) + +// Enum value maps for SourceCacheCapability_Mode. +var ( + SourceCacheCapability_Mode_name = map[int32]string{ + 0: "MODE_UNSPECIFIED", + 1: "MODE_DISABLED", + 2: "MODE_READ_WRITE", + } + SourceCacheCapability_Mode_value = map[string]int32{ + "MODE_UNSPECIFIED": 0, + "MODE_DISABLED": 1, + "MODE_READ_WRITE": 2, + } +) + +func (x SourceCacheCapability_Mode) Enum() *SourceCacheCapability_Mode { + p := new(SourceCacheCapability_Mode) + *p = x + return p +} + +func (x SourceCacheCapability_Mode) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (SourceCacheCapability_Mode) Descriptor() protoreflect.EnumDescriptor { + return file_c1_connector_v2_annotation_source_cache_proto_enumTypes[0].Descriptor() +} + +func (SourceCacheCapability_Mode) Type() protoreflect.EnumType { + return &file_c1_connector_v2_annotation_source_cache_proto_enumTypes[0] +} + +func (x SourceCacheCapability_Mode) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// SourceCacheCapability is attached to ConnectorServiceValidateResponse +// annotations to opt in to source-cache replay. Absent or any mode other +// than MODE_READ_WRITE means all source-cache annotations are ignored. +type SourceCacheCapability struct { + state protoimpl.MessageState `protogen:"opaque.v1"` + xxx_hidden_Mode SourceCacheCapability_Mode `protobuf:"varint,1,opt,name=mode,proto3,enum=c1.connector.v2.SourceCacheCapability_Mode"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SourceCacheCapability) Reset() { + *x = SourceCacheCapability{} + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SourceCacheCapability) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SourceCacheCapability) ProtoMessage() {} + +func (x *SourceCacheCapability) ProtoReflect() protoreflect.Message { + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *SourceCacheCapability) GetMode() SourceCacheCapability_Mode { + if x != nil { + return x.xxx_hidden_Mode + } + return SourceCacheCapability_MODE_UNSPECIFIED +} + +func (x *SourceCacheCapability) SetMode(v SourceCacheCapability_Mode) { + x.xxx_hidden_Mode = v +} + +type SourceCacheCapability_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + Mode SourceCacheCapability_Mode +} + +func (b0 SourceCacheCapability_builder) Build() *SourceCacheCapability { + m0 := &SourceCacheCapability{} + b, x := &b0, m0 + _, _ = b, x + x.xxx_hidden_Mode = b.Mode + return m0 +} + +// SourceCacheScope is attached to a list-response page whose rows were +// freshly fetched from upstream. The SDK stamps the page's rows with +// scope_hash so a future sync can replay them as a unit. +type SourceCacheScope struct { + state protoimpl.MessageState `protogen:"opaque.v1"` + xxx_hidden_ScopeHash string `protobuf:"bytes,1,opt,name=scope_hash,json=scopeHash,proto3"` + xxx_hidden_Etag string `protobuf:"bytes,2,opt,name=etag,proto3"` + xxx_hidden_DeletedIds []string `protobuf:"bytes,3,rep,name=deleted_ids,json=deletedIds,proto3"` + xxx_hidden_DeletedPrincipalIds []string `protobuf:"bytes,4,rep,name=deleted_principal_ids,json=deletedPrincipalIds,proto3"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SourceCacheScope) Reset() { + *x = SourceCacheScope{} + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SourceCacheScope) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SourceCacheScope) ProtoMessage() {} + +func (x *SourceCacheScope) ProtoReflect() protoreflect.Message { + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *SourceCacheScope) GetScopeHash() string { + if x != nil { + return x.xxx_hidden_ScopeHash + } + return "" +} + +func (x *SourceCacheScope) GetEtag() string { + if x != nil { + return x.xxx_hidden_Etag + } + return "" +} + +func (x *SourceCacheScope) GetDeletedIds() []string { + if x != nil { + return x.xxx_hidden_DeletedIds + } + return nil +} + +func (x *SourceCacheScope) GetDeletedPrincipalIds() []string { + if x != nil { + return x.xxx_hidden_DeletedPrincipalIds + } + return nil +} + +func (x *SourceCacheScope) SetScopeHash(v string) { + x.xxx_hidden_ScopeHash = v +} + +func (x *SourceCacheScope) SetEtag(v string) { + x.xxx_hidden_Etag = v +} + +func (x *SourceCacheScope) SetDeletedIds(v []string) { + x.xxx_hidden_DeletedIds = v +} + +func (x *SourceCacheScope) SetDeletedPrincipalIds(v []string) { + x.xxx_hidden_DeletedPrincipalIds = v +} + +type SourceCacheScope_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + // Connector-computed stable identifier for the canonical scope. Must be + // byte-stable across syncs for the same logical scope. Prefer an ORDERED + // natural identifier (e.g. the request URL, or "groups/{id}/members") + // over a random hash: scope-index writes lead with this value, so + // identifiers that correlate with fetch order keep index writes nearly + // append-ordered at scale. sourcecache.HashScope is the fallback when no + // compact natural form exists. + ScopeHash string + // Opaque validator to persist for this scope. May be empty on interim + // pages of a multi-page scope (e.g. Graph @odata.nextLink pages); the + // SDK writes the scope's manifest entry when a non-empty etag arrives. + // A 200 response with zero rows still persists the entry. + Etag string + // Tombstones applied after this page's rows commit. Lets every page of + // a multi-page delta round carry its own deletions as the provider + // delivers them, instead of buffering a whole round onto the first + // (replay-annotated) page. Same formats as SourceCacheReplay. + // + // PRECONDITION for tombstones anywhere in a round: the provider's delta + // must be coalesced — at most one add-or-tombstone per object per round + // (Microsoft Graph guarantees this by returning final object state). + // With interleaved add/remove events for one object, per-page ordering + // is deterministic (a page's rows upsert before its deletions apply) + // but cross-page re-adds after a tombstone are the connector's + // responsibility to order. + DeletedIds []string + // Principal-scoped grant tombstones: for RowKindGrants pages, each + // entry deletes EVERY grant row stamped with this scope whose principal + // id equals the entry — no principal resource type and no canonical + // grant-id reconstruction required (delta tombstones usually carry only + // a bare object id, and the object may no longer exist to look up). + // + // PRECONDITION: the scope must be partitioned so that "principal + // removed from scope" means every grant they have in the scope is gone + // — one scope per navigation with independent removal semantics (e.g. + // members and owners of a group are separate scopes, or membership + // removal would take the owner grant with it). + // + // For RowKindResources pages, each entry deletes the resource row(s) + // stamped with this scope whose resource id equals the entry, any + // resource type. + DeletedPrincipalIds []string +} + +func (b0 SourceCacheScope_builder) Build() *SourceCacheScope { + m0 := &SourceCacheScope{} + b, x := &b0, m0 + _, _ = b, x + x.xxx_hidden_ScopeHash = b.ScopeHash + x.xxx_hidden_Etag = b.Etag + x.xxx_hidden_DeletedIds = b.DeletedIds + x.xxx_hidden_DeletedPrincipalIds = b.DeletedPrincipalIds + return m0 +} + +// SourceCacheReplay is attached to a list-response page to tell the SDK to +// copy the previous sync's rows for scope_hash into the current sync. +// +// The row kind (resources, entitlements, grants) is determined by which +// RPC the annotation arrived on, never by the annotation itself. +// +// The connector must only emit this for a scope whose etag it received +// from the SDK's source-cache lookup during this same sync. A replay for +// an unknown scope fails the sync: the connector has already skipped row +// generation, so there is nothing to fall back to. +type SourceCacheReplay struct { + state protoimpl.MessageState `protogen:"opaque.v1"` + xxx_hidden_ScopeHash string `protobuf:"bytes,1,opt,name=scope_hash,json=scopeHash,proto3"` + xxx_hidden_Etag string `protobuf:"bytes,2,opt,name=etag,proto3"` + xxx_hidden_Overlay bool `protobuf:"varint,3,opt,name=overlay,proto3"` + xxx_hidden_DeletedIds []string `protobuf:"bytes,4,rep,name=deleted_ids,json=deletedIds,proto3"` + xxx_hidden_DeletedPrincipalIds []string `protobuf:"bytes,5,rep,name=deleted_principal_ids,json=deletedPrincipalIds,proto3"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SourceCacheReplay) Reset() { + *x = SourceCacheReplay{} + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SourceCacheReplay) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SourceCacheReplay) ProtoMessage() {} + +func (x *SourceCacheReplay) ProtoReflect() protoreflect.Message { + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *SourceCacheReplay) GetScopeHash() string { + if x != nil { + return x.xxx_hidden_ScopeHash + } + return "" +} + +func (x *SourceCacheReplay) GetEtag() string { + if x != nil { + return x.xxx_hidden_Etag + } + return "" +} + +func (x *SourceCacheReplay) GetOverlay() bool { + if x != nil { + return x.xxx_hidden_Overlay + } + return false +} + +func (x *SourceCacheReplay) GetDeletedIds() []string { + if x != nil { + return x.xxx_hidden_DeletedIds + } + return nil +} + +func (x *SourceCacheReplay) GetDeletedPrincipalIds() []string { + if x != nil { + return x.xxx_hidden_DeletedPrincipalIds + } + return nil +} + +func (x *SourceCacheReplay) SetScopeHash(v string) { + x.xxx_hidden_ScopeHash = v +} + +func (x *SourceCacheReplay) SetEtag(v string) { + x.xxx_hidden_Etag = v +} + +func (x *SourceCacheReplay) SetOverlay(v bool) { + x.xxx_hidden_Overlay = v +} + +func (x *SourceCacheReplay) SetDeletedIds(v []string) { + x.xxx_hidden_DeletedIds = v +} + +func (x *SourceCacheReplay) SetDeletedPrincipalIds(v []string) { + x.xxx_hidden_DeletedPrincipalIds = v +} + +type SourceCacheReplay_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + ScopeHash string + // Validator to persist for this scope in the current sync. For an HTTP + // 304 this is the unchanged etag. For a delta query this is the NEW + // token; it may instead be supplied by the final overlay page's + // SourceCacheScope.etag, in which case this field may be left empty. + Etag string + // When true, the response (and subsequent pages carrying + // SourceCacheScope with the same scope_hash) contains changed rows to + // upsert on top of the replayed base. When false the response must + // contain no rows for this scope. + Overlay bool + // Public canonical IDs (grant/entitlement IDs, or resource BIDs for + // RowKindResources) to delete from the current sync after the replay + // copy and this page's upserts. Used for delta-query tombstones (e.g. + // Microsoft Graph @removed entries). Subsequent pages of the round + // carry their tombstones on SourceCacheScope.deleted_ids. + DeletedIds []string + // Principal-scoped tombstones for this page; see + // SourceCacheScope.deleted_principal_ids for semantics and + // preconditions. + DeletedPrincipalIds []string +} + +func (b0 SourceCacheReplay_builder) Build() *SourceCacheReplay { + m0 := &SourceCacheReplay{} + b, x := &b0, m0 + _, _ = b, x + x.xxx_hidden_ScopeHash = b.ScopeHash + x.xxx_hidden_Etag = b.Etag + x.xxx_hidden_Overlay = b.Overlay + x.xxx_hidden_DeletedIds = b.DeletedIds + x.xxx_hidden_DeletedPrincipalIds = b.DeletedPrincipalIds + return m0 +} + +// SourceCacheLookupOffer is attached by the SDK to list REQUESTS when the +// syncer can answer source-cache lookup asks: the connector declared +// SourceCacheCapability and a warm previous-sync lookup is installed. +// Its presence is the connector's permission to answer with +// SourceCacheLookupAsk instead of rows. Connectors with a direct lookup +// (in-process, subprocess) never need to defer and may ignore it. +type SourceCacheLookupOffer struct { + state protoimpl.MessageState `protogen:"opaque.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SourceCacheLookupOffer) Reset() { + *x = SourceCacheLookupOffer{} + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SourceCacheLookupOffer) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SourceCacheLookupOffer) ProtoMessage() {} + +func (x *SourceCacheLookupOffer) ProtoReflect() protoreflect.Message { + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +type SourceCacheLookupOffer_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + +} + +func (b0 SourceCacheLookupOffer_builder) Build() *SourceCacheLookupOffer { + m0 := &SourceCacheLookupOffer{} + b, x := &b0, m0 + _, _ = b, x + return m0 +} + +// SourceCacheLookupAsk is attached to a list RESPONSE in place of rows: +// the connector needs previous-sync validators before it can serve the +// page. An ask response must carry NO rows, NO next page token, and no +// other source-cache annotations; the syncer consumes it (it never +// reaches page handling), resolves every query, and re-invokes the same +// request with SourceCacheLookupAnswers attached. +// +// Only legal on responses to requests that carried +// SourceCacheLookupOffer. Bounces are capped per REQUEST (a page-token +// advance is a new request and resets the cap, so a multi-page action +// may legally ask once per page); a connector that keeps asking without +// progressing past the cap fails the sync loudly. +type SourceCacheLookupAsk struct { + state protoimpl.MessageState `protogen:"opaque.v1"` + xxx_hidden_Queries *[]*SourceCacheLookupAsk_Query `protobuf:"bytes,1,rep,name=queries,proto3"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SourceCacheLookupAsk) Reset() { + *x = SourceCacheLookupAsk{} + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SourceCacheLookupAsk) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SourceCacheLookupAsk) ProtoMessage() {} + +func (x *SourceCacheLookupAsk) ProtoReflect() protoreflect.Message { + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[4] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *SourceCacheLookupAsk) GetQueries() []*SourceCacheLookupAsk_Query { + if x != nil { + if x.xxx_hidden_Queries != nil { + return *x.xxx_hidden_Queries + } + } + return nil +} + +func (x *SourceCacheLookupAsk) SetQueries(v []*SourceCacheLookupAsk_Query) { + x.xxx_hidden_Queries = &v +} + +type SourceCacheLookupAsk_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + Queries []*SourceCacheLookupAsk_Query +} + +func (b0 SourceCacheLookupAsk_builder) Build() *SourceCacheLookupAsk { + m0 := &SourceCacheLookupAsk{} + b, x := &b0, m0 + _, _ = b, x + x.xxx_hidden_Queries = &b.Queries + return m0 +} + +// SourceCacheLookupAnswers is attached by the SDK to the re-invoked +// REQUEST, carrying the resolution of a prior ask's queries. +// +// Every query of the prior ask is answered — found=true with the +// previous sync's etag, or found=false meaning fetch fresh. A found +// answer whose etag does not fit the per-request answer size budget is +// delivered as found=false: the scope degrades to a cold fetch +// (correct, just slower) instead of staying unresolved, because answers +// accumulate across bounces and an etag that did not fit once can never +// fit a later re-invoke. An ABSENT query (a phase-2 lookup for a scope +// the connector did not ask before) means unresolved: the connector may +// ask again, subject to the bounce cap. +type SourceCacheLookupAnswers struct { + state protoimpl.MessageState `protogen:"opaque.v1"` + xxx_hidden_Answers *[]*SourceCacheLookupAnswers_Answer `protobuf:"bytes,1,rep,name=answers,proto3"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SourceCacheLookupAnswers) Reset() { + *x = SourceCacheLookupAnswers{} + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SourceCacheLookupAnswers) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SourceCacheLookupAnswers) ProtoMessage() {} + +func (x *SourceCacheLookupAnswers) ProtoReflect() protoreflect.Message { + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[5] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *SourceCacheLookupAnswers) GetAnswers() []*SourceCacheLookupAnswers_Answer { + if x != nil { + if x.xxx_hidden_Answers != nil { + return *x.xxx_hidden_Answers + } + } + return nil +} + +func (x *SourceCacheLookupAnswers) SetAnswers(v []*SourceCacheLookupAnswers_Answer) { + x.xxx_hidden_Answers = &v +} + +type SourceCacheLookupAnswers_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + Answers []*SourceCacheLookupAnswers_Answer +} + +func (b0 SourceCacheLookupAnswers_builder) Build() *SourceCacheLookupAnswers { + m0 := &SourceCacheLookupAnswers{} + b, x := &b0, m0 + _, _ = b, x + x.xxx_hidden_Answers = &b.Answers + return m0 +} + +type SourceCacheLookupAsk_Query struct { + state protoimpl.MessageState `protogen:"opaque.v1"` + xxx_hidden_RowKind string `protobuf:"bytes,1,opt,name=row_kind,json=rowKind,proto3"` + xxx_hidden_ScopeHash string `protobuf:"bytes,2,opt,name=scope_hash,json=scopeHash,proto3"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SourceCacheLookupAsk_Query) Reset() { + *x = SourceCacheLookupAsk_Query{} + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SourceCacheLookupAsk_Query) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SourceCacheLookupAsk_Query) ProtoMessage() {} + +func (x *SourceCacheLookupAsk_Query) ProtoReflect() protoreflect.Message { + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[6] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *SourceCacheLookupAsk_Query) GetRowKind() string { + if x != nil { + return x.xxx_hidden_RowKind + } + return "" +} + +func (x *SourceCacheLookupAsk_Query) GetScopeHash() string { + if x != nil { + return x.xxx_hidden_ScopeHash + } + return "" +} + +func (x *SourceCacheLookupAsk_Query) SetRowKind(v string) { + x.xxx_hidden_RowKind = v +} + +func (x *SourceCacheLookupAsk_Query) SetScopeHash(v string) { + x.xxx_hidden_ScopeHash = v +} + +type SourceCacheLookupAsk_Query_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + // One of the sourcecache.RowKind values: "resources", + // "entitlements", "grants". + RowKind string + ScopeHash string +} + +func (b0 SourceCacheLookupAsk_Query_builder) Build() *SourceCacheLookupAsk_Query { + m0 := &SourceCacheLookupAsk_Query{} + b, x := &b0, m0 + _, _ = b, x + x.xxx_hidden_RowKind = b.RowKind + x.xxx_hidden_ScopeHash = b.ScopeHash + return m0 +} + +type SourceCacheLookupAnswers_Answer struct { + state protoimpl.MessageState `protogen:"opaque.v1"` + xxx_hidden_RowKind string `protobuf:"bytes,1,opt,name=row_kind,json=rowKind,proto3"` + xxx_hidden_ScopeHash string `protobuf:"bytes,2,opt,name=scope_hash,json=scopeHash,proto3"` + xxx_hidden_Found bool `protobuf:"varint,3,opt,name=found,proto3"` + xxx_hidden_Etag string `protobuf:"bytes,4,opt,name=etag,proto3"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SourceCacheLookupAnswers_Answer) Reset() { + *x = SourceCacheLookupAnswers_Answer{} + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SourceCacheLookupAnswers_Answer) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SourceCacheLookupAnswers_Answer) ProtoMessage() {} + +func (x *SourceCacheLookupAnswers_Answer) ProtoReflect() protoreflect.Message { + mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[7] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *SourceCacheLookupAnswers_Answer) GetRowKind() string { + if x != nil { + return x.xxx_hidden_RowKind + } + return "" +} + +func (x *SourceCacheLookupAnswers_Answer) GetScopeHash() string { + if x != nil { + return x.xxx_hidden_ScopeHash + } + return "" +} + +func (x *SourceCacheLookupAnswers_Answer) GetFound() bool { + if x != nil { + return x.xxx_hidden_Found + } + return false +} + +func (x *SourceCacheLookupAnswers_Answer) GetEtag() string { + if x != nil { + return x.xxx_hidden_Etag + } + return "" +} + +func (x *SourceCacheLookupAnswers_Answer) SetRowKind(v string) { + x.xxx_hidden_RowKind = v +} + +func (x *SourceCacheLookupAnswers_Answer) SetScopeHash(v string) { + x.xxx_hidden_ScopeHash = v +} + +func (x *SourceCacheLookupAnswers_Answer) SetFound(v bool) { + x.xxx_hidden_Found = v +} + +func (x *SourceCacheLookupAnswers_Answer) SetEtag(v string) { + x.xxx_hidden_Etag = v +} + +type SourceCacheLookupAnswers_Answer_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + RowKind string + ScopeHash string + Found bool + // The previous sync's validator; empty when found is false. Cap + // matches the lookup RPC (Graph delta tokens run long). + Etag string +} + +func (b0 SourceCacheLookupAnswers_Answer_builder) Build() *SourceCacheLookupAnswers_Answer { + m0 := &SourceCacheLookupAnswers_Answer{} + b, x := &b0, m0 + _, _ = b, x + x.xxx_hidden_RowKind = b.RowKind + x.xxx_hidden_ScopeHash = b.ScopeHash + x.xxx_hidden_Found = b.Found + x.xxx_hidden_Etag = b.Etag + return m0 +} + +var File_c1_connector_v2_annotation_source_cache_proto protoreflect.FileDescriptor + +const file_c1_connector_v2_annotation_source_cache_proto_rawDesc = "" + + "\n" + + "-c1/connector/v2/annotation_source_cache.proto\x12\x0fc1.connector.v2\x1a\x17validate/validate.proto\"\x9e\x01\n" + + "\x15SourceCacheCapability\x12?\n" + + "\x04mode\x18\x01 \x01(\x0e2+.c1.connector.v2.SourceCacheCapability.ModeR\x04mode\"D\n" + + "\x04Mode\x12\x14\n" + + "\x10MODE_UNSPECIFIED\x10\x00\x12\x11\n" + + "\rMODE_DISABLED\x10\x01\x12\x13\n" + + "\x0fMODE_READ_WRITE\x10\x02\"\x9a\x01\n" + + "\x10SourceCacheScope\x12\x1d\n" + + "\n" + + "scope_hash\x18\x01 \x01(\tR\tscopeHash\x12\x12\n" + + "\x04etag\x18\x02 \x01(\tR\x04etag\x12\x1f\n" + + "\vdeleted_ids\x18\x03 \x03(\tR\n" + + "deletedIds\x122\n" + + "\x15deleted_principal_ids\x18\x04 \x03(\tR\x13deletedPrincipalIds\"\xb5\x01\n" + + "\x11SourceCacheReplay\x12\x1d\n" + + "\n" + + "scope_hash\x18\x01 \x01(\tR\tscopeHash\x12\x12\n" + + "\x04etag\x18\x02 \x01(\tR\x04etag\x12\x18\n" + + "\aoverlay\x18\x03 \x01(\bR\aoverlay\x12\x1f\n" + + "\vdeleted_ids\x18\x04 \x03(\tR\n" + + "deletedIds\x122\n" + + "\x15deleted_principal_ids\x18\x05 \x03(\tR\x13deletedPrincipalIds\"\x18\n" + + "\x16SourceCacheLookupOffer\"\xc4\x01\n" + + "\x14SourceCacheLookupAsk\x12R\n" + + "\aqueries\x18\x01 \x03(\v2+.c1.connector.v2.SourceCacheLookupAsk.QueryB\v\xfaB\b\x92\x01\x05\b\x01\x10\x80 R\aqueries\x1aX\n" + + "\x05Query\x12$\n" + + "\brow_kind\x18\x01 \x01(\tB\t\xfaB\x06r\x04 \x01(@R\arowKind\x12)\n" + + "\n" + + "scope_hash\x18\x02 \x01(\tB\n" + + "\xfaB\ar\x05 \x01(\x80\x02R\tscopeHash\"\xfa\x01\n" + + "\x18SourceCacheLookupAnswers\x12J\n" + + "\aanswers\x18\x01 \x03(\v20.c1.connector.v2.SourceCacheLookupAnswers.AnswerR\aanswers\x1a\x91\x01\n" + + "\x06Answer\x12$\n" + + "\brow_kind\x18\x01 \x01(\tB\t\xfaB\x06r\x04 \x01(@R\arowKind\x12)\n" + + "\n" + + "scope_hash\x18\x02 \x01(\tB\n" + + "\xfaB\ar\x05 \x01(\x80\x02R\tscopeHash\x12\x14\n" + + "\x05found\x18\x03 \x01(\bR\x05found\x12 \n" + + "\x04etag\x18\x04 \x01(\tB\f\xfaB\tr\a(\x80\x80\x04\xd0\x01\x01R\x04etagB6Z4github.com/conductorone/baton-sdk/pb/c1/connector/v2b\x06proto3" + +var file_c1_connector_v2_annotation_source_cache_proto_enumTypes = make([]protoimpl.EnumInfo, 1) +var file_c1_connector_v2_annotation_source_cache_proto_msgTypes = make([]protoimpl.MessageInfo, 8) +var file_c1_connector_v2_annotation_source_cache_proto_goTypes = []any{ + (SourceCacheCapability_Mode)(0), // 0: c1.connector.v2.SourceCacheCapability.Mode + (*SourceCacheCapability)(nil), // 1: c1.connector.v2.SourceCacheCapability + (*SourceCacheScope)(nil), // 2: c1.connector.v2.SourceCacheScope + (*SourceCacheReplay)(nil), // 3: c1.connector.v2.SourceCacheReplay + (*SourceCacheLookupOffer)(nil), // 4: c1.connector.v2.SourceCacheLookupOffer + (*SourceCacheLookupAsk)(nil), // 5: c1.connector.v2.SourceCacheLookupAsk + (*SourceCacheLookupAnswers)(nil), // 6: c1.connector.v2.SourceCacheLookupAnswers + (*SourceCacheLookupAsk_Query)(nil), // 7: c1.connector.v2.SourceCacheLookupAsk.Query + (*SourceCacheLookupAnswers_Answer)(nil), // 8: c1.connector.v2.SourceCacheLookupAnswers.Answer +} +var file_c1_connector_v2_annotation_source_cache_proto_depIdxs = []int32{ + 0, // 0: c1.connector.v2.SourceCacheCapability.mode:type_name -> c1.connector.v2.SourceCacheCapability.Mode + 7, // 1: c1.connector.v2.SourceCacheLookupAsk.queries:type_name -> c1.connector.v2.SourceCacheLookupAsk.Query + 8, // 2: c1.connector.v2.SourceCacheLookupAnswers.answers:type_name -> c1.connector.v2.SourceCacheLookupAnswers.Answer + 3, // [3:3] is the sub-list for method output_type + 3, // [3:3] is the sub-list for method input_type + 3, // [3:3] is the sub-list for extension type_name + 3, // [3:3] is the sub-list for extension extendee + 0, // [0:3] is the sub-list for field type_name +} + +func init() { file_c1_connector_v2_annotation_source_cache_proto_init() } +func file_c1_connector_v2_annotation_source_cache_proto_init() { + if File_c1_connector_v2_annotation_source_cache_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_c1_connector_v2_annotation_source_cache_proto_rawDesc), len(file_c1_connector_v2_annotation_source_cache_proto_rawDesc)), + NumEnums: 1, + NumMessages: 8, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_c1_connector_v2_annotation_source_cache_proto_goTypes, + DependencyIndexes: file_c1_connector_v2_annotation_source_cache_proto_depIdxs, + EnumInfos: file_c1_connector_v2_annotation_source_cache_proto_enumTypes, + MessageInfos: file_c1_connector_v2_annotation_source_cache_proto_msgTypes, + }.Build() + File_c1_connector_v2_annotation_source_cache_proto = out.File + file_c1_connector_v2_annotation_source_cache_proto_goTypes = nil + file_c1_connector_v2_annotation_source_cache_proto_depIdxs = nil +} diff --git a/pb/c1/connector/v2/annotation_type_scoped_grants.pb.go b/pb/c1/connector/v2/annotation_type_scoped_grants.pb.go new file mode 100644 index 000000000..12b1a230b --- /dev/null +++ b/pb/c1/connector/v2/annotation_type_scoped_grants.pb.go @@ -0,0 +1,255 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.10 +// protoc (unknown) +// source: c1/connector/v2/annotation_type_scoped_grants.proto + +//go:build !protoopaque + +package v2 + +import ( + _ "github.com/envoyproxy/protoc-gen-validate/validate" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// Type-scoped grant ingestion. +// +// Some providers expose change/enumeration APIs whose natural unit is a +// whole collection (or a connector-defined shard of one), not a single +// resource — e.g. Microsoft Graph delta queries, where one stream yields +// membership changes for up to 50 groups. Forcing that shape through the +// per-resource ListGrants fan-out costs one request per resource even when +// nothing changed. +// +// A resource type annotated with TypeScopedGrants is EXCLUDED from the +// per-resource grants fan-out. Instead the syncer issues ListGrants calls +// whose resource carries ONLY the resource type (empty resource id); the +// connector answers with grants for the whole type, across as many +// paginated cursors as it chooses to spawn (see SpawnCursors). +// +// This is the grants-phase analogue of StaticEntitlements: the connector +// takes over enumeration for the type, and every downstream behavior +// (storage, source-cache scopes/replay/tombstones, grant expansion, rate +// limiting, page-token checkpointing) is unchanged because grants are +// self-describing rows. +// +// COMPLETENESS CONTRACT: a full sync of an annotated type must emit (or +// replay, via source-cache annotations) EVERY grant of that type. The +// syncer no longer visits each resource, so completeness rests entirely on +// the connector's enumeration. +type TypeScopedGrants struct { + state protoimpl.MessageState `protogen:"hybrid.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *TypeScopedGrants) Reset() { + *x = TypeScopedGrants{} + mi := &file_c1_connector_v2_annotation_type_scoped_grants_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *TypeScopedGrants) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*TypeScopedGrants) ProtoMessage() {} + +func (x *TypeScopedGrants) ProtoReflect() protoreflect.Message { + mi := &file_c1_connector_v2_annotation_type_scoped_grants_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +type TypeScopedGrants_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + +} + +func (b0 TypeScopedGrants_builder) Build() *TypeScopedGrants { + m0 := &TypeScopedGrants{} + b, x := &b0, m0 + _, _ = b, x + return m0 +} + +// SpawnCursors is attached to a ListGrants response to enqueue additional +// independent sibling cursors. Each token is delivered back to the +// connector as the page token of its own action — scheduled by the +// syncer's worker pool, rate-limited, and checkpointed like any other +// pagination. Honored on BOTH type-scoped and per-resource ListGrants +// responses; the spawned actions inherit the response's resource identity +// (type only for type-scoped calls, type+resource for per-resource calls). +// +// Typical uses: +// +// - Type-scoped planning: the first call computes shard assignments — +// e.g. one 50-id delta filter chunk per cursor — returns no rows, and +// spawns one cursor per shard. Cold enumeration then parallelizes +// across shards instead of serializing through one stream. +// - Per-resource parallel warm revalidation (page-numbered APIs): on the +// first page of a collection the connector already knows every other +// page's URL and stored validator, so it answers page one and spawns +// pages 2..N; the worker pool revalidates them concurrently instead +// of paying one round trip per page serially. +// +// Spawned cursors are ORDINARY pages that happen to be enqueued eagerly. +// The SDK assumes nothing about how they resolve: a spawned page may hit +// its source-cache lookup and replay, miss and fetch cold (e.g. a page +// boundary shifted since the last sync), and may continue a chain via its +// response's next page token. Any page of a fan-out may itself spawn. +// +// Progress accounting counts a resource as covered when its ORIGIN +// action's chain ends; spawned siblings never double-count it. +// +// Tokens are opaque to the SDK. They must be self-contained: a cursor may +// execute after a suspend/resume, on a different worker, so anything the +// connector needs to serve the cursor must be inside the token (or +// re-derivable from it) — for per-resource spawns the resource identity +// rides the action, so tokens only need the page coordinate. +type SpawnCursors struct { + state protoimpl.MessageState `protogen:"hybrid.v1"` + // Page tokens for the sibling cursors to enqueue, one action each. + // Capped per response (spawned actions persist in the checkpointed + // state token); chain additional spawns across pages to fan out wider. + PageTokens []string `protobuf:"bytes,1,rep,name=page_tokens,json=pageTokens,proto3" json:"page_tokens,omitempty"` + // Optional connector estimate of total grants across all cursors of this + // type, for progress reporting. Zero means unknown. + EstimatedTotal int64 `protobuf:"varint,2,opt,name=estimated_total,json=estimatedTotal,proto3" json:"estimated_total,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SpawnCursors) Reset() { + *x = SpawnCursors{} + mi := &file_c1_connector_v2_annotation_type_scoped_grants_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SpawnCursors) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SpawnCursors) ProtoMessage() {} + +func (x *SpawnCursors) ProtoReflect() protoreflect.Message { + mi := &file_c1_connector_v2_annotation_type_scoped_grants_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *SpawnCursors) GetPageTokens() []string { + if x != nil { + return x.PageTokens + } + return nil +} + +func (x *SpawnCursors) GetEstimatedTotal() int64 { + if x != nil { + return x.EstimatedTotal + } + return 0 +} + +func (x *SpawnCursors) SetPageTokens(v []string) { + x.PageTokens = v +} + +func (x *SpawnCursors) SetEstimatedTotal(v int64) { + x.EstimatedTotal = v +} + +type SpawnCursors_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + // Page tokens for the sibling cursors to enqueue, one action each. + // Capped per response (spawned actions persist in the checkpointed + // state token); chain additional spawns across pages to fan out wider. + PageTokens []string + // Optional connector estimate of total grants across all cursors of this + // type, for progress reporting. Zero means unknown. + EstimatedTotal int64 +} + +func (b0 SpawnCursors_builder) Build() *SpawnCursors { + m0 := &SpawnCursors{} + b, x := &b0, m0 + _, _ = b, x + x.PageTokens = b.PageTokens + x.EstimatedTotal = b.EstimatedTotal + return m0 +} + +var File_c1_connector_v2_annotation_type_scoped_grants_proto protoreflect.FileDescriptor + +const file_c1_connector_v2_annotation_type_scoped_grants_proto_rawDesc = "" + + "\n" + + "3c1/connector/v2/annotation_type_scoped_grants.proto\x12\x0fc1.connector.v2\x1a\x17validate/validate.proto\"\x12\n" + + "\x10TypeScopedGrants\"m\n" + + "\fSpawnCursors\x124\n" + + "\vpage_tokens\x18\x01 \x03(\tB\x13\xfaB\x10\x92\x01\r\x10\x80\b\"\br\x06\x10\x01\x18\x80\x80@R\n" + + "pageTokens\x12'\n" + + "\x0festimated_total\x18\x02 \x01(\x03R\x0eestimatedTotalB6Z4github.com/conductorone/baton-sdk/pb/c1/connector/v2b\x06proto3" + +var file_c1_connector_v2_annotation_type_scoped_grants_proto_msgTypes = make([]protoimpl.MessageInfo, 2) +var file_c1_connector_v2_annotation_type_scoped_grants_proto_goTypes = []any{ + (*TypeScopedGrants)(nil), // 0: c1.connector.v2.TypeScopedGrants + (*SpawnCursors)(nil), // 1: c1.connector.v2.SpawnCursors +} +var file_c1_connector_v2_annotation_type_scoped_grants_proto_depIdxs = []int32{ + 0, // [0:0] is the sub-list for method output_type + 0, // [0:0] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_c1_connector_v2_annotation_type_scoped_grants_proto_init() } +func file_c1_connector_v2_annotation_type_scoped_grants_proto_init() { + if File_c1_connector_v2_annotation_type_scoped_grants_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_c1_connector_v2_annotation_type_scoped_grants_proto_rawDesc), len(file_c1_connector_v2_annotation_type_scoped_grants_proto_rawDesc)), + NumEnums: 0, + NumMessages: 2, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_c1_connector_v2_annotation_type_scoped_grants_proto_goTypes, + DependencyIndexes: file_c1_connector_v2_annotation_type_scoped_grants_proto_depIdxs, + MessageInfos: file_c1_connector_v2_annotation_type_scoped_grants_proto_msgTypes, + }.Build() + File_c1_connector_v2_annotation_type_scoped_grants_proto = out.File + file_c1_connector_v2_annotation_type_scoped_grants_proto_goTypes = nil + file_c1_connector_v2_annotation_type_scoped_grants_proto_depIdxs = nil +} diff --git a/pb/c1/connector/v2/annotation_type_scoped_grants.pb.validate.go b/pb/c1/connector/v2/annotation_type_scoped_grants.pb.validate.go new file mode 100644 index 000000000..49a808027 --- /dev/null +++ b/pb/c1/connector/v2/annotation_type_scoped_grants.pb.validate.go @@ -0,0 +1,264 @@ +// Code generated by protoc-gen-validate. DO NOT EDIT. +// source: c1/connector/v2/annotation_type_scoped_grants.proto + +package v2 + +import ( + "bytes" + "errors" + "fmt" + "net" + "net/mail" + "net/url" + "regexp" + "sort" + "strings" + "time" + "unicode/utf8" + + "google.golang.org/protobuf/types/known/anypb" +) + +// ensure the imports are used +var ( + _ = bytes.MinRead + _ = errors.New("") + _ = fmt.Print + _ = utf8.UTFMax + _ = (*regexp.Regexp)(nil) + _ = (*strings.Reader)(nil) + _ = net.IPv4len + _ = time.Duration(0) + _ = (*url.URL)(nil) + _ = (*mail.Address)(nil) + _ = anypb.Any{} + _ = sort.Sort +) + +// Validate checks the field values on TypeScopedGrants with the rules defined +// in the proto definition for this message. If any rules are violated, the +// first error encountered is returned, or nil if there are no violations. +func (m *TypeScopedGrants) Validate() error { + return m.validate(false) +} + +// ValidateAll checks the field values on TypeScopedGrants with the rules +// defined in the proto definition for this message. If any rules are +// violated, the result is a list of violation errors wrapped in +// TypeScopedGrantsMultiError, or nil if none found. +func (m *TypeScopedGrants) ValidateAll() error { + return m.validate(true) +} + +func (m *TypeScopedGrants) validate(all bool) error { + if m == nil { + return nil + } + + var errors []error + + if len(errors) > 0 { + return TypeScopedGrantsMultiError(errors) + } + + return nil +} + +// TypeScopedGrantsMultiError is an error wrapping multiple validation errors +// returned by TypeScopedGrants.ValidateAll() if the designated constraints +// aren't met. +type TypeScopedGrantsMultiError []error + +// Error returns a concatenation of all the error messages it wraps. +func (m TypeScopedGrantsMultiError) Error() string { + msgs := make([]string, 0, len(m)) + for _, err := range m { + msgs = append(msgs, err.Error()) + } + return strings.Join(msgs, "; ") +} + +// AllErrors returns a list of validation violation errors. +func (m TypeScopedGrantsMultiError) AllErrors() []error { return m } + +// TypeScopedGrantsValidationError is the validation error returned by +// TypeScopedGrants.Validate if the designated constraints aren't met. +type TypeScopedGrantsValidationError struct { + field string + reason string + cause error + key bool +} + +// Field function returns field value. +func (e TypeScopedGrantsValidationError) Field() string { return e.field } + +// Reason function returns reason value. +func (e TypeScopedGrantsValidationError) Reason() string { return e.reason } + +// Cause function returns cause value. +func (e TypeScopedGrantsValidationError) Cause() error { return e.cause } + +// Key function returns key value. +func (e TypeScopedGrantsValidationError) Key() bool { return e.key } + +// ErrorName returns error name. +func (e TypeScopedGrantsValidationError) ErrorName() string { return "TypeScopedGrantsValidationError" } + +// Error satisfies the builtin error interface +func (e TypeScopedGrantsValidationError) Error() string { + cause := "" + if e.cause != nil { + cause = fmt.Sprintf(" | caused by: %v", e.cause) + } + + key := "" + if e.key { + key = "key for " + } + + return fmt.Sprintf( + "invalid %sTypeScopedGrants.%s: %s%s", + key, + e.field, + e.reason, + cause) +} + +var _ error = TypeScopedGrantsValidationError{} + +var _ interface { + Field() string + Reason() string + Key() bool + Cause() error + ErrorName() string +} = TypeScopedGrantsValidationError{} + +// Validate checks the field values on SpawnCursors with the rules defined in +// the proto definition for this message. If any rules are violated, the first +// error encountered is returned, or nil if there are no violations. +func (m *SpawnCursors) Validate() error { + return m.validate(false) +} + +// ValidateAll checks the field values on SpawnCursors with the rules defined +// in the proto definition for this message. If any rules are violated, the +// result is a list of violation errors wrapped in SpawnCursorsMultiError, or +// nil if none found. +func (m *SpawnCursors) ValidateAll() error { + return m.validate(true) +} + +func (m *SpawnCursors) validate(all bool) error { + if m == nil { + return nil + } + + var errors []error + + if len(m.GetPageTokens()) > 1024 { + err := SpawnCursorsValidationError{ + field: "PageTokens", + reason: "value must contain no more than 1024 item(s)", + } + if !all { + return err + } + errors = append(errors, err) + } + + for idx, item := range m.GetPageTokens() { + _, _ = idx, item + + if l := utf8.RuneCountInString(item); l < 1 || l > 1048576 { + err := SpawnCursorsValidationError{ + field: fmt.Sprintf("PageTokens[%v]", idx), + reason: "value length must be between 1 and 1048576 runes, inclusive", + } + if !all { + return err + } + errors = append(errors, err) + } + + } + + // no validation rules for EstimatedTotal + + if len(errors) > 0 { + return SpawnCursorsMultiError(errors) + } + + return nil +} + +// SpawnCursorsMultiError is an error wrapping multiple validation errors +// returned by SpawnCursors.ValidateAll() if the designated constraints aren't met. +type SpawnCursorsMultiError []error + +// Error returns a concatenation of all the error messages it wraps. +func (m SpawnCursorsMultiError) Error() string { + msgs := make([]string, 0, len(m)) + for _, err := range m { + msgs = append(msgs, err.Error()) + } + return strings.Join(msgs, "; ") +} + +// AllErrors returns a list of validation violation errors. +func (m SpawnCursorsMultiError) AllErrors() []error { return m } + +// SpawnCursorsValidationError is the validation error returned by +// SpawnCursors.Validate if the designated constraints aren't met. +type SpawnCursorsValidationError struct { + field string + reason string + cause error + key bool +} + +// Field function returns field value. +func (e SpawnCursorsValidationError) Field() string { return e.field } + +// Reason function returns reason value. +func (e SpawnCursorsValidationError) Reason() string { return e.reason } + +// Cause function returns cause value. +func (e SpawnCursorsValidationError) Cause() error { return e.cause } + +// Key function returns key value. +func (e SpawnCursorsValidationError) Key() bool { return e.key } + +// ErrorName returns error name. +func (e SpawnCursorsValidationError) ErrorName() string { return "SpawnCursorsValidationError" } + +// Error satisfies the builtin error interface +func (e SpawnCursorsValidationError) Error() string { + cause := "" + if e.cause != nil { + cause = fmt.Sprintf(" | caused by: %v", e.cause) + } + + key := "" + if e.key { + key = "key for " + } + + return fmt.Sprintf( + "invalid %sSpawnCursors.%s: %s%s", + key, + e.field, + e.reason, + cause) +} + +var _ error = SpawnCursorsValidationError{} + +var _ interface { + Field() string + Reason() string + Key() bool + Cause() error + ErrorName() string +} = SpawnCursorsValidationError{} diff --git a/pb/c1/connector/v2/annotation_type_scoped_grants_protoopaque.pb.go b/pb/c1/connector/v2/annotation_type_scoped_grants_protoopaque.pb.go new file mode 100644 index 000000000..499c83d81 --- /dev/null +++ b/pb/c1/connector/v2/annotation_type_scoped_grants_protoopaque.pb.go @@ -0,0 +1,250 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.10 +// protoc (unknown) +// source: c1/connector/v2/annotation_type_scoped_grants.proto + +//go:build protoopaque + +package v2 + +import ( + _ "github.com/envoyproxy/protoc-gen-validate/validate" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// Type-scoped grant ingestion. +// +// Some providers expose change/enumeration APIs whose natural unit is a +// whole collection (or a connector-defined shard of one), not a single +// resource — e.g. Microsoft Graph delta queries, where one stream yields +// membership changes for up to 50 groups. Forcing that shape through the +// per-resource ListGrants fan-out costs one request per resource even when +// nothing changed. +// +// A resource type annotated with TypeScopedGrants is EXCLUDED from the +// per-resource grants fan-out. Instead the syncer issues ListGrants calls +// whose resource carries ONLY the resource type (empty resource id); the +// connector answers with grants for the whole type, across as many +// paginated cursors as it chooses to spawn (see SpawnCursors). +// +// This is the grants-phase analogue of StaticEntitlements: the connector +// takes over enumeration for the type, and every downstream behavior +// (storage, source-cache scopes/replay/tombstones, grant expansion, rate +// limiting, page-token checkpointing) is unchanged because grants are +// self-describing rows. +// +// COMPLETENESS CONTRACT: a full sync of an annotated type must emit (or +// replay, via source-cache annotations) EVERY grant of that type. The +// syncer no longer visits each resource, so completeness rests entirely on +// the connector's enumeration. +type TypeScopedGrants struct { + state protoimpl.MessageState `protogen:"opaque.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *TypeScopedGrants) Reset() { + *x = TypeScopedGrants{} + mi := &file_c1_connector_v2_annotation_type_scoped_grants_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *TypeScopedGrants) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*TypeScopedGrants) ProtoMessage() {} + +func (x *TypeScopedGrants) ProtoReflect() protoreflect.Message { + mi := &file_c1_connector_v2_annotation_type_scoped_grants_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +type TypeScopedGrants_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + +} + +func (b0 TypeScopedGrants_builder) Build() *TypeScopedGrants { + m0 := &TypeScopedGrants{} + b, x := &b0, m0 + _, _ = b, x + return m0 +} + +// SpawnCursors is attached to a ListGrants response to enqueue additional +// independent sibling cursors. Each token is delivered back to the +// connector as the page token of its own action — scheduled by the +// syncer's worker pool, rate-limited, and checkpointed like any other +// pagination. Honored on BOTH type-scoped and per-resource ListGrants +// responses; the spawned actions inherit the response's resource identity +// (type only for type-scoped calls, type+resource for per-resource calls). +// +// Typical uses: +// +// - Type-scoped planning: the first call computes shard assignments — +// e.g. one 50-id delta filter chunk per cursor — returns no rows, and +// spawns one cursor per shard. Cold enumeration then parallelizes +// across shards instead of serializing through one stream. +// - Per-resource parallel warm revalidation (page-numbered APIs): on the +// first page of a collection the connector already knows every other +// page's URL and stored validator, so it answers page one and spawns +// pages 2..N; the worker pool revalidates them concurrently instead +// of paying one round trip per page serially. +// +// Spawned cursors are ORDINARY pages that happen to be enqueued eagerly. +// The SDK assumes nothing about how they resolve: a spawned page may hit +// its source-cache lookup and replay, miss and fetch cold (e.g. a page +// boundary shifted since the last sync), and may continue a chain via its +// response's next page token. Any page of a fan-out may itself spawn. +// +// Progress accounting counts a resource as covered when its ORIGIN +// action's chain ends; spawned siblings never double-count it. +// +// Tokens are opaque to the SDK. They must be self-contained: a cursor may +// execute after a suspend/resume, on a different worker, so anything the +// connector needs to serve the cursor must be inside the token (or +// re-derivable from it) — for per-resource spawns the resource identity +// rides the action, so tokens only need the page coordinate. +type SpawnCursors struct { + state protoimpl.MessageState `protogen:"opaque.v1"` + xxx_hidden_PageTokens []string `protobuf:"bytes,1,rep,name=page_tokens,json=pageTokens,proto3"` + xxx_hidden_EstimatedTotal int64 `protobuf:"varint,2,opt,name=estimated_total,json=estimatedTotal,proto3"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SpawnCursors) Reset() { + *x = SpawnCursors{} + mi := &file_c1_connector_v2_annotation_type_scoped_grants_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SpawnCursors) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SpawnCursors) ProtoMessage() {} + +func (x *SpawnCursors) ProtoReflect() protoreflect.Message { + mi := &file_c1_connector_v2_annotation_type_scoped_grants_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *SpawnCursors) GetPageTokens() []string { + if x != nil { + return x.xxx_hidden_PageTokens + } + return nil +} + +func (x *SpawnCursors) GetEstimatedTotal() int64 { + if x != nil { + return x.xxx_hidden_EstimatedTotal + } + return 0 +} + +func (x *SpawnCursors) SetPageTokens(v []string) { + x.xxx_hidden_PageTokens = v +} + +func (x *SpawnCursors) SetEstimatedTotal(v int64) { + x.xxx_hidden_EstimatedTotal = v +} + +type SpawnCursors_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + // Page tokens for the sibling cursors to enqueue, one action each. + // Capped per response (spawned actions persist in the checkpointed + // state token); chain additional spawns across pages to fan out wider. + PageTokens []string + // Optional connector estimate of total grants across all cursors of this + // type, for progress reporting. Zero means unknown. + EstimatedTotal int64 +} + +func (b0 SpawnCursors_builder) Build() *SpawnCursors { + m0 := &SpawnCursors{} + b, x := &b0, m0 + _, _ = b, x + x.xxx_hidden_PageTokens = b.PageTokens + x.xxx_hidden_EstimatedTotal = b.EstimatedTotal + return m0 +} + +var File_c1_connector_v2_annotation_type_scoped_grants_proto protoreflect.FileDescriptor + +const file_c1_connector_v2_annotation_type_scoped_grants_proto_rawDesc = "" + + "\n" + + "3c1/connector/v2/annotation_type_scoped_grants.proto\x12\x0fc1.connector.v2\x1a\x17validate/validate.proto\"\x12\n" + + "\x10TypeScopedGrants\"m\n" + + "\fSpawnCursors\x124\n" + + "\vpage_tokens\x18\x01 \x03(\tB\x13\xfaB\x10\x92\x01\r\x10\x80\b\"\br\x06\x10\x01\x18\x80\x80@R\n" + + "pageTokens\x12'\n" + + "\x0festimated_total\x18\x02 \x01(\x03R\x0eestimatedTotalB6Z4github.com/conductorone/baton-sdk/pb/c1/connector/v2b\x06proto3" + +var file_c1_connector_v2_annotation_type_scoped_grants_proto_msgTypes = make([]protoimpl.MessageInfo, 2) +var file_c1_connector_v2_annotation_type_scoped_grants_proto_goTypes = []any{ + (*TypeScopedGrants)(nil), // 0: c1.connector.v2.TypeScopedGrants + (*SpawnCursors)(nil), // 1: c1.connector.v2.SpawnCursors +} +var file_c1_connector_v2_annotation_type_scoped_grants_proto_depIdxs = []int32{ + 0, // [0:0] is the sub-list for method output_type + 0, // [0:0] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_c1_connector_v2_annotation_type_scoped_grants_proto_init() } +func file_c1_connector_v2_annotation_type_scoped_grants_proto_init() { + if File_c1_connector_v2_annotation_type_scoped_grants_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_c1_connector_v2_annotation_type_scoped_grants_proto_rawDesc), len(file_c1_connector_v2_annotation_type_scoped_grants_proto_rawDesc)), + NumEnums: 0, + NumMessages: 2, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_c1_connector_v2_annotation_type_scoped_grants_proto_goTypes, + DependencyIndexes: file_c1_connector_v2_annotation_type_scoped_grants_proto_depIdxs, + MessageInfos: file_c1_connector_v2_annotation_type_scoped_grants_proto_msgTypes, + }.Build() + File_c1_connector_v2_annotation_type_scoped_grants_proto = out.File + file_c1_connector_v2_annotation_type_scoped_grants_proto_goTypes = nil + file_c1_connector_v2_annotation_type_scoped_grants_proto_depIdxs = nil +} diff --git a/pb/c1/connectorapi/baton/v1/source_cache.pb.go b/pb/c1/connectorapi/baton/v1/source_cache.pb.go new file mode 100644 index 000000000..5d0c2090c --- /dev/null +++ b/pb/c1/connectorapi/baton/v1/source_cache.pb.go @@ -0,0 +1,249 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.10 +// protoc (unknown) +// source: c1/connectorapi/baton/v1/source_cache.proto + +//go:build !protoopaque + +package v1 + +import ( + _ "github.com/envoyproxy/protoc-gen-validate/validate" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type LookupRequest struct { + state protoimpl.MessageState `protogen:"hybrid.v1"` + // Row kind: resources / entitlements / grants + // (pkg/sourcecache.RowKind values). Entries are partitioned by row + // kind, so one scope hash can carry a different validator per kind. + RowKind string `protobuf:"bytes,1,opt,name=row_kind,json=rowKind,proto3" json:"row_kind,omitempty"` + // Connector-defined stable scope identifier (conventionally a hex hash + // of the canonical scope; see pkg/sourcecache.HashScope). Opaque to the + // parent; matched verbatim against the previous sync's source-cache + // entries. + ScopeHash string `protobuf:"bytes,2,opt,name=scope_hash,json=scopeHash,proto3" json:"scope_hash,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *LookupRequest) Reset() { + *x = LookupRequest{} + mi := &file_c1_connectorapi_baton_v1_source_cache_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *LookupRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*LookupRequest) ProtoMessage() {} + +func (x *LookupRequest) ProtoReflect() protoreflect.Message { + mi := &file_c1_connectorapi_baton_v1_source_cache_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *LookupRequest) GetRowKind() string { + if x != nil { + return x.RowKind + } + return "" +} + +func (x *LookupRequest) GetScopeHash() string { + if x != nil { + return x.ScopeHash + } + return "" +} + +func (x *LookupRequest) SetRowKind(v string) { + x.RowKind = v +} + +func (x *LookupRequest) SetScopeHash(v string) { + x.ScopeHash = v +} + +type LookupRequest_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + // Row kind: resources / entitlements / grants + // (pkg/sourcecache.RowKind values). Entries are partitioned by row + // kind, so one scope hash can carry a different validator per kind. + RowKind string + // Connector-defined stable scope identifier (conventionally a hex hash + // of the canonical scope; see pkg/sourcecache.HashScope). Opaque to the + // parent; matched verbatim against the previous sync's source-cache + // entries. + ScopeHash string +} + +func (b0 LookupRequest_builder) Build() *LookupRequest { + m0 := &LookupRequest{} + b, x := &b0, m0 + _, _ = b, x + x.RowKind = b.RowKind + x.ScopeHash = b.ScopeHash + return m0 +} + +type LookupResponse struct { + state protoimpl.MessageState `protogen:"hybrid.v1"` + // False means no prior entry exists for (row_kind, scope_hash): the + // connector must fetch fresh and must not emit SourceCacheReplay for + // this scope. + Found bool `protobuf:"varint,1,opt,name=found,proto3" json:"found,omitempty"` + // The opaque validator the previous sync recorded for this scope (HTTP + // ETag, delta token, ...). Empty when found is false. The cap is a + // sanity bound sized for Microsoft Graph delta tokens, which are known + // to run to thousands of characters; storage imposes no limit. + Etag string `protobuf:"bytes,2,opt,name=etag,proto3" json:"etag,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *LookupResponse) Reset() { + *x = LookupResponse{} + mi := &file_c1_connectorapi_baton_v1_source_cache_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *LookupResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*LookupResponse) ProtoMessage() {} + +func (x *LookupResponse) ProtoReflect() protoreflect.Message { + mi := &file_c1_connectorapi_baton_v1_source_cache_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *LookupResponse) GetFound() bool { + if x != nil { + return x.Found + } + return false +} + +func (x *LookupResponse) GetEtag() string { + if x != nil { + return x.Etag + } + return "" +} + +func (x *LookupResponse) SetFound(v bool) { + x.Found = v +} + +func (x *LookupResponse) SetEtag(v string) { + x.Etag = v +} + +type LookupResponse_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + // False means no prior entry exists for (row_kind, scope_hash): the + // connector must fetch fresh and must not emit SourceCacheReplay for + // this scope. + Found bool + // The opaque validator the previous sync recorded for this scope (HTTP + // ETag, delta token, ...). Empty when found is false. The cap is a + // sanity bound sized for Microsoft Graph delta tokens, which are known + // to run to thousands of characters; storage imposes no limit. + Etag string +} + +func (b0 LookupResponse_builder) Build() *LookupResponse { + m0 := &LookupResponse{} + b, x := &b0, m0 + _, _ = b, x + x.Found = b.Found + x.Etag = b.Etag + return m0 +} + +var File_c1_connectorapi_baton_v1_source_cache_proto protoreflect.FileDescriptor + +const file_c1_connectorapi_baton_v1_source_cache_proto_rawDesc = "" + + "\n" + + "+c1/connectorapi/baton/v1/source_cache.proto\x12\x18c1.connectorapi.baton.v1\x1a\x17validate/validate.proto\"`\n" + + "\rLookupRequest\x12$\n" + + "\brow_kind\x18\x01 \x01(\tB\t\xfaB\x06r\x04\x10\x01\x18@R\arowKind\x12)\n" + + "\n" + + "scope_hash\x18\x02 \x01(\tB\n" + + "\xfaB\ar\x05\x10\x01\x18\x80\x02R\tscopeHash\"E\n" + + "\x0eLookupResponse\x12\x14\n" + + "\x05found\x18\x01 \x01(\bR\x05found\x12\x1d\n" + + "\x04etag\x18\x02 \x01(\tB\t\xfaB\x06r\x04\x18\x80\x80\x04R\x04etag2x\n" + + "\x17BatonSourceCacheService\x12]\n" + + "\x06Lookup\x12'.c1.connectorapi.baton.v1.LookupRequest\x1a(.c1.connectorapi.baton.v1.LookupResponse\"\x00B7Z5gitlab.com/ductone/c1/pkg/pb/c1/connectorapi/baton/v1b\x06proto3" + +var file_c1_connectorapi_baton_v1_source_cache_proto_msgTypes = make([]protoimpl.MessageInfo, 2) +var file_c1_connectorapi_baton_v1_source_cache_proto_goTypes = []any{ + (*LookupRequest)(nil), // 0: c1.connectorapi.baton.v1.LookupRequest + (*LookupResponse)(nil), // 1: c1.connectorapi.baton.v1.LookupResponse +} +var file_c1_connectorapi_baton_v1_source_cache_proto_depIdxs = []int32{ + 0, // 0: c1.connectorapi.baton.v1.BatonSourceCacheService.Lookup:input_type -> c1.connectorapi.baton.v1.LookupRequest + 1, // 1: c1.connectorapi.baton.v1.BatonSourceCacheService.Lookup:output_type -> c1.connectorapi.baton.v1.LookupResponse + 1, // [1:2] is the sub-list for method output_type + 0, // [0:1] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_c1_connectorapi_baton_v1_source_cache_proto_init() } +func file_c1_connectorapi_baton_v1_source_cache_proto_init() { + if File_c1_connectorapi_baton_v1_source_cache_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_c1_connectorapi_baton_v1_source_cache_proto_rawDesc), len(file_c1_connectorapi_baton_v1_source_cache_proto_rawDesc)), + NumEnums: 0, + NumMessages: 2, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_c1_connectorapi_baton_v1_source_cache_proto_goTypes, + DependencyIndexes: file_c1_connectorapi_baton_v1_source_cache_proto_depIdxs, + MessageInfos: file_c1_connectorapi_baton_v1_source_cache_proto_msgTypes, + }.Build() + File_c1_connectorapi_baton_v1_source_cache_proto = out.File + file_c1_connectorapi_baton_v1_source_cache_proto_goTypes = nil + file_c1_connectorapi_baton_v1_source_cache_proto_depIdxs = nil +} diff --git a/pb/c1/connectorapi/baton/v1/source_cache.pb.validate.go b/pb/c1/connectorapi/baton/v1/source_cache.pb.validate.go new file mode 100644 index 000000000..dfc7cca7c --- /dev/null +++ b/pb/c1/connectorapi/baton/v1/source_cache.pb.validate.go @@ -0,0 +1,271 @@ +// Code generated by protoc-gen-validate. DO NOT EDIT. +// source: c1/connectorapi/baton/v1/source_cache.proto + +package v1 + +import ( + "bytes" + "errors" + "fmt" + "net" + "net/mail" + "net/url" + "regexp" + "sort" + "strings" + "time" + "unicode/utf8" + + "google.golang.org/protobuf/types/known/anypb" +) + +// ensure the imports are used +var ( + _ = bytes.MinRead + _ = errors.New("") + _ = fmt.Print + _ = utf8.UTFMax + _ = (*regexp.Regexp)(nil) + _ = (*strings.Reader)(nil) + _ = net.IPv4len + _ = time.Duration(0) + _ = (*url.URL)(nil) + _ = (*mail.Address)(nil) + _ = anypb.Any{} + _ = sort.Sort +) + +// Validate checks the field values on LookupRequest with the rules defined in +// the proto definition for this message. If any rules are violated, the first +// error encountered is returned, or nil if there are no violations. +func (m *LookupRequest) Validate() error { + return m.validate(false) +} + +// ValidateAll checks the field values on LookupRequest with the rules defined +// in the proto definition for this message. If any rules are violated, the +// result is a list of violation errors wrapped in LookupRequestMultiError, or +// nil if none found. +func (m *LookupRequest) ValidateAll() error { + return m.validate(true) +} + +func (m *LookupRequest) validate(all bool) error { + if m == nil { + return nil + } + + var errors []error + + if l := utf8.RuneCountInString(m.GetRowKind()); l < 1 || l > 64 { + err := LookupRequestValidationError{ + field: "RowKind", + reason: "value length must be between 1 and 64 runes, inclusive", + } + if !all { + return err + } + errors = append(errors, err) + } + + if l := utf8.RuneCountInString(m.GetScopeHash()); l < 1 || l > 256 { + err := LookupRequestValidationError{ + field: "ScopeHash", + reason: "value length must be between 1 and 256 runes, inclusive", + } + if !all { + return err + } + errors = append(errors, err) + } + + if len(errors) > 0 { + return LookupRequestMultiError(errors) + } + + return nil +} + +// LookupRequestMultiError is an error wrapping multiple validation errors +// returned by LookupRequest.ValidateAll() if the designated constraints +// aren't met. +type LookupRequestMultiError []error + +// Error returns a concatenation of all the error messages it wraps. +func (m LookupRequestMultiError) Error() string { + msgs := make([]string, 0, len(m)) + for _, err := range m { + msgs = append(msgs, err.Error()) + } + return strings.Join(msgs, "; ") +} + +// AllErrors returns a list of validation violation errors. +func (m LookupRequestMultiError) AllErrors() []error { return m } + +// LookupRequestValidationError is the validation error returned by +// LookupRequest.Validate if the designated constraints aren't met. +type LookupRequestValidationError struct { + field string + reason string + cause error + key bool +} + +// Field function returns field value. +func (e LookupRequestValidationError) Field() string { return e.field } + +// Reason function returns reason value. +func (e LookupRequestValidationError) Reason() string { return e.reason } + +// Cause function returns cause value. +func (e LookupRequestValidationError) Cause() error { return e.cause } + +// Key function returns key value. +func (e LookupRequestValidationError) Key() bool { return e.key } + +// ErrorName returns error name. +func (e LookupRequestValidationError) ErrorName() string { return "LookupRequestValidationError" } + +// Error satisfies the builtin error interface +func (e LookupRequestValidationError) Error() string { + cause := "" + if e.cause != nil { + cause = fmt.Sprintf(" | caused by: %v", e.cause) + } + + key := "" + if e.key { + key = "key for " + } + + return fmt.Sprintf( + "invalid %sLookupRequest.%s: %s%s", + key, + e.field, + e.reason, + cause) +} + +var _ error = LookupRequestValidationError{} + +var _ interface { + Field() string + Reason() string + Key() bool + Cause() error + ErrorName() string +} = LookupRequestValidationError{} + +// Validate checks the field values on LookupResponse with the rules defined in +// the proto definition for this message. If any rules are violated, the first +// error encountered is returned, or nil if there are no violations. +func (m *LookupResponse) Validate() error { + return m.validate(false) +} + +// ValidateAll checks the field values on LookupResponse with the rules defined +// in the proto definition for this message. If any rules are violated, the +// result is a list of violation errors wrapped in LookupResponseMultiError, +// or nil if none found. +func (m *LookupResponse) ValidateAll() error { + return m.validate(true) +} + +func (m *LookupResponse) validate(all bool) error { + if m == nil { + return nil + } + + var errors []error + + // no validation rules for Found + + if utf8.RuneCountInString(m.GetEtag()) > 65536 { + err := LookupResponseValidationError{ + field: "Etag", + reason: "value length must be at most 65536 runes", + } + if !all { + return err + } + errors = append(errors, err) + } + + if len(errors) > 0 { + return LookupResponseMultiError(errors) + } + + return nil +} + +// LookupResponseMultiError is an error wrapping multiple validation errors +// returned by LookupResponse.ValidateAll() if the designated constraints +// aren't met. +type LookupResponseMultiError []error + +// Error returns a concatenation of all the error messages it wraps. +func (m LookupResponseMultiError) Error() string { + msgs := make([]string, 0, len(m)) + for _, err := range m { + msgs = append(msgs, err.Error()) + } + return strings.Join(msgs, "; ") +} + +// AllErrors returns a list of validation violation errors. +func (m LookupResponseMultiError) AllErrors() []error { return m } + +// LookupResponseValidationError is the validation error returned by +// LookupResponse.Validate if the designated constraints aren't met. +type LookupResponseValidationError struct { + field string + reason string + cause error + key bool +} + +// Field function returns field value. +func (e LookupResponseValidationError) Field() string { return e.field } + +// Reason function returns reason value. +func (e LookupResponseValidationError) Reason() string { return e.reason } + +// Cause function returns cause value. +func (e LookupResponseValidationError) Cause() error { return e.cause } + +// Key function returns key value. +func (e LookupResponseValidationError) Key() bool { return e.key } + +// ErrorName returns error name. +func (e LookupResponseValidationError) ErrorName() string { return "LookupResponseValidationError" } + +// Error satisfies the builtin error interface +func (e LookupResponseValidationError) Error() string { + cause := "" + if e.cause != nil { + cause = fmt.Sprintf(" | caused by: %v", e.cause) + } + + key := "" + if e.key { + key = "key for " + } + + return fmt.Sprintf( + "invalid %sLookupResponse.%s: %s%s", + key, + e.field, + e.reason, + cause) +} + +var _ error = LookupResponseValidationError{} + +var _ interface { + Field() string + Reason() string + Key() bool + Cause() error + ErrorName() string +} = LookupResponseValidationError{} diff --git a/pb/c1/connectorapi/baton/v1/source_cache_grpc.pb.go b/pb/c1/connectorapi/baton/v1/source_cache_grpc.pb.go new file mode 100644 index 000000000..8db25cd34 --- /dev/null +++ b/pb/c1/connectorapi/baton/v1/source_cache_grpc.pb.go @@ -0,0 +1,145 @@ +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.5.1 +// - protoc (unknown) +// source: c1/connectorapi/baton/v1/source_cache.proto + +package v1 + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.64.0 or later. +const _ = grpc.SupportPackageIsVersion9 + +const ( + BatonSourceCacheService_Lookup_FullMethodName = "/c1.connectorapi.baton.v1.BatonSourceCacheService/Lookup" +) + +// BatonSourceCacheServiceClient is the client API for BatonSourceCacheService service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +// +// BatonSourceCacheService is the dedicated parent-side RPC the connector +// subprocess uses to ask "do I have a previous validator (etag / delta +// token) for this scope?" before revalidating upstream. +// +// This is intentionally NOT routed through the session-store gRPC service: +// session data goes through the connector's local MemorySessionCache +// (otter), which would subject sync-scoped validator state to generic +// TTL/eviction policies and burn bounded cache weight on it. A dedicated +// service keeps the message shape explicit and the path uncached. +// +// The parent has exactly one active lookup registered at a time (set +// per-sync, cleared at sync end), so no sync_id travels on the wire. +type BatonSourceCacheServiceClient interface { + Lookup(ctx context.Context, in *LookupRequest, opts ...grpc.CallOption) (*LookupResponse, error) +} + +type batonSourceCacheServiceClient struct { + cc grpc.ClientConnInterface +} + +func NewBatonSourceCacheServiceClient(cc grpc.ClientConnInterface) BatonSourceCacheServiceClient { + return &batonSourceCacheServiceClient{cc} +} + +func (c *batonSourceCacheServiceClient) Lookup(ctx context.Context, in *LookupRequest, opts ...grpc.CallOption) (*LookupResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(LookupResponse) + err := c.cc.Invoke(ctx, BatonSourceCacheService_Lookup_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + +// BatonSourceCacheServiceServer is the server API for BatonSourceCacheService service. +// All implementations should embed UnimplementedBatonSourceCacheServiceServer +// for forward compatibility. +// +// BatonSourceCacheService is the dedicated parent-side RPC the connector +// subprocess uses to ask "do I have a previous validator (etag / delta +// token) for this scope?" before revalidating upstream. +// +// This is intentionally NOT routed through the session-store gRPC service: +// session data goes through the connector's local MemorySessionCache +// (otter), which would subject sync-scoped validator state to generic +// TTL/eviction policies and burn bounded cache weight on it. A dedicated +// service keeps the message shape explicit and the path uncached. +// +// The parent has exactly one active lookup registered at a time (set +// per-sync, cleared at sync end), so no sync_id travels on the wire. +type BatonSourceCacheServiceServer interface { + Lookup(context.Context, *LookupRequest) (*LookupResponse, error) +} + +// UnimplementedBatonSourceCacheServiceServer should be embedded to have +// forward compatible implementations. +// +// NOTE: this should be embedded by value instead of pointer to avoid a nil +// pointer dereference when methods are called. +type UnimplementedBatonSourceCacheServiceServer struct{} + +func (UnimplementedBatonSourceCacheServiceServer) Lookup(context.Context, *LookupRequest) (*LookupResponse, error) { + return nil, status.Errorf(codes.Unimplemented, "method Lookup not implemented") +} +func (UnimplementedBatonSourceCacheServiceServer) testEmbeddedByValue() {} + +// UnsafeBatonSourceCacheServiceServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to BatonSourceCacheServiceServer will +// result in compilation errors. +type UnsafeBatonSourceCacheServiceServer interface { + mustEmbedUnimplementedBatonSourceCacheServiceServer() +} + +func RegisterBatonSourceCacheServiceServer(s grpc.ServiceRegistrar, srv BatonSourceCacheServiceServer) { + // If the following call pancis, it indicates UnimplementedBatonSourceCacheServiceServer was + // embedded by pointer and is nil. This will cause panics if an + // unimplemented method is ever invoked, so we test this at initialization + // time to prevent it from happening at runtime later due to I/O. + if t, ok := srv.(interface{ testEmbeddedByValue() }); ok { + t.testEmbeddedByValue() + } + s.RegisterService(&BatonSourceCacheService_ServiceDesc, srv) +} + +func _BatonSourceCacheService_Lookup_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(LookupRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(BatonSourceCacheServiceServer).Lookup(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: BatonSourceCacheService_Lookup_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(BatonSourceCacheServiceServer).Lookup(ctx, req.(*LookupRequest)) + } + return interceptor(ctx, in, info, handler) +} + +// BatonSourceCacheService_ServiceDesc is the grpc.ServiceDesc for BatonSourceCacheService service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var BatonSourceCacheService_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "c1.connectorapi.baton.v1.BatonSourceCacheService", + HandlerType: (*BatonSourceCacheServiceServer)(nil), + Methods: []grpc.MethodDesc{ + { + MethodName: "Lookup", + Handler: _BatonSourceCacheService_Lookup_Handler, + }, + }, + Streams: []grpc.StreamDesc{}, + Metadata: "c1/connectorapi/baton/v1/source_cache.proto", +} diff --git a/pb/c1/connectorapi/baton/v1/source_cache_protoopaque.pb.go b/pb/c1/connectorapi/baton/v1/source_cache_protoopaque.pb.go new file mode 100644 index 000000000..9bfc8953c --- /dev/null +++ b/pb/c1/connectorapi/baton/v1/source_cache_protoopaque.pb.go @@ -0,0 +1,235 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.10 +// protoc (unknown) +// source: c1/connectorapi/baton/v1/source_cache.proto + +//go:build protoopaque + +package v1 + +import ( + _ "github.com/envoyproxy/protoc-gen-validate/validate" + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type LookupRequest struct { + state protoimpl.MessageState `protogen:"opaque.v1"` + xxx_hidden_RowKind string `protobuf:"bytes,1,opt,name=row_kind,json=rowKind,proto3"` + xxx_hidden_ScopeHash string `protobuf:"bytes,2,opt,name=scope_hash,json=scopeHash,proto3"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *LookupRequest) Reset() { + *x = LookupRequest{} + mi := &file_c1_connectorapi_baton_v1_source_cache_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *LookupRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*LookupRequest) ProtoMessage() {} + +func (x *LookupRequest) ProtoReflect() protoreflect.Message { + mi := &file_c1_connectorapi_baton_v1_source_cache_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *LookupRequest) GetRowKind() string { + if x != nil { + return x.xxx_hidden_RowKind + } + return "" +} + +func (x *LookupRequest) GetScopeHash() string { + if x != nil { + return x.xxx_hidden_ScopeHash + } + return "" +} + +func (x *LookupRequest) SetRowKind(v string) { + x.xxx_hidden_RowKind = v +} + +func (x *LookupRequest) SetScopeHash(v string) { + x.xxx_hidden_ScopeHash = v +} + +type LookupRequest_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + // Row kind: resources / entitlements / grants + // (pkg/sourcecache.RowKind values). Entries are partitioned by row + // kind, so one scope hash can carry a different validator per kind. + RowKind string + // Connector-defined stable scope identifier (conventionally a hex hash + // of the canonical scope; see pkg/sourcecache.HashScope). Opaque to the + // parent; matched verbatim against the previous sync's source-cache + // entries. + ScopeHash string +} + +func (b0 LookupRequest_builder) Build() *LookupRequest { + m0 := &LookupRequest{} + b, x := &b0, m0 + _, _ = b, x + x.xxx_hidden_RowKind = b.RowKind + x.xxx_hidden_ScopeHash = b.ScopeHash + return m0 +} + +type LookupResponse struct { + state protoimpl.MessageState `protogen:"opaque.v1"` + xxx_hidden_Found bool `protobuf:"varint,1,opt,name=found,proto3"` + xxx_hidden_Etag string `protobuf:"bytes,2,opt,name=etag,proto3"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *LookupResponse) Reset() { + *x = LookupResponse{} + mi := &file_c1_connectorapi_baton_v1_source_cache_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *LookupResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*LookupResponse) ProtoMessage() {} + +func (x *LookupResponse) ProtoReflect() protoreflect.Message { + mi := &file_c1_connectorapi_baton_v1_source_cache_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *LookupResponse) GetFound() bool { + if x != nil { + return x.xxx_hidden_Found + } + return false +} + +func (x *LookupResponse) GetEtag() string { + if x != nil { + return x.xxx_hidden_Etag + } + return "" +} + +func (x *LookupResponse) SetFound(v bool) { + x.xxx_hidden_Found = v +} + +func (x *LookupResponse) SetEtag(v string) { + x.xxx_hidden_Etag = v +} + +type LookupResponse_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + // False means no prior entry exists for (row_kind, scope_hash): the + // connector must fetch fresh and must not emit SourceCacheReplay for + // this scope. + Found bool + // The opaque validator the previous sync recorded for this scope (HTTP + // ETag, delta token, ...). Empty when found is false. The cap is a + // sanity bound sized for Microsoft Graph delta tokens, which are known + // to run to thousands of characters; storage imposes no limit. + Etag string +} + +func (b0 LookupResponse_builder) Build() *LookupResponse { + m0 := &LookupResponse{} + b, x := &b0, m0 + _, _ = b, x + x.xxx_hidden_Found = b.Found + x.xxx_hidden_Etag = b.Etag + return m0 +} + +var File_c1_connectorapi_baton_v1_source_cache_proto protoreflect.FileDescriptor + +const file_c1_connectorapi_baton_v1_source_cache_proto_rawDesc = "" + + "\n" + + "+c1/connectorapi/baton/v1/source_cache.proto\x12\x18c1.connectorapi.baton.v1\x1a\x17validate/validate.proto\"`\n" + + "\rLookupRequest\x12$\n" + + "\brow_kind\x18\x01 \x01(\tB\t\xfaB\x06r\x04\x10\x01\x18@R\arowKind\x12)\n" + + "\n" + + "scope_hash\x18\x02 \x01(\tB\n" + + "\xfaB\ar\x05\x10\x01\x18\x80\x02R\tscopeHash\"E\n" + + "\x0eLookupResponse\x12\x14\n" + + "\x05found\x18\x01 \x01(\bR\x05found\x12\x1d\n" + + "\x04etag\x18\x02 \x01(\tB\t\xfaB\x06r\x04\x18\x80\x80\x04R\x04etag2x\n" + + "\x17BatonSourceCacheService\x12]\n" + + "\x06Lookup\x12'.c1.connectorapi.baton.v1.LookupRequest\x1a(.c1.connectorapi.baton.v1.LookupResponse\"\x00B7Z5gitlab.com/ductone/c1/pkg/pb/c1/connectorapi/baton/v1b\x06proto3" + +var file_c1_connectorapi_baton_v1_source_cache_proto_msgTypes = make([]protoimpl.MessageInfo, 2) +var file_c1_connectorapi_baton_v1_source_cache_proto_goTypes = []any{ + (*LookupRequest)(nil), // 0: c1.connectorapi.baton.v1.LookupRequest + (*LookupResponse)(nil), // 1: c1.connectorapi.baton.v1.LookupResponse +} +var file_c1_connectorapi_baton_v1_source_cache_proto_depIdxs = []int32{ + 0, // 0: c1.connectorapi.baton.v1.BatonSourceCacheService.Lookup:input_type -> c1.connectorapi.baton.v1.LookupRequest + 1, // 1: c1.connectorapi.baton.v1.BatonSourceCacheService.Lookup:output_type -> c1.connectorapi.baton.v1.LookupResponse + 1, // [1:2] is the sub-list for method output_type + 0, // [0:1] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_c1_connectorapi_baton_v1_source_cache_proto_init() } +func file_c1_connectorapi_baton_v1_source_cache_proto_init() { + if File_c1_connectorapi_baton_v1_source_cache_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_c1_connectorapi_baton_v1_source_cache_proto_rawDesc), len(file_c1_connectorapi_baton_v1_source_cache_proto_rawDesc)), + NumEnums: 0, + NumMessages: 2, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_c1_connectorapi_baton_v1_source_cache_proto_goTypes, + DependencyIndexes: file_c1_connectorapi_baton_v1_source_cache_proto_depIdxs, + MessageInfos: file_c1_connectorapi_baton_v1_source_cache_proto_msgTypes, + }.Build() + File_c1_connectorapi_baton_v1_source_cache_proto = out.File + file_c1_connectorapi_baton_v1_source_cache_proto_goTypes = nil + file_c1_connectorapi_baton_v1_source_cache_proto_depIdxs = nil +} diff --git a/pb/c1/storage/v3/records.pb.go b/pb/c1/storage/v3/records.pb.go index 312f32a0e..90030dbe4 100644 --- a/pb/c1/storage/v3/records.pb.go +++ b/pb/c1/storage/v3/records.pb.go @@ -455,8 +455,11 @@ type ResourceRecord struct { Parent *ResourceRef `protobuf:"bytes,6,opt,name=parent,proto3" json:"parent,omitempty"` Annotations []*anypb.Any `protobuf:"bytes,7,rep,name=annotations,proto3" json:"annotations,omitempty"` DiscoveredAt *timestamppb.Timestamp `protobuf:"bytes,8,opt,name=discovered_at,json=discoveredAt,proto3" json:"discovered_at,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // Source-cache scope stamp (see annotation_source_cache.proto). + // Empty for rows not produced under a source-cache scope. + SourceScopeHash string `protobuf:"bytes,9,opt,name=source_scope_hash,json=sourceScopeHash,proto3" json:"source_scope_hash,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *ResourceRecord) Reset() { @@ -533,6 +536,13 @@ func (x *ResourceRecord) GetDiscoveredAt() *timestamppb.Timestamp { return nil } +func (x *ResourceRecord) GetSourceScopeHash() string { + if x != nil { + return x.SourceScopeHash + } + return "" +} + func (x *ResourceRecord) SetResourceTypeId(v string) { x.ResourceTypeId = v } @@ -561,6 +571,10 @@ func (x *ResourceRecord) SetDiscoveredAt(v *timestamppb.Timestamp) { x.DiscoveredAt = v } +func (x *ResourceRecord) SetSourceScopeHash(v string) { + x.SourceScopeHash = v +} + func (x *ResourceRecord) HasParent() bool { if x == nil { return false @@ -593,6 +607,9 @@ type ResourceRecord_builder struct { Parent *ResourceRef Annotations []*anypb.Any DiscoveredAt *timestamppb.Timestamp + // Source-cache scope stamp (see annotation_source_cache.proto). + // Empty for rows not produced under a source-cache scope. + SourceScopeHash string } func (b0 ResourceRecord_builder) Build() *ResourceRecord { @@ -606,6 +623,7 @@ func (b0 ResourceRecord_builder) Build() *ResourceRecord { x.Parent = b.Parent x.Annotations = b.Annotations x.DiscoveredAt = b.DiscoveredAt + x.SourceScopeHash = b.SourceScopeHash return m0 } @@ -632,8 +650,11 @@ type EntitlementRecord struct { // resource_types table when needed. Consumed by // pkg/sync/syncer.go's principal-type narrowing. GrantableToResourceTypeIds []string `protobuf:"bytes,10,rep,name=grantable_to_resource_type_ids,json=grantableToResourceTypeIds,proto3" json:"grantable_to_resource_type_ids,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // Source-cache scope stamp (see annotation_source_cache.proto). + // Empty for rows not produced under a source-cache scope. + SourceScopeHash string `protobuf:"bytes,11,opt,name=source_scope_hash,json=sourceScopeHash,proto3" json:"source_scope_hash,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *EntitlementRecord) Reset() { @@ -724,6 +745,13 @@ func (x *EntitlementRecord) GetGrantableToResourceTypeIds() []string { return nil } +func (x *EntitlementRecord) GetSourceScopeHash() string { + if x != nil { + return x.SourceScopeHash + } + return "" +} + func (x *EntitlementRecord) SetExternalId(v string) { x.ExternalId = v } @@ -760,6 +788,10 @@ func (x *EntitlementRecord) SetGrantableToResourceTypeIds(v []string) { x.GrantableToResourceTypeIds = v } +func (x *EntitlementRecord) SetSourceScopeHash(v string) { + x.SourceScopeHash = v +} + func (x *EntitlementRecord) HasResource() bool { if x == nil { return false @@ -806,6 +838,9 @@ type EntitlementRecord_builder struct { // resource_types table when needed. Consumed by // pkg/sync/syncer.go's principal-type narrowing. GrantableToResourceTypeIds []string + // Source-cache scope stamp (see annotation_source_cache.proto). + // Empty for rows not produced under a source-cache scope. + SourceScopeHash string } func (b0 EntitlementRecord_builder) Build() *EntitlementRecord { @@ -821,6 +856,7 @@ func (b0 EntitlementRecord_builder) Build() *EntitlementRecord { x.DiscoveredAt = b.DiscoveredAt x.Slug = b.Slug x.GrantableToResourceTypeIds = b.GrantableToResourceTypeIds + x.SourceScopeHash = b.SourceScopeHash return m0 } @@ -846,9 +882,15 @@ type GrantRecord struct { Annotations []*anypb.Any `protobuf:"bytes,8,rep,name=annotations,proto3" json:"annotations,omitempty"` // map — same wire shape as // c1.connector.v2.GrantSources.sources. - Sources map[string]*GrantSourceRecord `protobuf:"bytes,9,rep,name=sources,proto3" json:"sources,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + Sources map[string]*GrantSourceRecord `protobuf:"bytes,9,rep,name=sources,proto3" json:"sources,omitempty" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + // Source-cache scope stamp (see annotation_source_cache.proto). + // Empty for rows not produced under a source-cache scope — notably + // expander-derived grants, which are recreated by expansion each sync + // and never replayed. StoreExpandedGrants preserves this field on + // rewrites of existing rows, exactly like expansion/needs_expansion. + SourceScopeHash string `protobuf:"bytes,10,opt,name=source_scope_hash,json=sourceScopeHash,proto3" json:"source_scope_hash,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *GrantRecord) Reset() { @@ -932,6 +974,13 @@ func (x *GrantRecord) GetSources() map[string]*GrantSourceRecord { return nil } +func (x *GrantRecord) GetSourceScopeHash() string { + if x != nil { + return x.SourceScopeHash + } + return "" +} + func (x *GrantRecord) SetExternalId(v string) { x.ExternalId = v } @@ -964,6 +1013,10 @@ func (x *GrantRecord) SetSources(v map[string]*GrantSourceRecord) { x.Sources = v } +func (x *GrantRecord) SetSourceScopeHash(v string) { + x.SourceScopeHash = v +} + func (x *GrantRecord) HasEntitlement() bool { if x == nil { return false @@ -1032,6 +1085,12 @@ type GrantRecord_builder struct { // map — same wire shape as // c1.connector.v2.GrantSources.sources. Sources map[string]*GrantSourceRecord + // Source-cache scope stamp (see annotation_source_cache.proto). + // Empty for rows not produced under a source-cache scope — notably + // expander-derived grants, which are recreated by expansion each sync + // and never replayed. StoreExpandedGrants preserves this field on + // rewrites of existing rows, exactly like expansion/needs_expansion. + SourceScopeHash string } func (b0 GrantRecord_builder) Build() *GrantRecord { @@ -1046,6 +1105,7 @@ func (b0 GrantRecord_builder) Build() *GrantRecord { x.NeedsExpansion = b.NeedsExpansion x.Annotations = b.Annotations x.Sources = b.Sources + x.SourceScopeHash = b.SourceScopeHash return m0 } @@ -1177,15 +1237,25 @@ func (b0 AssetRecord_builder) Build() *AssetRecord { } type SyncRunRecord struct { - state protoimpl.MessageState `protogen:"hybrid.v1"` - SyncId string `protobuf:"bytes,1,opt,name=sync_id,json=syncId,proto3" json:"sync_id,omitempty"` - Type SyncType `protobuf:"varint,2,opt,name=type,proto3,enum=c1.storage.v3.SyncType" json:"type,omitempty"` - ParentSyncId string `protobuf:"bytes,3,opt,name=parent_sync_id,json=parentSyncId,proto3" json:"parent_sync_id,omitempty"` - StartedAt *timestamppb.Timestamp `protobuf:"bytes,4,opt,name=started_at,json=startedAt,proto3" json:"started_at,omitempty"` - EndedAt *timestamppb.Timestamp `protobuf:"bytes,5,opt,name=ended_at,json=endedAt,proto3" json:"ended_at,omitempty"` - SyncToken string `protobuf:"bytes,6,opt,name=sync_token,json=syncToken,proto3" json:"sync_token,omitempty"` - SupportsDiff bool `protobuf:"varint,7,opt,name=supports_diff,json=supportsDiff,proto3" json:"supports_diff,omitempty"` - LinkedSyncId string `protobuf:"bytes,8,opt,name=linked_sync_id,json=linkedSyncId,proto3" json:"linked_sync_id,omitempty"` + state protoimpl.MessageState `protogen:"hybrid.v1"` + SyncId string `protobuf:"bytes,1,opt,name=sync_id,json=syncId,proto3" json:"sync_id,omitempty"` + Type SyncType `protobuf:"varint,2,opt,name=type,proto3,enum=c1.storage.v3.SyncType" json:"type,omitempty"` + ParentSyncId string `protobuf:"bytes,3,opt,name=parent_sync_id,json=parentSyncId,proto3" json:"parent_sync_id,omitempty"` + StartedAt *timestamppb.Timestamp `protobuf:"bytes,4,opt,name=started_at,json=startedAt,proto3" json:"started_at,omitempty"` + EndedAt *timestamppb.Timestamp `protobuf:"bytes,5,opt,name=ended_at,json=endedAt,proto3" json:"ended_at,omitempty"` + SyncToken string `protobuf:"bytes,6,opt,name=sync_token,json=syncToken,proto3" json:"sync_token,omitempty"` + SupportsDiff bool `protobuf:"varint,7,opt,name=supports_diff,json=supportsDiff,proto3" json:"supports_diff,omitempty"` + LinkedSyncId string `protobuf:"bytes,8,opt,name=linked_sync_id,json=linkedSyncId,proto3" json:"linked_sync_id,omitempty"` + // compacted marks a sync produced by compaction (fold or rebuild) + // rather than by a real connector run. Compacted artifacts are + // keep-newer UPSERT merges — base rows a newer input deleted survive — + // so no input sync's upstream validators (source-cache etags) describe + // their contents. Consequence, enforced by the syncer: a compacted + // sync CANNOT be used as a source-cache replay source; feeding one to + // WithPreviousSyncC1ZPath degrades to a cold (full-fetch) sync. + // Orchestrators deciding which artifact to materialize as the previous + // sync should read this flag instead of guessing from provenance. + Compacted bool `protobuf:"varint,9,opt,name=compacted,proto3" json:"compacted,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -1271,6 +1341,13 @@ func (x *SyncRunRecord) GetLinkedSyncId() string { return "" } +func (x *SyncRunRecord) GetCompacted() bool { + if x != nil { + return x.Compacted + } + return false +} + func (x *SyncRunRecord) SetSyncId(v string) { x.SyncId = v } @@ -1303,6 +1380,10 @@ func (x *SyncRunRecord) SetLinkedSyncId(v string) { x.LinkedSyncId = v } +func (x *SyncRunRecord) SetCompacted(v bool) { + x.Compacted = v +} + func (x *SyncRunRecord) HasStartedAt() bool { if x == nil { return false @@ -1336,6 +1417,16 @@ type SyncRunRecord_builder struct { SyncToken string SupportsDiff bool LinkedSyncId string + // compacted marks a sync produced by compaction (fold or rebuild) + // rather than by a real connector run. Compacted artifacts are + // keep-newer UPSERT merges — base rows a newer input deleted survive — + // so no input sync's upstream validators (source-cache etags) describe + // their contents. Consequence, enforced by the syncer: a compacted + // sync CANNOT be used as a source-cache replay source; feeding one to + // WithPreviousSyncC1ZPath degrades to a cold (full-fetch) sync. + // Orchestrators deciding which artifact to materialize as the previous + // sync should read this flag instead of guessing from provenance. + Compacted bool } func (b0 SyncRunRecord_builder) Build() *SyncRunRecord { @@ -1350,6 +1441,7 @@ func (b0 SyncRunRecord_builder) Build() *SyncRunRecord { x.SyncToken = b.SyncToken x.SupportsDiff = b.SupportsDiff x.LinkedSyncId = b.LinkedSyncId + x.Compacted = b.Compacted return m0 } @@ -1653,6 +1745,130 @@ func (b0 SessionRecord_builder) Build() *SessionRecord { return m0 } +// SourceCacheEntryRecord is the per-scope manifest entry for source-cache +// replay (see c1/connector/v2/annotation_source_cache.proto). One entry +// per (row_kind, scope_hash), written for every freshly fetched scope — +// including zero-row responses — and rewritten on replay with the scope's +// current validator. The previous sync's entries (read from the previous +// c1z) are the lookup surface connectors revalidate against. +type SourceCacheEntryRecord struct { + state protoimpl.MessageState `protogen:"hybrid.v1"` + // Row kind partition: "resources", "entitlements", or "grants" + // (pkg/sourcecache.RowKind values). + RowKind string `protobuf:"bytes,1,opt,name=row_kind,json=rowKind,proto3" json:"row_kind,omitempty"` + // Connector-computed scope hash (lowercase hex). + ScopeHash string `protobuf:"bytes,2,opt,name=scope_hash,json=scopeHash,proto3" json:"scope_hash,omitempty"` + // Opaque upstream validator: HTTP ETag, delta token, etc. + Etag string `protobuf:"bytes,3,opt,name=etag,proto3" json:"etag,omitempty"` + DiscoveredAt *timestamppb.Timestamp `protobuf:"bytes,4,opt,name=discovered_at,json=discoveredAt,proto3" json:"discovered_at,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SourceCacheEntryRecord) Reset() { + *x = SourceCacheEntryRecord{} + mi := &file_c1_storage_v3_records_proto_msgTypes[10] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SourceCacheEntryRecord) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SourceCacheEntryRecord) ProtoMessage() {} + +func (x *SourceCacheEntryRecord) ProtoReflect() protoreflect.Message { + mi := &file_c1_storage_v3_records_proto_msgTypes[10] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *SourceCacheEntryRecord) GetRowKind() string { + if x != nil { + return x.RowKind + } + return "" +} + +func (x *SourceCacheEntryRecord) GetScopeHash() string { + if x != nil { + return x.ScopeHash + } + return "" +} + +func (x *SourceCacheEntryRecord) GetEtag() string { + if x != nil { + return x.Etag + } + return "" +} + +func (x *SourceCacheEntryRecord) GetDiscoveredAt() *timestamppb.Timestamp { + if x != nil { + return x.DiscoveredAt + } + return nil +} + +func (x *SourceCacheEntryRecord) SetRowKind(v string) { + x.RowKind = v +} + +func (x *SourceCacheEntryRecord) SetScopeHash(v string) { + x.ScopeHash = v +} + +func (x *SourceCacheEntryRecord) SetEtag(v string) { + x.Etag = v +} + +func (x *SourceCacheEntryRecord) SetDiscoveredAt(v *timestamppb.Timestamp) { + x.DiscoveredAt = v +} + +func (x *SourceCacheEntryRecord) HasDiscoveredAt() bool { + if x == nil { + return false + } + return x.DiscoveredAt != nil +} + +func (x *SourceCacheEntryRecord) ClearDiscoveredAt() { + x.DiscoveredAt = nil +} + +type SourceCacheEntryRecord_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + // Row kind partition: "resources", "entitlements", or "grants" + // (pkg/sourcecache.RowKind values). + RowKind string + // Connector-computed scope hash (lowercase hex). + ScopeHash string + // Opaque upstream validator: HTTP ETag, delta token, etc. + Etag string + DiscoveredAt *timestamppb.Timestamp +} + +func (b0 SourceCacheEntryRecord_builder) Build() *SourceCacheEntryRecord { + m0 := &SourceCacheEntryRecord{} + b, x := &b0, m0 + _, _ = b, x + x.RowKind = b.RowKind + x.ScopeHash = b.ScopeHash + x.Etag = b.Etag + x.DiscoveredAt = b.DiscoveredAt + return m0 +} + var File_c1_storage_v3_records_proto protoreflect.FileDescriptor const file_c1_storage_v3_records_proto_rawDesc = "" + @@ -1677,7 +1893,7 @@ const file_c1_storage_v3_records_proto_rawDesc = "" + "\rdiscovered_at\x18\x06 \x01(\v2\x1a.google.protobuf.TimestampR\fdiscoveredAt\x12 \n" + "\vdescription\x18\a \x01(\tR\vdescription\x12-\n" + "\x12sourced_externally\x18\b \x01(\bR\x11sourcedExternally:!\x82\xf9+\x1d\n" + - "\x0eresource_types\x12\vexternal_idJ\x04\b\x01\x10\x02R\async_id\"\xbc\x03\n" + + "\x0eresource_types\x12\vexternal_idJ\x04\b\x01\x10\x02R\async_id\"\x98\x04\n" + "\x0eResourceRecord\x12(\n" + "\x10resource_type_id\x18\x02 \x01(\tR\x0eresourceTypeId\x12\x1f\n" + "\vresource_id\x18\x03 \x01(\tR\n" + @@ -1687,8 +1903,10 @@ const file_c1_storage_v3_records_proto_rawDesc = "" + "\x06parent\x18\x06 \x01(\v2\x1a.c1.storage.v3.ResourceRefB.\x8a\xf9+*\n" + "\tby_parent\x1a\x10resource_type_id\x1a\vresource_idR\x06parent\x126\n" + "\vannotations\x18\a \x03(\v2\x14.google.protobuf.AnyR\vannotations\x12?\n" + - "\rdiscovered_at\x18\b \x01(\v2\x1a.google.protobuf.TimestampR\fdiscoveredAt:.\x82\xf9+*\n" + - "\tresources\x12\x10resource_type_id\x12\vresource_idJ\x04\b\x01\x10\x02R\async_id\"\xfe\x03\n" + + "\rdiscovered_at\x18\b \x01(\v2\x1a.google.protobuf.TimestampR\fdiscoveredAt\x12Z\n" + + "\x11source_scope_hash\x18\t \x01(\tB.\x8a\xf9+*\n" + + "\x0fby_source_scope\"\x17source_scope_hash != ''R\x0fsourceScopeHash:.\x82\xf9+*\n" + + "\tresources\x12\x10resource_type_id\x12\vresource_idJ\x04\b\x01\x10\x02R\async_id\"\xda\x04\n" + "\x11EntitlementRecord\x12\x1f\n" + "\vexternal_id\x18\x02 \x01(\tR\n" + "externalId\x12h\n" + @@ -1701,8 +1919,10 @@ const file_c1_storage_v3_records_proto_rawDesc = "" + "\rdiscovered_at\x18\b \x01(\v2\x1a.google.protobuf.TimestampR\fdiscoveredAt\x12\x12\n" + "\x04slug\x18\t \x01(\tR\x04slug\x12B\n" + "\x1egrantable_to_resource_type_ids\x18\n" + - " \x03(\tR\x1agrantableToResourceTypeIds:\x1f\x82\xf9+\x1b\n" + - "\fentitlements\x12\vexternal_idJ\x04\b\x01\x10\x02R\async_id\"\x9a\x06\n" + + " \x03(\tR\x1agrantableToResourceTypeIds\x12Z\n" + + "\x11source_scope_hash\x18\v \x01(\tB.\x8a\xf9+*\n" + + "\x0fby_source_scope\"\x17source_scope_hash != ''R\x0fsourceScopeHash:\x1f\x82\xf9+\x1b\n" + + "\fentitlements\x12\vexternal_idJ\x04\b\x01\x10\x02R\async_id\"\xf6\x06\n" + "\vGrantRecord\x12\x1f\n" + "\vexternal_id\x18\x02 \x01(\tR\n" + "externalId\x12\x98\x01\n" + @@ -1715,7 +1935,10 @@ const file_c1_storage_v3_records_proto_rawDesc = "" + "\x0fneeds_expansion\x18\a \x01(\bB0\x8a\xf9+,\n" + "\x12by_needs_expansion\"\x16needs_expansion = trueR\x0eneedsExpansion\x126\n" + "\vannotations\x18\b \x03(\v2\x14.google.protobuf.AnyR\vannotations\x12A\n" + - "\asources\x18\t \x03(\v2'.c1.storage.v3.GrantRecord.SourcesEntryR\asources\x1a\\\n" + + "\asources\x18\t \x03(\v2'.c1.storage.v3.GrantRecord.SourcesEntryR\asources\x12Z\n" + + "\x11source_scope_hash\x18\n" + + " \x01(\tB.\x8a\xf9+*\n" + + "\x0fby_source_scope\"\x17source_scope_hash != ''R\x0fsourceScopeHash\x1a\\\n" + "\fSourcesEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x126\n" + "\x05value\x18\x02 \x01(\v2 .c1.storage.v3.GrantSourceRecordR\x05value:\x028\x01:\x19\x82\xf9+\x15\n" + @@ -1727,7 +1950,7 @@ const file_c1_storage_v3_records_proto_rawDesc = "" + "\fcontent_type\x18\x03 \x01(\tR\vcontentType\x12\x12\n" + "\x04data\x18\x04 \x01(\fR\x04data\x12?\n" + "\rdiscovered_at\x18\x05 \x01(\v2\x1a.google.protobuf.TimestampR\fdiscoveredAt:\"\x82\xf9+\x1e\n" + - "\x06assets\x12\async_id\x12\vexternal_id\"\xf1\x02\n" + + "\x06assets\x12\async_id\x12\vexternal_id\"\x8f\x03\n" + "\rSyncRunRecord\x12\x17\n" + "\async_id\x18\x01 \x01(\tR\x06syncId\x12+\n" + "\x04type\x18\x02 \x01(\x0e2\x17.c1.storage.v3.SyncTypeR\x04type\x12$\n" + @@ -1738,7 +1961,8 @@ const file_c1_storage_v3_records_proto_rawDesc = "" + "\n" + "sync_token\x18\x06 \x01(\tR\tsyncToken\x12#\n" + "\rsupports_diff\x18\a \x01(\bR\fsupportsDiff\x12$\n" + - "\x0elinked_sync_id\x18\b \x01(\tR\flinkedSyncId:\x18\x82\xf9+\x14\n" + + "\x0elinked_sync_id\x18\b \x01(\tR\flinkedSyncId\x12\x1c\n" + + "\tcompacted\x18\t \x01(\bR\tcompacted:\x18\x82\xf9+\x14\n" + "\tsync_runs\x12\async_id\"\xff\x06\n" + "\x0fSyncStatsRecord\x12\x17\n" + "\async_id\x18\x01 \x01(\tR\x06syncId\x12%\n" + @@ -1765,7 +1989,15 @@ const file_c1_storage_v3_records_proto_rawDesc = "" + "\async_id\x18\x01 \x01(\tR\x06syncId\x12\x10\n" + "\x03key\x18\x02 \x01(\tR\x03key\x12\x14\n" + "\x05value\x18\x03 \x01(\fR\x05value:\x1c\x82\xf9+\x18\n" + - "\bsessions\x12\async_id\x12\x03key*\xae\x01\n" + + "\bsessions\x12\async_id\x12\x03key\"\xd9\x01\n" + + "\x16SourceCacheEntryRecord\x12\x19\n" + + "\brow_kind\x18\x01 \x01(\tR\arowKind\x12\x1d\n" + + "\n" + + "scope_hash\x18\x02 \x01(\tR\tscopeHash\x12\x12\n" + + "\x04etag\x18\x03 \x01(\tR\x04etag\x12?\n" + + "\rdiscovered_at\x18\x04 \x01(\v2\x1a.google.protobuf.TimestampR\fdiscoveredAt:0\x82\xf9+,\n" + + "\x14source_cache_entries\x12\brow_kind\x12\n" + + "scope_hash*\xae\x01\n" + "\bSyncType\x12\x19\n" + "\x15SYNC_TYPE_UNSPECIFIED\x10\x00\x12\x12\n" + "\x0eSYNC_TYPE_FULL\x10\x01\x12\x15\n" + @@ -1775,58 +2007,60 @@ const file_c1_storage_v3_records_proto_rawDesc = "" + "\x1bSYNC_TYPE_PARTIAL_DELETIONS\x10\x05B4Z2github.com/conductorone/baton-sdk/pb/c1/storage/v3b\x06proto3" var file_c1_storage_v3_records_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_c1_storage_v3_records_proto_msgTypes = make([]protoimpl.MessageInfo, 14) +var file_c1_storage_v3_records_proto_msgTypes = make([]protoimpl.MessageInfo, 15) var file_c1_storage_v3_records_proto_goTypes = []any{ - (SyncType)(0), // 0: c1.storage.v3.SyncType - (*GrantExpandableRecord)(nil), // 1: c1.storage.v3.GrantExpandableRecord - (*GrantSourceRecord)(nil), // 2: c1.storage.v3.GrantSourceRecord - (*ResourceTypeRecord)(nil), // 3: c1.storage.v3.ResourceTypeRecord - (*ResourceRecord)(nil), // 4: c1.storage.v3.ResourceRecord - (*EntitlementRecord)(nil), // 5: c1.storage.v3.EntitlementRecord - (*GrantRecord)(nil), // 6: c1.storage.v3.GrantRecord - (*AssetRecord)(nil), // 7: c1.storage.v3.AssetRecord - (*SyncRunRecord)(nil), // 8: c1.storage.v3.SyncRunRecord - (*SyncStatsRecord)(nil), // 9: c1.storage.v3.SyncStatsRecord - (*SessionRecord)(nil), // 10: c1.storage.v3.SessionRecord - nil, // 11: c1.storage.v3.GrantRecord.SourcesEntry - nil, // 12: c1.storage.v3.SyncStatsRecord.ResourcesByResourceTypeEntry - nil, // 13: c1.storage.v3.SyncStatsRecord.GrantsByEntitlementResourceTypeEntry - nil, // 14: c1.storage.v3.SyncStatsRecord.EntitlementsByResourceTypeEntry - (*anypb.Any)(nil), // 15: google.protobuf.Any - (*timestamppb.Timestamp)(nil), // 16: google.protobuf.Timestamp - (*ResourceRef)(nil), // 17: c1.storage.v3.ResourceRef - (*EntitlementRef)(nil), // 18: c1.storage.v3.EntitlementRef - (*PrincipalRef)(nil), // 19: c1.storage.v3.PrincipalRef + (SyncType)(0), // 0: c1.storage.v3.SyncType + (*GrantExpandableRecord)(nil), // 1: c1.storage.v3.GrantExpandableRecord + (*GrantSourceRecord)(nil), // 2: c1.storage.v3.GrantSourceRecord + (*ResourceTypeRecord)(nil), // 3: c1.storage.v3.ResourceTypeRecord + (*ResourceRecord)(nil), // 4: c1.storage.v3.ResourceRecord + (*EntitlementRecord)(nil), // 5: c1.storage.v3.EntitlementRecord + (*GrantRecord)(nil), // 6: c1.storage.v3.GrantRecord + (*AssetRecord)(nil), // 7: c1.storage.v3.AssetRecord + (*SyncRunRecord)(nil), // 8: c1.storage.v3.SyncRunRecord + (*SyncStatsRecord)(nil), // 9: c1.storage.v3.SyncStatsRecord + (*SessionRecord)(nil), // 10: c1.storage.v3.SessionRecord + (*SourceCacheEntryRecord)(nil), // 11: c1.storage.v3.SourceCacheEntryRecord + nil, // 12: c1.storage.v3.GrantRecord.SourcesEntry + nil, // 13: c1.storage.v3.SyncStatsRecord.ResourcesByResourceTypeEntry + nil, // 14: c1.storage.v3.SyncStatsRecord.GrantsByEntitlementResourceTypeEntry + nil, // 15: c1.storage.v3.SyncStatsRecord.EntitlementsByResourceTypeEntry + (*anypb.Any)(nil), // 16: google.protobuf.Any + (*timestamppb.Timestamp)(nil), // 17: google.protobuf.Timestamp + (*ResourceRef)(nil), // 18: c1.storage.v3.ResourceRef + (*EntitlementRef)(nil), // 19: c1.storage.v3.EntitlementRef + (*PrincipalRef)(nil), // 20: c1.storage.v3.PrincipalRef } var file_c1_storage_v3_records_proto_depIdxs = []int32{ - 15, // 0: c1.storage.v3.ResourceTypeRecord.annotations:type_name -> google.protobuf.Any - 16, // 1: c1.storage.v3.ResourceTypeRecord.discovered_at:type_name -> google.protobuf.Timestamp - 17, // 2: c1.storage.v3.ResourceRecord.parent:type_name -> c1.storage.v3.ResourceRef - 15, // 3: c1.storage.v3.ResourceRecord.annotations:type_name -> google.protobuf.Any - 16, // 4: c1.storage.v3.ResourceRecord.discovered_at:type_name -> google.protobuf.Timestamp - 17, // 5: c1.storage.v3.EntitlementRecord.resource:type_name -> c1.storage.v3.ResourceRef - 15, // 6: c1.storage.v3.EntitlementRecord.annotations:type_name -> google.protobuf.Any - 16, // 7: c1.storage.v3.EntitlementRecord.discovered_at:type_name -> google.protobuf.Timestamp - 18, // 8: c1.storage.v3.GrantRecord.entitlement:type_name -> c1.storage.v3.EntitlementRef - 19, // 9: c1.storage.v3.GrantRecord.principal:type_name -> c1.storage.v3.PrincipalRef - 16, // 10: c1.storage.v3.GrantRecord.discovered_at:type_name -> google.protobuf.Timestamp + 16, // 0: c1.storage.v3.ResourceTypeRecord.annotations:type_name -> google.protobuf.Any + 17, // 1: c1.storage.v3.ResourceTypeRecord.discovered_at:type_name -> google.protobuf.Timestamp + 18, // 2: c1.storage.v3.ResourceRecord.parent:type_name -> c1.storage.v3.ResourceRef + 16, // 3: c1.storage.v3.ResourceRecord.annotations:type_name -> google.protobuf.Any + 17, // 4: c1.storage.v3.ResourceRecord.discovered_at:type_name -> google.protobuf.Timestamp + 18, // 5: c1.storage.v3.EntitlementRecord.resource:type_name -> c1.storage.v3.ResourceRef + 16, // 6: c1.storage.v3.EntitlementRecord.annotations:type_name -> google.protobuf.Any + 17, // 7: c1.storage.v3.EntitlementRecord.discovered_at:type_name -> google.protobuf.Timestamp + 19, // 8: c1.storage.v3.GrantRecord.entitlement:type_name -> c1.storage.v3.EntitlementRef + 20, // 9: c1.storage.v3.GrantRecord.principal:type_name -> c1.storage.v3.PrincipalRef + 17, // 10: c1.storage.v3.GrantRecord.discovered_at:type_name -> google.protobuf.Timestamp 1, // 11: c1.storage.v3.GrantRecord.expansion:type_name -> c1.storage.v3.GrantExpandableRecord - 15, // 12: c1.storage.v3.GrantRecord.annotations:type_name -> google.protobuf.Any - 11, // 13: c1.storage.v3.GrantRecord.sources:type_name -> c1.storage.v3.GrantRecord.SourcesEntry - 16, // 14: c1.storage.v3.AssetRecord.discovered_at:type_name -> google.protobuf.Timestamp + 16, // 12: c1.storage.v3.GrantRecord.annotations:type_name -> google.protobuf.Any + 12, // 13: c1.storage.v3.GrantRecord.sources:type_name -> c1.storage.v3.GrantRecord.SourcesEntry + 17, // 14: c1.storage.v3.AssetRecord.discovered_at:type_name -> google.protobuf.Timestamp 0, // 15: c1.storage.v3.SyncRunRecord.type:type_name -> c1.storage.v3.SyncType - 16, // 16: c1.storage.v3.SyncRunRecord.started_at:type_name -> google.protobuf.Timestamp - 16, // 17: c1.storage.v3.SyncRunRecord.ended_at:type_name -> google.protobuf.Timestamp - 12, // 18: c1.storage.v3.SyncStatsRecord.resources_by_resource_type:type_name -> c1.storage.v3.SyncStatsRecord.ResourcesByResourceTypeEntry - 13, // 19: c1.storage.v3.SyncStatsRecord.grants_by_entitlement_resource_type:type_name -> c1.storage.v3.SyncStatsRecord.GrantsByEntitlementResourceTypeEntry - 14, // 20: c1.storage.v3.SyncStatsRecord.entitlements_by_resource_type:type_name -> c1.storage.v3.SyncStatsRecord.EntitlementsByResourceTypeEntry - 16, // 21: c1.storage.v3.SyncStatsRecord.written_at:type_name -> google.protobuf.Timestamp - 2, // 22: c1.storage.v3.GrantRecord.SourcesEntry.value:type_name -> c1.storage.v3.GrantSourceRecord - 23, // [23:23] is the sub-list for method output_type - 23, // [23:23] is the sub-list for method input_type - 23, // [23:23] is the sub-list for extension type_name - 23, // [23:23] is the sub-list for extension extendee - 0, // [0:23] is the sub-list for field type_name + 17, // 16: c1.storage.v3.SyncRunRecord.started_at:type_name -> google.protobuf.Timestamp + 17, // 17: c1.storage.v3.SyncRunRecord.ended_at:type_name -> google.protobuf.Timestamp + 13, // 18: c1.storage.v3.SyncStatsRecord.resources_by_resource_type:type_name -> c1.storage.v3.SyncStatsRecord.ResourcesByResourceTypeEntry + 14, // 19: c1.storage.v3.SyncStatsRecord.grants_by_entitlement_resource_type:type_name -> c1.storage.v3.SyncStatsRecord.GrantsByEntitlementResourceTypeEntry + 15, // 20: c1.storage.v3.SyncStatsRecord.entitlements_by_resource_type:type_name -> c1.storage.v3.SyncStatsRecord.EntitlementsByResourceTypeEntry + 17, // 21: c1.storage.v3.SyncStatsRecord.written_at:type_name -> google.protobuf.Timestamp + 17, // 22: c1.storage.v3.SourceCacheEntryRecord.discovered_at:type_name -> google.protobuf.Timestamp + 2, // 23: c1.storage.v3.GrantRecord.SourcesEntry.value:type_name -> c1.storage.v3.GrantSourceRecord + 24, // [24:24] is the sub-list for method output_type + 24, // [24:24] is the sub-list for method input_type + 24, // [24:24] is the sub-list for extension type_name + 24, // [24:24] is the sub-list for extension extendee + 0, // [0:24] is the sub-list for field type_name } func init() { file_c1_storage_v3_records_proto_init() } @@ -1842,7 +2076,7 @@ func file_c1_storage_v3_records_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_c1_storage_v3_records_proto_rawDesc), len(file_c1_storage_v3_records_proto_rawDesc)), NumEnums: 1, - NumMessages: 14, + NumMessages: 15, NumExtensions: 0, NumServices: 0, }, diff --git a/pb/c1/storage/v3/records.pb.validate.go b/pb/c1/storage/v3/records.pb.validate.go index 292877242..4d040b1e1 100644 --- a/pb/c1/storage/v3/records.pb.validate.go +++ b/pb/c1/storage/v3/records.pb.validate.go @@ -544,6 +544,8 @@ func (m *ResourceRecord) validate(all bool) error { } } + // no validation rules for SourceScopeHash + if len(errors) > 0 { return ResourceRecordMultiError(errors) } @@ -746,6 +748,8 @@ func (m *EntitlementRecord) validate(all bool) error { // no validation rules for Slug + // no validation rules for SourceScopeHash + if len(errors) > 0 { return EntitlementRecordMultiError(errors) } @@ -1048,6 +1052,8 @@ func (m *GrantRecord) validate(all bool) error { } } + // no validation rules for SourceScopeHash + if len(errors) > 0 { return GrantRecordMultiError(errors) } @@ -1353,6 +1359,8 @@ func (m *SyncRunRecord) validate(all bool) error { // no validation rules for LinkedSyncId + // no validation rules for Compacted + if len(errors) > 0 { return SyncRunRecordMultiError(errors) } @@ -1683,3 +1691,140 @@ var _ interface { Cause() error ErrorName() string } = SessionRecordValidationError{} + +// Validate checks the field values on SourceCacheEntryRecord with the rules +// defined in the proto definition for this message. If any rules are +// violated, the first error encountered is returned, or nil if there are no violations. +func (m *SourceCacheEntryRecord) Validate() error { + return m.validate(false) +} + +// ValidateAll checks the field values on SourceCacheEntryRecord with the rules +// defined in the proto definition for this message. If any rules are +// violated, the result is a list of violation errors wrapped in +// SourceCacheEntryRecordMultiError, or nil if none found. +func (m *SourceCacheEntryRecord) ValidateAll() error { + return m.validate(true) +} + +func (m *SourceCacheEntryRecord) validate(all bool) error { + if m == nil { + return nil + } + + var errors []error + + // no validation rules for RowKind + + // no validation rules for ScopeHash + + // no validation rules for Etag + + if all { + switch v := interface{}(m.GetDiscoveredAt()).(type) { + case interface{ ValidateAll() error }: + if err := v.ValidateAll(); err != nil { + errors = append(errors, SourceCacheEntryRecordValidationError{ + field: "DiscoveredAt", + reason: "embedded message failed validation", + cause: err, + }) + } + case interface{ Validate() error }: + if err := v.Validate(); err != nil { + errors = append(errors, SourceCacheEntryRecordValidationError{ + field: "DiscoveredAt", + reason: "embedded message failed validation", + cause: err, + }) + } + } + } else if v, ok := interface{}(m.GetDiscoveredAt()).(interface{ Validate() error }); ok { + if err := v.Validate(); err != nil { + return SourceCacheEntryRecordValidationError{ + field: "DiscoveredAt", + reason: "embedded message failed validation", + cause: err, + } + } + } + + if len(errors) > 0 { + return SourceCacheEntryRecordMultiError(errors) + } + + return nil +} + +// SourceCacheEntryRecordMultiError is an error wrapping multiple validation +// errors returned by SourceCacheEntryRecord.ValidateAll() if the designated +// constraints aren't met. +type SourceCacheEntryRecordMultiError []error + +// Error returns a concatenation of all the error messages it wraps. +func (m SourceCacheEntryRecordMultiError) Error() string { + msgs := make([]string, 0, len(m)) + for _, err := range m { + msgs = append(msgs, err.Error()) + } + return strings.Join(msgs, "; ") +} + +// AllErrors returns a list of validation violation errors. +func (m SourceCacheEntryRecordMultiError) AllErrors() []error { return m } + +// SourceCacheEntryRecordValidationError is the validation error returned by +// SourceCacheEntryRecord.Validate if the designated constraints aren't met. +type SourceCacheEntryRecordValidationError struct { + field string + reason string + cause error + key bool +} + +// Field function returns field value. +func (e SourceCacheEntryRecordValidationError) Field() string { return e.field } + +// Reason function returns reason value. +func (e SourceCacheEntryRecordValidationError) Reason() string { return e.reason } + +// Cause function returns cause value. +func (e SourceCacheEntryRecordValidationError) Cause() error { return e.cause } + +// Key function returns key value. +func (e SourceCacheEntryRecordValidationError) Key() bool { return e.key } + +// ErrorName returns error name. +func (e SourceCacheEntryRecordValidationError) ErrorName() string { + return "SourceCacheEntryRecordValidationError" +} + +// Error satisfies the builtin error interface +func (e SourceCacheEntryRecordValidationError) Error() string { + cause := "" + if e.cause != nil { + cause = fmt.Sprintf(" | caused by: %v", e.cause) + } + + key := "" + if e.key { + key = "key for " + } + + return fmt.Sprintf( + "invalid %sSourceCacheEntryRecord.%s: %s%s", + key, + e.field, + e.reason, + cause) +} + +var _ error = SourceCacheEntryRecordValidationError{} + +var _ interface { + Field() string + Reason() string + Key() bool + Cause() error + ErrorName() string +} = SourceCacheEntryRecordValidationError{} diff --git a/pb/c1/storage/v3/records_protoopaque.pb.go b/pb/c1/storage/v3/records_protoopaque.pb.go index bf2acc8c6..4830bbcfc 100644 --- a/pb/c1/storage/v3/records_protoopaque.pb.go +++ b/pb/c1/storage/v3/records_protoopaque.pb.go @@ -441,16 +441,17 @@ func (b0 ResourceTypeRecord_builder) Build() *ResourceTypeRecord { } type ResourceRecord struct { - state protoimpl.MessageState `protogen:"opaque.v1"` - xxx_hidden_ResourceTypeId string `protobuf:"bytes,2,opt,name=resource_type_id,json=resourceTypeId,proto3"` - xxx_hidden_ResourceId string `protobuf:"bytes,3,opt,name=resource_id,json=resourceId,proto3"` - xxx_hidden_DisplayName string `protobuf:"bytes,4,opt,name=display_name,json=displayName,proto3"` - xxx_hidden_Description string `protobuf:"bytes,5,opt,name=description,proto3"` - xxx_hidden_Parent *ResourceRef `protobuf:"bytes,6,opt,name=parent,proto3"` - xxx_hidden_Annotations *[]*anypb.Any `protobuf:"bytes,7,rep,name=annotations,proto3"` - xxx_hidden_DiscoveredAt *timestamppb.Timestamp `protobuf:"bytes,8,opt,name=discovered_at,json=discoveredAt,proto3"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"opaque.v1"` + xxx_hidden_ResourceTypeId string `protobuf:"bytes,2,opt,name=resource_type_id,json=resourceTypeId,proto3"` + xxx_hidden_ResourceId string `protobuf:"bytes,3,opt,name=resource_id,json=resourceId,proto3"` + xxx_hidden_DisplayName string `protobuf:"bytes,4,opt,name=display_name,json=displayName,proto3"` + xxx_hidden_Description string `protobuf:"bytes,5,opt,name=description,proto3"` + xxx_hidden_Parent *ResourceRef `protobuf:"bytes,6,opt,name=parent,proto3"` + xxx_hidden_Annotations *[]*anypb.Any `protobuf:"bytes,7,rep,name=annotations,proto3"` + xxx_hidden_DiscoveredAt *timestamppb.Timestamp `protobuf:"bytes,8,opt,name=discovered_at,json=discoveredAt,proto3"` + xxx_hidden_SourceScopeHash string `protobuf:"bytes,9,opt,name=source_scope_hash,json=sourceScopeHash,proto3"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *ResourceRecord) Reset() { @@ -529,6 +530,13 @@ func (x *ResourceRecord) GetDiscoveredAt() *timestamppb.Timestamp { return nil } +func (x *ResourceRecord) GetSourceScopeHash() string { + if x != nil { + return x.xxx_hidden_SourceScopeHash + } + return "" +} + func (x *ResourceRecord) SetResourceTypeId(v string) { x.xxx_hidden_ResourceTypeId = v } @@ -557,6 +565,10 @@ func (x *ResourceRecord) SetDiscoveredAt(v *timestamppb.Timestamp) { x.xxx_hidden_DiscoveredAt = v } +func (x *ResourceRecord) SetSourceScopeHash(v string) { + x.xxx_hidden_SourceScopeHash = v +} + func (x *ResourceRecord) HasParent() bool { if x == nil { return false @@ -589,6 +601,9 @@ type ResourceRecord_builder struct { Parent *ResourceRef Annotations []*anypb.Any DiscoveredAt *timestamppb.Timestamp + // Source-cache scope stamp (see annotation_source_cache.proto). + // Empty for rows not produced under a source-cache scope. + SourceScopeHash string } func (b0 ResourceRecord_builder) Build() *ResourceRecord { @@ -602,6 +617,7 @@ func (b0 ResourceRecord_builder) Build() *ResourceRecord { x.xxx_hidden_Parent = b.Parent x.xxx_hidden_Annotations = &b.Annotations x.xxx_hidden_DiscoveredAt = b.DiscoveredAt + x.xxx_hidden_SourceScopeHash = b.SourceScopeHash return m0 } @@ -616,6 +632,7 @@ type EntitlementRecord struct { xxx_hidden_DiscoveredAt *timestamppb.Timestamp `protobuf:"bytes,8,opt,name=discovered_at,json=discoveredAt,proto3"` xxx_hidden_Slug string `protobuf:"bytes,9,opt,name=slug,proto3"` xxx_hidden_GrantableToResourceTypeIds []string `protobuf:"bytes,10,rep,name=grantable_to_resource_type_ids,json=grantableToResourceTypeIds,proto3"` + xxx_hidden_SourceScopeHash string `protobuf:"bytes,11,opt,name=source_scope_hash,json=sourceScopeHash,proto3"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -710,6 +727,13 @@ func (x *EntitlementRecord) GetGrantableToResourceTypeIds() []string { return nil } +func (x *EntitlementRecord) GetSourceScopeHash() string { + if x != nil { + return x.xxx_hidden_SourceScopeHash + } + return "" +} + func (x *EntitlementRecord) SetExternalId(v string) { x.xxx_hidden_ExternalId = v } @@ -746,6 +770,10 @@ func (x *EntitlementRecord) SetGrantableToResourceTypeIds(v []string) { x.xxx_hidden_GrantableToResourceTypeIds = v } +func (x *EntitlementRecord) SetSourceScopeHash(v string) { + x.xxx_hidden_SourceScopeHash = v +} + func (x *EntitlementRecord) HasResource() bool { if x == nil { return false @@ -792,6 +820,9 @@ type EntitlementRecord_builder struct { // resource_types table when needed. Consumed by // pkg/sync/syncer.go's principal-type narrowing. GrantableToResourceTypeIds []string + // Source-cache scope stamp (see annotation_source_cache.proto). + // Empty for rows not produced under a source-cache scope. + SourceScopeHash string } func (b0 EntitlementRecord_builder) Build() *EntitlementRecord { @@ -807,21 +838,23 @@ func (b0 EntitlementRecord_builder) Build() *EntitlementRecord { x.xxx_hidden_DiscoveredAt = b.DiscoveredAt x.xxx_hidden_Slug = b.Slug x.xxx_hidden_GrantableToResourceTypeIds = b.GrantableToResourceTypeIds + x.xxx_hidden_SourceScopeHash = b.SourceScopeHash return m0 } type GrantRecord struct { - state protoimpl.MessageState `protogen:"opaque.v1"` - xxx_hidden_ExternalId string `protobuf:"bytes,2,opt,name=external_id,json=externalId,proto3"` - xxx_hidden_Entitlement *EntitlementRef `protobuf:"bytes,3,opt,name=entitlement,proto3"` - xxx_hidden_Principal *PrincipalRef `protobuf:"bytes,4,opt,name=principal,proto3"` - xxx_hidden_DiscoveredAt *timestamppb.Timestamp `protobuf:"bytes,5,opt,name=discovered_at,json=discoveredAt,proto3"` - xxx_hidden_Expansion *GrantExpandableRecord `protobuf:"bytes,6,opt,name=expansion,proto3"` - xxx_hidden_NeedsExpansion bool `protobuf:"varint,7,opt,name=needs_expansion,json=needsExpansion,proto3"` - xxx_hidden_Annotations *[]*anypb.Any `protobuf:"bytes,8,rep,name=annotations,proto3"` - xxx_hidden_Sources map[string]*GrantSourceRecord `protobuf:"bytes,9,rep,name=sources,proto3" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"opaque.v1"` + xxx_hidden_ExternalId string `protobuf:"bytes,2,opt,name=external_id,json=externalId,proto3"` + xxx_hidden_Entitlement *EntitlementRef `protobuf:"bytes,3,opt,name=entitlement,proto3"` + xxx_hidden_Principal *PrincipalRef `protobuf:"bytes,4,opt,name=principal,proto3"` + xxx_hidden_DiscoveredAt *timestamppb.Timestamp `protobuf:"bytes,5,opt,name=discovered_at,json=discoveredAt,proto3"` + xxx_hidden_Expansion *GrantExpandableRecord `protobuf:"bytes,6,opt,name=expansion,proto3"` + xxx_hidden_NeedsExpansion bool `protobuf:"varint,7,opt,name=needs_expansion,json=needsExpansion,proto3"` + xxx_hidden_Annotations *[]*anypb.Any `protobuf:"bytes,8,rep,name=annotations,proto3"` + xxx_hidden_Sources map[string]*GrantSourceRecord `protobuf:"bytes,9,rep,name=sources,proto3" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + xxx_hidden_SourceScopeHash string `protobuf:"bytes,10,opt,name=source_scope_hash,json=sourceScopeHash,proto3"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *GrantRecord) Reset() { @@ -907,6 +940,13 @@ func (x *GrantRecord) GetSources() map[string]*GrantSourceRecord { return nil } +func (x *GrantRecord) GetSourceScopeHash() string { + if x != nil { + return x.xxx_hidden_SourceScopeHash + } + return "" +} + func (x *GrantRecord) SetExternalId(v string) { x.xxx_hidden_ExternalId = v } @@ -939,6 +979,10 @@ func (x *GrantRecord) SetSources(v map[string]*GrantSourceRecord) { x.xxx_hidden_Sources = v } +func (x *GrantRecord) SetSourceScopeHash(v string) { + x.xxx_hidden_SourceScopeHash = v +} + func (x *GrantRecord) HasEntitlement() bool { if x == nil { return false @@ -1007,6 +1051,12 @@ type GrantRecord_builder struct { // map — same wire shape as // c1.connector.v2.GrantSources.sources. Sources map[string]*GrantSourceRecord + // Source-cache scope stamp (see annotation_source_cache.proto). + // Empty for rows not produced under a source-cache scope — notably + // expander-derived grants, which are recreated by expansion each sync + // and never replayed. StoreExpandedGrants preserves this field on + // rewrites of existing rows, exactly like expansion/needs_expansion. + SourceScopeHash string } func (b0 GrantRecord_builder) Build() *GrantRecord { @@ -1021,6 +1071,7 @@ func (b0 GrantRecord_builder) Build() *GrantRecord { x.xxx_hidden_NeedsExpansion = b.NeedsExpansion x.xxx_hidden_Annotations = &b.Annotations x.xxx_hidden_Sources = b.Sources + x.xxx_hidden_SourceScopeHash = b.SourceScopeHash return m0 } @@ -1161,6 +1212,7 @@ type SyncRunRecord struct { xxx_hidden_SyncToken string `protobuf:"bytes,6,opt,name=sync_token,json=syncToken,proto3"` xxx_hidden_SupportsDiff bool `protobuf:"varint,7,opt,name=supports_diff,json=supportsDiff,proto3"` xxx_hidden_LinkedSyncId string `protobuf:"bytes,8,opt,name=linked_sync_id,json=linkedSyncId,proto3"` + xxx_hidden_Compacted bool `protobuf:"varint,9,opt,name=compacted,proto3"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -1246,6 +1298,13 @@ func (x *SyncRunRecord) GetLinkedSyncId() string { return "" } +func (x *SyncRunRecord) GetCompacted() bool { + if x != nil { + return x.xxx_hidden_Compacted + } + return false +} + func (x *SyncRunRecord) SetSyncId(v string) { x.xxx_hidden_SyncId = v } @@ -1278,6 +1337,10 @@ func (x *SyncRunRecord) SetLinkedSyncId(v string) { x.xxx_hidden_LinkedSyncId = v } +func (x *SyncRunRecord) SetCompacted(v bool) { + x.xxx_hidden_Compacted = v +} + func (x *SyncRunRecord) HasStartedAt() bool { if x == nil { return false @@ -1311,6 +1374,16 @@ type SyncRunRecord_builder struct { SyncToken string SupportsDiff bool LinkedSyncId string + // compacted marks a sync produced by compaction (fold or rebuild) + // rather than by a real connector run. Compacted artifacts are + // keep-newer UPSERT merges — base rows a newer input deleted survive — + // so no input sync's upstream validators (source-cache etags) describe + // their contents. Consequence, enforced by the syncer: a compacted + // sync CANNOT be used as a source-cache replay source; feeding one to + // WithPreviousSyncC1ZPath degrades to a cold (full-fetch) sync. + // Orchestrators deciding which artifact to materialize as the previous + // sync should read this flag instead of guessing from provenance. + Compacted bool } func (b0 SyncRunRecord_builder) Build() *SyncRunRecord { @@ -1325,6 +1398,7 @@ func (b0 SyncRunRecord_builder) Build() *SyncRunRecord { x.xxx_hidden_SyncToken = b.SyncToken x.xxx_hidden_SupportsDiff = b.SupportsDiff x.xxx_hidden_LinkedSyncId = b.LinkedSyncId + x.xxx_hidden_Compacted = b.Compacted return m0 } @@ -1622,6 +1696,126 @@ func (b0 SessionRecord_builder) Build() *SessionRecord { return m0 } +// SourceCacheEntryRecord is the per-scope manifest entry for source-cache +// replay (see c1/connector/v2/annotation_source_cache.proto). One entry +// per (row_kind, scope_hash), written for every freshly fetched scope — +// including zero-row responses — and rewritten on replay with the scope's +// current validator. The previous sync's entries (read from the previous +// c1z) are the lookup surface connectors revalidate against. +type SourceCacheEntryRecord struct { + state protoimpl.MessageState `protogen:"opaque.v1"` + xxx_hidden_RowKind string `protobuf:"bytes,1,opt,name=row_kind,json=rowKind,proto3"` + xxx_hidden_ScopeHash string `protobuf:"bytes,2,opt,name=scope_hash,json=scopeHash,proto3"` + xxx_hidden_Etag string `protobuf:"bytes,3,opt,name=etag,proto3"` + xxx_hidden_DiscoveredAt *timestamppb.Timestamp `protobuf:"bytes,4,opt,name=discovered_at,json=discoveredAt,proto3"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SourceCacheEntryRecord) Reset() { + *x = SourceCacheEntryRecord{} + mi := &file_c1_storage_v3_records_proto_msgTypes[10] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SourceCacheEntryRecord) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SourceCacheEntryRecord) ProtoMessage() {} + +func (x *SourceCacheEntryRecord) ProtoReflect() protoreflect.Message { + mi := &file_c1_storage_v3_records_proto_msgTypes[10] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *SourceCacheEntryRecord) GetRowKind() string { + if x != nil { + return x.xxx_hidden_RowKind + } + return "" +} + +func (x *SourceCacheEntryRecord) GetScopeHash() string { + if x != nil { + return x.xxx_hidden_ScopeHash + } + return "" +} + +func (x *SourceCacheEntryRecord) GetEtag() string { + if x != nil { + return x.xxx_hidden_Etag + } + return "" +} + +func (x *SourceCacheEntryRecord) GetDiscoveredAt() *timestamppb.Timestamp { + if x != nil { + return x.xxx_hidden_DiscoveredAt + } + return nil +} + +func (x *SourceCacheEntryRecord) SetRowKind(v string) { + x.xxx_hidden_RowKind = v +} + +func (x *SourceCacheEntryRecord) SetScopeHash(v string) { + x.xxx_hidden_ScopeHash = v +} + +func (x *SourceCacheEntryRecord) SetEtag(v string) { + x.xxx_hidden_Etag = v +} + +func (x *SourceCacheEntryRecord) SetDiscoveredAt(v *timestamppb.Timestamp) { + x.xxx_hidden_DiscoveredAt = v +} + +func (x *SourceCacheEntryRecord) HasDiscoveredAt() bool { + if x == nil { + return false + } + return x.xxx_hidden_DiscoveredAt != nil +} + +func (x *SourceCacheEntryRecord) ClearDiscoveredAt() { + x.xxx_hidden_DiscoveredAt = nil +} + +type SourceCacheEntryRecord_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + // Row kind partition: "resources", "entitlements", or "grants" + // (pkg/sourcecache.RowKind values). + RowKind string + // Connector-computed scope hash (lowercase hex). + ScopeHash string + // Opaque upstream validator: HTTP ETag, delta token, etc. + Etag string + DiscoveredAt *timestamppb.Timestamp +} + +func (b0 SourceCacheEntryRecord_builder) Build() *SourceCacheEntryRecord { + m0 := &SourceCacheEntryRecord{} + b, x := &b0, m0 + _, _ = b, x + x.xxx_hidden_RowKind = b.RowKind + x.xxx_hidden_ScopeHash = b.ScopeHash + x.xxx_hidden_Etag = b.Etag + x.xxx_hidden_DiscoveredAt = b.DiscoveredAt + return m0 +} + var File_c1_storage_v3_records_proto protoreflect.FileDescriptor const file_c1_storage_v3_records_proto_rawDesc = "" + @@ -1646,7 +1840,7 @@ const file_c1_storage_v3_records_proto_rawDesc = "" + "\rdiscovered_at\x18\x06 \x01(\v2\x1a.google.protobuf.TimestampR\fdiscoveredAt\x12 \n" + "\vdescription\x18\a \x01(\tR\vdescription\x12-\n" + "\x12sourced_externally\x18\b \x01(\bR\x11sourcedExternally:!\x82\xf9+\x1d\n" + - "\x0eresource_types\x12\vexternal_idJ\x04\b\x01\x10\x02R\async_id\"\xbc\x03\n" + + "\x0eresource_types\x12\vexternal_idJ\x04\b\x01\x10\x02R\async_id\"\x98\x04\n" + "\x0eResourceRecord\x12(\n" + "\x10resource_type_id\x18\x02 \x01(\tR\x0eresourceTypeId\x12\x1f\n" + "\vresource_id\x18\x03 \x01(\tR\n" + @@ -1656,8 +1850,10 @@ const file_c1_storage_v3_records_proto_rawDesc = "" + "\x06parent\x18\x06 \x01(\v2\x1a.c1.storage.v3.ResourceRefB.\x8a\xf9+*\n" + "\tby_parent\x1a\x10resource_type_id\x1a\vresource_idR\x06parent\x126\n" + "\vannotations\x18\a \x03(\v2\x14.google.protobuf.AnyR\vannotations\x12?\n" + - "\rdiscovered_at\x18\b \x01(\v2\x1a.google.protobuf.TimestampR\fdiscoveredAt:.\x82\xf9+*\n" + - "\tresources\x12\x10resource_type_id\x12\vresource_idJ\x04\b\x01\x10\x02R\async_id\"\xfe\x03\n" + + "\rdiscovered_at\x18\b \x01(\v2\x1a.google.protobuf.TimestampR\fdiscoveredAt\x12Z\n" + + "\x11source_scope_hash\x18\t \x01(\tB.\x8a\xf9+*\n" + + "\x0fby_source_scope\"\x17source_scope_hash != ''R\x0fsourceScopeHash:.\x82\xf9+*\n" + + "\tresources\x12\x10resource_type_id\x12\vresource_idJ\x04\b\x01\x10\x02R\async_id\"\xda\x04\n" + "\x11EntitlementRecord\x12\x1f\n" + "\vexternal_id\x18\x02 \x01(\tR\n" + "externalId\x12h\n" + @@ -1670,8 +1866,10 @@ const file_c1_storage_v3_records_proto_rawDesc = "" + "\rdiscovered_at\x18\b \x01(\v2\x1a.google.protobuf.TimestampR\fdiscoveredAt\x12\x12\n" + "\x04slug\x18\t \x01(\tR\x04slug\x12B\n" + "\x1egrantable_to_resource_type_ids\x18\n" + - " \x03(\tR\x1agrantableToResourceTypeIds:\x1f\x82\xf9+\x1b\n" + - "\fentitlements\x12\vexternal_idJ\x04\b\x01\x10\x02R\async_id\"\x9a\x06\n" + + " \x03(\tR\x1agrantableToResourceTypeIds\x12Z\n" + + "\x11source_scope_hash\x18\v \x01(\tB.\x8a\xf9+*\n" + + "\x0fby_source_scope\"\x17source_scope_hash != ''R\x0fsourceScopeHash:\x1f\x82\xf9+\x1b\n" + + "\fentitlements\x12\vexternal_idJ\x04\b\x01\x10\x02R\async_id\"\xf6\x06\n" + "\vGrantRecord\x12\x1f\n" + "\vexternal_id\x18\x02 \x01(\tR\n" + "externalId\x12\x98\x01\n" + @@ -1684,7 +1882,10 @@ const file_c1_storage_v3_records_proto_rawDesc = "" + "\x0fneeds_expansion\x18\a \x01(\bB0\x8a\xf9+,\n" + "\x12by_needs_expansion\"\x16needs_expansion = trueR\x0eneedsExpansion\x126\n" + "\vannotations\x18\b \x03(\v2\x14.google.protobuf.AnyR\vannotations\x12A\n" + - "\asources\x18\t \x03(\v2'.c1.storage.v3.GrantRecord.SourcesEntryR\asources\x1a\\\n" + + "\asources\x18\t \x03(\v2'.c1.storage.v3.GrantRecord.SourcesEntryR\asources\x12Z\n" + + "\x11source_scope_hash\x18\n" + + " \x01(\tB.\x8a\xf9+*\n" + + "\x0fby_source_scope\"\x17source_scope_hash != ''R\x0fsourceScopeHash\x1a\\\n" + "\fSourcesEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x126\n" + "\x05value\x18\x02 \x01(\v2 .c1.storage.v3.GrantSourceRecordR\x05value:\x028\x01:\x19\x82\xf9+\x15\n" + @@ -1696,7 +1897,7 @@ const file_c1_storage_v3_records_proto_rawDesc = "" + "\fcontent_type\x18\x03 \x01(\tR\vcontentType\x12\x12\n" + "\x04data\x18\x04 \x01(\fR\x04data\x12?\n" + "\rdiscovered_at\x18\x05 \x01(\v2\x1a.google.protobuf.TimestampR\fdiscoveredAt:\"\x82\xf9+\x1e\n" + - "\x06assets\x12\async_id\x12\vexternal_id\"\xf1\x02\n" + + "\x06assets\x12\async_id\x12\vexternal_id\"\x8f\x03\n" + "\rSyncRunRecord\x12\x17\n" + "\async_id\x18\x01 \x01(\tR\x06syncId\x12+\n" + "\x04type\x18\x02 \x01(\x0e2\x17.c1.storage.v3.SyncTypeR\x04type\x12$\n" + @@ -1707,7 +1908,8 @@ const file_c1_storage_v3_records_proto_rawDesc = "" + "\n" + "sync_token\x18\x06 \x01(\tR\tsyncToken\x12#\n" + "\rsupports_diff\x18\a \x01(\bR\fsupportsDiff\x12$\n" + - "\x0elinked_sync_id\x18\b \x01(\tR\flinkedSyncId:\x18\x82\xf9+\x14\n" + + "\x0elinked_sync_id\x18\b \x01(\tR\flinkedSyncId\x12\x1c\n" + + "\tcompacted\x18\t \x01(\bR\tcompacted:\x18\x82\xf9+\x14\n" + "\tsync_runs\x12\async_id\"\xff\x06\n" + "\x0fSyncStatsRecord\x12\x17\n" + "\async_id\x18\x01 \x01(\tR\x06syncId\x12%\n" + @@ -1734,7 +1936,15 @@ const file_c1_storage_v3_records_proto_rawDesc = "" + "\async_id\x18\x01 \x01(\tR\x06syncId\x12\x10\n" + "\x03key\x18\x02 \x01(\tR\x03key\x12\x14\n" + "\x05value\x18\x03 \x01(\fR\x05value:\x1c\x82\xf9+\x18\n" + - "\bsessions\x12\async_id\x12\x03key*\xae\x01\n" + + "\bsessions\x12\async_id\x12\x03key\"\xd9\x01\n" + + "\x16SourceCacheEntryRecord\x12\x19\n" + + "\brow_kind\x18\x01 \x01(\tR\arowKind\x12\x1d\n" + + "\n" + + "scope_hash\x18\x02 \x01(\tR\tscopeHash\x12\x12\n" + + "\x04etag\x18\x03 \x01(\tR\x04etag\x12?\n" + + "\rdiscovered_at\x18\x04 \x01(\v2\x1a.google.protobuf.TimestampR\fdiscoveredAt:0\x82\xf9+,\n" + + "\x14source_cache_entries\x12\brow_kind\x12\n" + + "scope_hash*\xae\x01\n" + "\bSyncType\x12\x19\n" + "\x15SYNC_TYPE_UNSPECIFIED\x10\x00\x12\x12\n" + "\x0eSYNC_TYPE_FULL\x10\x01\x12\x15\n" + @@ -1744,58 +1954,60 @@ const file_c1_storage_v3_records_proto_rawDesc = "" + "\x1bSYNC_TYPE_PARTIAL_DELETIONS\x10\x05B4Z2github.com/conductorone/baton-sdk/pb/c1/storage/v3b\x06proto3" var file_c1_storage_v3_records_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_c1_storage_v3_records_proto_msgTypes = make([]protoimpl.MessageInfo, 14) +var file_c1_storage_v3_records_proto_msgTypes = make([]protoimpl.MessageInfo, 15) var file_c1_storage_v3_records_proto_goTypes = []any{ - (SyncType)(0), // 0: c1.storage.v3.SyncType - (*GrantExpandableRecord)(nil), // 1: c1.storage.v3.GrantExpandableRecord - (*GrantSourceRecord)(nil), // 2: c1.storage.v3.GrantSourceRecord - (*ResourceTypeRecord)(nil), // 3: c1.storage.v3.ResourceTypeRecord - (*ResourceRecord)(nil), // 4: c1.storage.v3.ResourceRecord - (*EntitlementRecord)(nil), // 5: c1.storage.v3.EntitlementRecord - (*GrantRecord)(nil), // 6: c1.storage.v3.GrantRecord - (*AssetRecord)(nil), // 7: c1.storage.v3.AssetRecord - (*SyncRunRecord)(nil), // 8: c1.storage.v3.SyncRunRecord - (*SyncStatsRecord)(nil), // 9: c1.storage.v3.SyncStatsRecord - (*SessionRecord)(nil), // 10: c1.storage.v3.SessionRecord - nil, // 11: c1.storage.v3.GrantRecord.SourcesEntry - nil, // 12: c1.storage.v3.SyncStatsRecord.ResourcesByResourceTypeEntry - nil, // 13: c1.storage.v3.SyncStatsRecord.GrantsByEntitlementResourceTypeEntry - nil, // 14: c1.storage.v3.SyncStatsRecord.EntitlementsByResourceTypeEntry - (*anypb.Any)(nil), // 15: google.protobuf.Any - (*timestamppb.Timestamp)(nil), // 16: google.protobuf.Timestamp - (*ResourceRef)(nil), // 17: c1.storage.v3.ResourceRef - (*EntitlementRef)(nil), // 18: c1.storage.v3.EntitlementRef - (*PrincipalRef)(nil), // 19: c1.storage.v3.PrincipalRef + (SyncType)(0), // 0: c1.storage.v3.SyncType + (*GrantExpandableRecord)(nil), // 1: c1.storage.v3.GrantExpandableRecord + (*GrantSourceRecord)(nil), // 2: c1.storage.v3.GrantSourceRecord + (*ResourceTypeRecord)(nil), // 3: c1.storage.v3.ResourceTypeRecord + (*ResourceRecord)(nil), // 4: c1.storage.v3.ResourceRecord + (*EntitlementRecord)(nil), // 5: c1.storage.v3.EntitlementRecord + (*GrantRecord)(nil), // 6: c1.storage.v3.GrantRecord + (*AssetRecord)(nil), // 7: c1.storage.v3.AssetRecord + (*SyncRunRecord)(nil), // 8: c1.storage.v3.SyncRunRecord + (*SyncStatsRecord)(nil), // 9: c1.storage.v3.SyncStatsRecord + (*SessionRecord)(nil), // 10: c1.storage.v3.SessionRecord + (*SourceCacheEntryRecord)(nil), // 11: c1.storage.v3.SourceCacheEntryRecord + nil, // 12: c1.storage.v3.GrantRecord.SourcesEntry + nil, // 13: c1.storage.v3.SyncStatsRecord.ResourcesByResourceTypeEntry + nil, // 14: c1.storage.v3.SyncStatsRecord.GrantsByEntitlementResourceTypeEntry + nil, // 15: c1.storage.v3.SyncStatsRecord.EntitlementsByResourceTypeEntry + (*anypb.Any)(nil), // 16: google.protobuf.Any + (*timestamppb.Timestamp)(nil), // 17: google.protobuf.Timestamp + (*ResourceRef)(nil), // 18: c1.storage.v3.ResourceRef + (*EntitlementRef)(nil), // 19: c1.storage.v3.EntitlementRef + (*PrincipalRef)(nil), // 20: c1.storage.v3.PrincipalRef } var file_c1_storage_v3_records_proto_depIdxs = []int32{ - 15, // 0: c1.storage.v3.ResourceTypeRecord.annotations:type_name -> google.protobuf.Any - 16, // 1: c1.storage.v3.ResourceTypeRecord.discovered_at:type_name -> google.protobuf.Timestamp - 17, // 2: c1.storage.v3.ResourceRecord.parent:type_name -> c1.storage.v3.ResourceRef - 15, // 3: c1.storage.v3.ResourceRecord.annotations:type_name -> google.protobuf.Any - 16, // 4: c1.storage.v3.ResourceRecord.discovered_at:type_name -> google.protobuf.Timestamp - 17, // 5: c1.storage.v3.EntitlementRecord.resource:type_name -> c1.storage.v3.ResourceRef - 15, // 6: c1.storage.v3.EntitlementRecord.annotations:type_name -> google.protobuf.Any - 16, // 7: c1.storage.v3.EntitlementRecord.discovered_at:type_name -> google.protobuf.Timestamp - 18, // 8: c1.storage.v3.GrantRecord.entitlement:type_name -> c1.storage.v3.EntitlementRef - 19, // 9: c1.storage.v3.GrantRecord.principal:type_name -> c1.storage.v3.PrincipalRef - 16, // 10: c1.storage.v3.GrantRecord.discovered_at:type_name -> google.protobuf.Timestamp + 16, // 0: c1.storage.v3.ResourceTypeRecord.annotations:type_name -> google.protobuf.Any + 17, // 1: c1.storage.v3.ResourceTypeRecord.discovered_at:type_name -> google.protobuf.Timestamp + 18, // 2: c1.storage.v3.ResourceRecord.parent:type_name -> c1.storage.v3.ResourceRef + 16, // 3: c1.storage.v3.ResourceRecord.annotations:type_name -> google.protobuf.Any + 17, // 4: c1.storage.v3.ResourceRecord.discovered_at:type_name -> google.protobuf.Timestamp + 18, // 5: c1.storage.v3.EntitlementRecord.resource:type_name -> c1.storage.v3.ResourceRef + 16, // 6: c1.storage.v3.EntitlementRecord.annotations:type_name -> google.protobuf.Any + 17, // 7: c1.storage.v3.EntitlementRecord.discovered_at:type_name -> google.protobuf.Timestamp + 19, // 8: c1.storage.v3.GrantRecord.entitlement:type_name -> c1.storage.v3.EntitlementRef + 20, // 9: c1.storage.v3.GrantRecord.principal:type_name -> c1.storage.v3.PrincipalRef + 17, // 10: c1.storage.v3.GrantRecord.discovered_at:type_name -> google.protobuf.Timestamp 1, // 11: c1.storage.v3.GrantRecord.expansion:type_name -> c1.storage.v3.GrantExpandableRecord - 15, // 12: c1.storage.v3.GrantRecord.annotations:type_name -> google.protobuf.Any - 11, // 13: c1.storage.v3.GrantRecord.sources:type_name -> c1.storage.v3.GrantRecord.SourcesEntry - 16, // 14: c1.storage.v3.AssetRecord.discovered_at:type_name -> google.protobuf.Timestamp + 16, // 12: c1.storage.v3.GrantRecord.annotations:type_name -> google.protobuf.Any + 12, // 13: c1.storage.v3.GrantRecord.sources:type_name -> c1.storage.v3.GrantRecord.SourcesEntry + 17, // 14: c1.storage.v3.AssetRecord.discovered_at:type_name -> google.protobuf.Timestamp 0, // 15: c1.storage.v3.SyncRunRecord.type:type_name -> c1.storage.v3.SyncType - 16, // 16: c1.storage.v3.SyncRunRecord.started_at:type_name -> google.protobuf.Timestamp - 16, // 17: c1.storage.v3.SyncRunRecord.ended_at:type_name -> google.protobuf.Timestamp - 12, // 18: c1.storage.v3.SyncStatsRecord.resources_by_resource_type:type_name -> c1.storage.v3.SyncStatsRecord.ResourcesByResourceTypeEntry - 13, // 19: c1.storage.v3.SyncStatsRecord.grants_by_entitlement_resource_type:type_name -> c1.storage.v3.SyncStatsRecord.GrantsByEntitlementResourceTypeEntry - 14, // 20: c1.storage.v3.SyncStatsRecord.entitlements_by_resource_type:type_name -> c1.storage.v3.SyncStatsRecord.EntitlementsByResourceTypeEntry - 16, // 21: c1.storage.v3.SyncStatsRecord.written_at:type_name -> google.protobuf.Timestamp - 2, // 22: c1.storage.v3.GrantRecord.SourcesEntry.value:type_name -> c1.storage.v3.GrantSourceRecord - 23, // [23:23] is the sub-list for method output_type - 23, // [23:23] is the sub-list for method input_type - 23, // [23:23] is the sub-list for extension type_name - 23, // [23:23] is the sub-list for extension extendee - 0, // [0:23] is the sub-list for field type_name + 17, // 16: c1.storage.v3.SyncRunRecord.started_at:type_name -> google.protobuf.Timestamp + 17, // 17: c1.storage.v3.SyncRunRecord.ended_at:type_name -> google.protobuf.Timestamp + 13, // 18: c1.storage.v3.SyncStatsRecord.resources_by_resource_type:type_name -> c1.storage.v3.SyncStatsRecord.ResourcesByResourceTypeEntry + 14, // 19: c1.storage.v3.SyncStatsRecord.grants_by_entitlement_resource_type:type_name -> c1.storage.v3.SyncStatsRecord.GrantsByEntitlementResourceTypeEntry + 15, // 20: c1.storage.v3.SyncStatsRecord.entitlements_by_resource_type:type_name -> c1.storage.v3.SyncStatsRecord.EntitlementsByResourceTypeEntry + 17, // 21: c1.storage.v3.SyncStatsRecord.written_at:type_name -> google.protobuf.Timestamp + 17, // 22: c1.storage.v3.SourceCacheEntryRecord.discovered_at:type_name -> google.protobuf.Timestamp + 2, // 23: c1.storage.v3.GrantRecord.SourcesEntry.value:type_name -> c1.storage.v3.GrantSourceRecord + 24, // [24:24] is the sub-list for method output_type + 24, // [24:24] is the sub-list for method input_type + 24, // [24:24] is the sub-list for extension type_name + 24, // [24:24] is the sub-list for extension extendee + 0, // [0:24] is the sub-list for field type_name } func init() { file_c1_storage_v3_records_proto_init() } @@ -1811,7 +2023,7 @@ func file_c1_storage_v3_records_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_c1_storage_v3_records_proto_rawDesc), len(file_c1_storage_v3_records_proto_rawDesc)), NumEnums: 1, - NumMessages: 14, + NumMessages: 15, NumExtensions: 0, NumServices: 0, }, diff --git a/pkg/cli/cli.go b/pkg/cli/cli.go index 036e9ee9a..59d872484 100644 --- a/pkg/cli/cli.go +++ b/pkg/cli/cli.go @@ -11,6 +11,7 @@ import ( "github.com/conductorone/baton-sdk/pkg/connectorbuilder" "github.com/conductorone/baton-sdk/pkg/field" + "github.com/conductorone/baton-sdk/pkg/sourcecache" "github.com/conductorone/baton-sdk/pkg/types" "github.com/conductorone/baton-sdk/pkg/types/sessions" "github.com/spf13/cobra" @@ -20,7 +21,12 @@ import ( ) type RunTimeOpts struct { - SessionStore sessions.SessionStore + SessionStore sessions.SessionStore + // SourceCacheLookup resolves a scope's previous-sync validator for + // source-cache replay (see pkg/sourcecache). In subprocess mode this + // is a gRPC client to the parent SDK's BatonSourceCacheService; when + // unset the framework falls back to NoopLookup. + SourceCacheLookup sourcecache.Lookup TokenSource oauth2.TokenSource SelectedAuthMethod string SyncResourceTypeIDs []string diff --git a/pkg/cli/commands.go b/pkg/cli/commands.go index 19dd598f8..11cf9648e 100644 --- a/pkg/cli/commands.go +++ b/pkg/cli/commands.go @@ -9,6 +9,7 @@ import ( "fmt" "io" "os" + "sync" "time" "github.com/conductorone/baton-sdk/pkg/connectorbuilder" @@ -35,6 +36,7 @@ import ( "github.com/conductorone/baton-sdk/pkg/field" "github.com/conductorone/baton-sdk/pkg/logging" "github.com/conductorone/baton-sdk/pkg/session" + "github.com/conductorone/baton-sdk/pkg/sourcecache" "github.com/conductorone/baton-sdk/pkg/tempdir" "github.com/conductorone/baton-sdk/pkg/types/sessions" "github.com/conductorone/baton-sdk/pkg/uhttp" @@ -53,29 +55,55 @@ type eventLogEnabledKey struct{} type ContrainstSetter func(*cobra.Command, field.Configuration) error -// In one shot & service mode, the child process uses this client to connect to the session store server... +// parentControlPlaneDialer opens (lazily, at most once) a single gRPC +// connection from the connector subprocess back to the parent SDK's +// control-plane listener. The same connection multiplexes +// BatonSessionService (connector session data) and BatonSourceCacheService +// (source-cache scope lookups) — the two services share a listener on the +// parent (internal/connector.runServer), so they share a client conn here. // -// which uses the C1Z for storage. Unfortunately the C1Z is instantiated well after we fork the child process, -// so there is quite a bit of pass through. -func getGRPCSessionStoreClient(ctx context.Context, serverCfg *v1.ServerConfig) func(ctx context.Context, opt ...sessions.SessionStoreConstructorOption) (sessions.SessionStore, error) { - return func(_ context.Context, opt ...sessions.SessionStoreConstructorOption) (sessions.SessionStore, error) { - l := ctxzap.Extract(ctx) - clientTLSConfig, err := utls2.ClientConfig(ctx, serverCfg.GetCredential()) - if err != nil { - return nil, err +// dial returns (nil, nil) when the parent did not start a control-plane +// listener (session store disabled); callers substitute no-op +// implementations in that case. +type parentControlPlaneDialer struct { + once sync.Once + conn *grpc.ClientConn + dialErr error + ctx context.Context + cfg *v1.ServerConfig +} + +func newParentControlPlaneDialer(ctx context.Context, serverCfg *v1.ServerConfig) *parentControlPlaneDialer { + return &parentControlPlaneDialer{ctx: ctx, cfg: serverCfg} +} + +// Close releases the underlying gRPC connection if dial ever succeeded. +// Safe when dial never ran or failed, and safe to call multiple times. +func (d *parentControlPlaneDialer) Close() { + if d.conn == nil { + return + } + _ = d.conn.Close() +} + +func (d *parentControlPlaneDialer) dial() (*grpc.ClientConn, error) { + d.once.Do(func() { + if d.cfg.GetSessionStoreListenPort() == 0 { + return } - if serverCfg.GetSessionStoreListenPort() == 0 { - return &session.NoOpSessionStore{}, nil + clientTLSConfig, err := utls2.ClientConfig(d.ctx, d.cfg.GetCredential()) + if err != nil { + d.dialErr = err + return } - // connected, grpc will handle retries for us. - dialCtx, canc := context.WithTimeout(ctx, 5*time.Second) + dialCtx, canc := context.WithTimeout(d.ctx, 5*time.Second) defer canc() var dialErr error var conn *grpc.ClientConn for { conn, err = grpc.DialContext( //nolint:staticcheck // grpc.DialContext is deprecated but we are using it still. - ctx, - fmt.Sprintf("127.0.0.1:%d", serverCfg.GetSessionStoreListenPort()), + d.ctx, + fmt.Sprintf("127.0.0.1:%d", d.cfg.GetSessionStoreListenPort()), grpc.WithTransportCredentials(credentials.NewTLS(clientTLSConfig)), grpc.WithBlock(), //nolint:staticcheck // grpc.WithBlock is deprecated but we are using it still. ) @@ -84,26 +112,59 @@ func getGRPCSessionStoreClient(ctx context.Context, serverCfg *v1.ServerConfig) select { case <-time.After(time.Millisecond * 500): case <-dialCtx.Done(): - return nil, dialErr + d.dialErr = dialErr + return } continue } break } + d.conn = conn + }) + return d.conn, d.dialErr +} +// In one shot & service mode, the child process uses this client to connect to the session store server... +// +// which uses the C1Z for storage. Unfortunately the C1Z is instantiated well after we fork the child process, +// so there is quite a bit of pass through. +func getGRPCSessionStoreClient(ctx context.Context, dialer *parentControlPlaneDialer) func(ctx context.Context, opt ...sessions.SessionStoreConstructorOption) (sessions.SessionStore, error) { + return func(_ context.Context, _ ...sessions.SessionStoreConstructorOption) (sessions.SessionStore, error) { + l := ctxzap.Extract(ctx) + conn, err := dialer.dial() + if err != nil { + return nil, err + } + if conn == nil { + return &session.NoOpSessionStore{}, nil + } client := baton_v1.NewBatonSessionServiceClient(conn) - ss, err := session.NewGRPCSessionStore(ctx, client, opt...) + ss, err := session.NewGRPCSessionStore(ctx, client) if err != nil { - err2 := conn.Close() - if err2 != nil { - l.Error("error closing connection", zap.Error(err2)) - } + l.Error("error creating session store client", zap.Error(err)) return nil, err } return ss, nil } } +// buildGRPCSourceCacheLookup returns the connector-side source-cache Lookup +// that talks to the parent's BatonSourceCacheService over the same +// loopback-TLS connection the session client uses. Returns NoopLookup when +// no parent control-plane listener is configured (matches the "no previous +// sync" behavior). +func buildGRPCSourceCacheLookup(dialer *parentControlPlaneDialer) (sourcecache.Lookup, error) { + conn, err := dialer.dial() + if err != nil { + return nil, err + } + if conn == nil { + return sourcecache.NoopLookup{}, nil + } + client := baton_v1.NewBatonSourceCacheServiceClient(conn) + return sourcecache.NewGRPCLookup(client), nil +} + func MakeMainCommand[T field.Configurable]( ctx context.Context, name string, @@ -377,13 +438,20 @@ func MakeMainCommand[T field.Configurable]( } } - if v.GetBool(field.ParallelSyncField.GetName()) { - opts = append(opts, connectorrunner.WithWorkerCount(-1)) - } - + // Worker count resolves through viper with no zero sentinel in the + // default chain: explicit flag > env (BATON_WORKERS) > config file > + // connector default (field.WithConnectorDefault on WorkerCountField) + // > shared default (0). A resolved 0 — explicit or defaulted — means + // sequential, which is the runner's zero-value behavior (normalized + // to one worker downstream), so no option is appended; -1 means + // auto-detect. The deprecated --parallel-sync only applies when + // workers resolved to 0. workers := v.GetInt(field.WorkerCountField.GetName()) - if workers != 0 { + switch { + case workers != 0: opts = append(opts, connectorrunner.WithWorkerCount(workers)) + case v.GetBool(field.ParallelSyncField.GetName()): + opts = append(opts, connectorrunner.WithWorkerCount(-1)) } c1zTmpDir := tempdir.Resolve(v.GetString("c1z-temp-dir")) @@ -396,13 +464,20 @@ func MakeMainCommand[T field.Configurable]( if v.GetString("external-resource-c1z") != "" { externalResourceC1ZPath := v.GetString("external-resource-c1z") - _, err := os.Open(externalResourceC1ZPath) - if err != nil { + if _, err := os.Stat(externalResourceC1ZPath); err != nil { return fmt.Errorf("the specified external resource c1z file does not exist: %s", externalResourceC1ZPath) } opts = append(opts, connectorrunner.WithExternalResourceC1Z(externalResourceC1ZPath)) } + if v.GetString(field.PreviousSyncC1ZField.GetName()) != "" { + previousSyncC1ZPath := v.GetString(field.PreviousSyncC1ZField.GetName()) + if _, err := os.Stat(previousSyncC1ZPath); err != nil { + return fmt.Errorf("the specified previous sync c1z file does not exist: %s", previousSyncC1ZPath) + } + opts = append(opts, connectorrunner.WithPreviousSyncC1Z(previousSyncC1ZPath)) + } + if v.GetString("external-resource-entitlement-id-filter") != "" { externalResourceEntitlementIdFilter := v.GetString("external-resource-entitlement-id-filter") opts = append(opts, connectorrunner.WithExternalResourceEntitlementFilter(externalResourceEntitlementIdFilter)) @@ -626,7 +701,17 @@ func MakeGRPCServerCommand[T field.Configurable]( runCtx = context.WithValue(runCtx, uhttp.ContextHTTPTimeoutKey, time.Duration(httpTimeout)*time.Second) sessionStoreMaximumSize := v.GetInt(field.ServerSessionStoreMaximumSizeField.GetName()) - sessionConstructor := getGRPCSessionStoreClient(runCtx, serverCfg) + // One dialer per subprocess; the session store client and the + // source-cache lookup client share the underlying gRPC connection + // (both services are registered on the same parent listener in + // internal/connector.runServer). + controlPlaneDialer := newParentControlPlaneDialer(runCtx, serverCfg) + defer controlPlaneDialer.Close() + sessionConstructor := getGRPCSessionStoreClient(runCtx, controlPlaneDialer) + sourceCacheLookup, err := buildGRPCSourceCacheLookup(controlPlaneDialer) + if err != nil { + return fmt.Errorf("failed to build source cache lookup: %w", err) + } c, err := getconnector(runCtx, t, RunTimeOpts{ SessionStore: NewLazyCachingSessionStore(sessionConstructor, func(otterOptions *otter.Options[string, []byte]) { if sessionStoreMaximumSize <= 0 { @@ -635,6 +720,7 @@ func MakeGRPCServerCommand[T field.Configurable]( otterOptions.MaximumWeight = uint64(sessionStoreMaximumSize) } }), + SourceCacheLookup: sourceCacheLookup, SelectedAuthMethod: v.GetString("auth-method"), SyncResourceTypeIDs: v.GetStringSlice("sync-resource-types"), }) diff --git a/pkg/cli/connector_default_test.go b/pkg/cli/connector_default_test.go new file mode 100644 index 000000000..930fda637 --- /dev/null +++ b/pkg/cli/connector_default_test.go @@ -0,0 +1,77 @@ +package cli + +import ( + "testing" + + "github.com/spf13/viper" + "github.com/stretchr/testify/require" + + "github.com/conductorone/baton-sdk/pkg/field" +) + +// TestWithConnectorDefault_WorkerCount pins the per-connector default +// mechanism end to end at the flag layer: the connector's default value +// lands on the cobra flag (so --help shows it and viper resolves it when +// nothing else is set), while explicit values — including the documented +// zero sentinel for sequential sync — still win. +func TestWithConnectorDefault_WorkerCount(t *testing.T) { + newCmdAndViper := func(t *testing.T) (*viper.Viper, *field.SchemaField) { + t.Helper() + f := field.WithConnectorDefault(field.WorkerCountField, 16) + return viper.New(), &f + } + + t.Run("default flows to flag, help, and viper", func(t *testing.T) { + v, f := newCmdAndViper(t) + cmd := setupCommand(t, field.Configuration{Fields: []field.SchemaField{*f}}, v) + + flag := cmd.PersistentFlags().Lookup("workers") + require.NotNil(t, flag) + require.Equal(t, "16", flag.DefValue, "connector default must be the flag default (what --help renders)") + require.False(t, flag.Changed) + + require.NoError(t, v.BindPFlags(cmd.PersistentFlags())) + require.Equal(t, 16, v.GetInt("workers"), "unset resolves to the connector default") + }) + + t.Run("explicit zero beats the connector default (sequential stays reachable)", func(t *testing.T) { + v, f := newCmdAndViper(t) + cmd := setupCommand(t, field.Configuration{Fields: []field.SchemaField{*f}}, v) + + require.NoError(t, cmd.PersistentFlags().Set("workers", "0")) + require.NoError(t, v.BindPFlags(cmd.PersistentFlags())) + require.Equal(t, 0, v.GetInt("workers")) + }) + + t.Run("explicit value beats the connector default", func(t *testing.T) { + v, f := newCmdAndViper(t) + cmd := setupCommand(t, field.Configuration{Fields: []field.SchemaField{*f}}, v) + + require.NoError(t, cmd.PersistentFlags().Set("workers", "4")) + require.NoError(t, v.BindPFlags(cmd.PersistentFlags())) + require.Equal(t, 4, v.GetInt("workers")) + }) + + t.Run("config file beats the connector default", func(t *testing.T) { + dir := t.TempDir() + writeYAML(t, dir, "workers: 8\n") + v := viper.New() + v.SetConfigName("config") + v.SetConfigType("yaml") + v.AddConfigPath(dir) + require.NoError(t, v.ReadInConfig()) + + f := field.WithConnectorDefault(field.WorkerCountField, 16) + cmd := setupCommand(t, field.Configuration{Fields: []field.SchemaField{f}}, v) + require.NoError(t, v.BindPFlags(cmd.PersistentFlags())) + require.Equal(t, 8, v.GetInt("workers")) + }) + + t.Run("original shared field is untouched", func(t *testing.T) { + require.Equal(t, 0, field.WorkerCountField.DefaultValue) + require.False(t, field.WorkerCountField.WasReExported) + f := field.WithConnectorDefault(field.WorkerCountField, 16) + require.Equal(t, 16, f.DefaultValue) + require.True(t, f.WasReExported, "the copy must be re-exported so DefineConfiguration replaces the SDK copy instead of erroring on the duplicate") + }) +} diff --git a/pkg/config/config.go b/pkg/config/config.go index 3586605fa..f250a20ae 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -40,6 +40,7 @@ func RunConnector[T field.Configurable]( } builderOpts = append(builderOpts, connectorbuilder.WithSessionStore(runTimeOpts.SessionStore)) + builderOpts = append(builderOpts, connectorbuilder.WithSourceCache(runTimeOpts.SourceCacheLookup)) c, err := connectorbuilder.NewConnector(ctx, connector, builderOpts...) if err != nil { @@ -160,6 +161,20 @@ func DefineConfigurationV2[T field.Configurable]( } confschema.Fields = fields + // The replay flags are hidden by default (source-cache replay is + // author-opt-in functionality); surface them in help only for + // connectors whose author baked the capability into their runner + // options. Hidden flags still parse, so this is a visibility decision, + // not a behavioral one. + if connectorrunner.DeclaresPreviousSyncCapability(ctx, options...) { + for i, f := range confschema.Fields { + switch f.FieldName { + case field.PreviousSyncC1ZField.FieldName, field.KeepPreviousSyncC1ZField.FieldName: + confschema.Fields[i].SyncerConfig.Hidden = false + } + } + } + // setup CLI with cobra mainCMD := &cobra.Command{ Use: connectorName, @@ -292,6 +307,12 @@ func verifyStructFields[T field.Configurable](schema field.Configuration) error return fmt.Errorf("T must be a struct type, got %v", configType.Kind()) //nolint:staticcheck // we want to capital letter here } for _, field := range schema.Fields { + if field.WasReExported { + // Re-exported shared SDK fields (field.WithConnectorDefault) + // are parsed by the SDK's own flag handling, not the + // connector's configuration struct. + continue + } fieldFound := false for i := 0; i < configType.NumField(); i++ { structField := configType.Field(i) diff --git a/pkg/connectorbuilder/connectorbuilder.go b/pkg/connectorbuilder/connectorbuilder.go index 223beb018..17677d98d 100644 --- a/pkg/connectorbuilder/connectorbuilder.go +++ b/pkg/connectorbuilder/connectorbuilder.go @@ -6,6 +6,7 @@ import ( "fmt" "slices" "sort" + stdsync "sync" "time" "github.com/go-jose/go-jose/v4" @@ -23,6 +24,7 @@ import ( "github.com/conductorone/baton-sdk/pkg/retry" "github.com/conductorone/baton-sdk/pkg/sdk" "github.com/conductorone/baton-sdk/pkg/session" + "github.com/conductorone/baton-sdk/pkg/sourcecache" "github.com/conductorone/baton-sdk/pkg/types" "github.com/conductorone/baton-sdk/pkg/types/sessions" "github.com/conductorone/baton-sdk/pkg/types/tasks" @@ -72,11 +74,17 @@ type closeWithoutContext interface { } type builder struct { - ticketingEnabled bool - m *metrics.M - nowFunc func() time.Time - clientSecret *jose.JSONWebKey - sessionStore sessions.SessionStore + ticketingEnabled bool + m *metrics.M + nowFunc func() time.Time + clientSecret *jose.JSONWebKey + sessionStore sessions.SessionStore + // sourceCache is the single connector-facing lookup slot, guarded by + // sourceCacheMu: SetSourceCache (per-sync install/clear from the + // runner) can race list RPCs reading it via syncOpAttrs. Access + // through sourceCacheLookup / SetSourceCache only. + sourceCacheMu stdsync.RWMutex + sourceCache sourcecache.Lookup metadataProvider MetadataProvider validateProvider ValidateProvider ticketManager TicketManagerLimited @@ -242,6 +250,64 @@ func WithSessionStore(ss sessions.SessionStore) Opt { } } +// WithSourceCache supplies a direct connector-facing source-cache Lookup +// exposed on SyncOpAttrs.SourceCache (see pkg/sourcecache). +// +// A nil lookup deliberately leaves the builder unset: SyncOpAttrs still +// supplies NoopLookup by default, but an unset builder is how the +// single-shot ask/answer continuation recognizes that it may install its +// per-request ContinuationLookup. Lambda construction passes nil here. +func WithSourceCache(lookup sourcecache.Lookup) Opt { + return func(b *builder) error { + if lookup == nil { + return nil + } + b.sourceCacheMu.Lock() + b.sourceCache = lookup + b.sourceCacheMu.Unlock() + return nil + } +} + +// SetSourceCache implements sourcecache.SetLookup so an in-process runner +// (the syncer, via its connector client) can install/clear the active +// lookup per sync. Subprocess mode instead routes lookups over +// BatonSourceCacheService and never calls this. +// +// The slot is guarded: the runner supports concurrent tasks, so a set or +// clear can race in-flight list RPCs reading the slot via syncOpAttrs. +// The slot is still SINGLE — there is no sync-id routing — so two live +// syncs installing lookups cross-wire each other (one sync's connector +// calls resolve against the other's previous store) and the first to +// finish blinds the survivor when it clears. That topology isn't +// supported; warn loudly when an install lands on top of an existing +// non-noop lookup so it is visible instead of silently wrong. +func (b *builder) SetSourceCache(ctx context.Context, lookup sourcecache.Lookup) { + installing := lookup != nil + if lookup == nil { + lookup = sourcecache.NoopLookup{} + } + b.sourceCacheMu.Lock() + prev := b.sourceCache + b.sourceCache = lookup + b.sourceCacheMu.Unlock() + if !installing || prev == nil { + return + } + if _, prevIsNoop := prev.(sourcecache.NoopLookup); !prevIsNoop { + ctxzap.Extract(ctx).Warn("source cache: installing a lookup over an existing one; " + + "concurrent syncs against one connector share a single lookup slot and will cross-wire or blind each other") + } +} + +// sourceCacheLookup returns the active slot value (nil when never set — +// the continuation's signal that it may install a per-request lookup). +func (b *builder) sourceCacheLookup() sourcecache.Lookup { + b.sourceCacheMu.RLock() + defer b.sourceCacheMu.RUnlock() + return b.sourceCache +} + func (b *builder) options(opts ...Opt) error { for _, opt := range opts { if err := opt(b); err != nil { diff --git a/pkg/connectorbuilder/continuation_test.go b/pkg/connectorbuilder/continuation_test.go new file mode 100644 index 000000000..ad871cc2f --- /dev/null +++ b/pkg/connectorbuilder/continuation_test.go @@ -0,0 +1,222 @@ +package connectorbuilder + +// Lookup continuation, builder side: a connector without a runner-supplied +// lookup gets a per-request ContinuationLookup when the request carries +// SourceCacheLookupOffer/Answers. Phase 1 (deferred lookups) must become an +// ask response with no failure recorded; phase 2 must serve answers to the +// unchanged connector code; a swallowed ErrLookupDeferred must fail loudly. + +import ( + "context" + "fmt" + "testing" + + "github.com/stretchr/testify/require" + + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + "github.com/conductorone/baton-sdk/pkg/annotations" + "github.com/conductorone/baton-sdk/pkg/sourcecache" + "github.com/conductorone/baton-sdk/pkg/types/resource" +) + +// continuationTestSyncer's Grants consults the source-cache lookup for one +// scope per page and serves either a replay annotation (hit) or a cold row +// + scope annotation (miss). Mode switches test the misuse paths. +type continuationTestSyncer struct { + testResourceSyncerV2Simple + + scope string + swallow bool // ignore ErrLookupDeferred and return rows anyway + batch bool // resolve via LookupMany over three scopes + + lookupCalls int +} + +func (s *continuationTestSyncer) Grants(ctx context.Context, r *v2.Resource, opts resource.SyncOpAttrs) ([]*v2.Grant, *resource.SyncOpResults, error) { + s.lookupCalls++ + if s.batch { + queries := []sourcecache.Query{ + {RowKind: sourcecache.RowKindGrants, ScopeHash: s.scope + "/1"}, + {RowKind: sourcecache.RowKindGrants, ScopeHash: s.scope + "/2"}, + {RowKind: sourcecache.RowKindGrants, ScopeHash: s.scope + "/3"}, + } + answers, err := sourcecache.LookupMany(ctx, opts.SourceCache, queries) + if err != nil { + return nil, nil, fmt.Errorf("batch revalidation: %w", err) + } + ret := &resource.SyncOpResults{Annotations: annotations.Annotations{}} + for _, a := range answers { + if !a.Found || a.ETag != "current" { + return nil, nil, fmt.Errorf("test wants all hits, got %+v", a) + } + } + ret.Annotations.Update(v2.SourceCacheReplay_builder{ScopeHash: s.scope + "/1", Etag: "current"}.Build()) + return nil, ret, nil + } + + entry, found, err := opts.SourceCache.LookupPreviousSourceCache(ctx, sourcecache.RowKindGrants, s.scope) + if err != nil && !s.swallow { + // Idiomatic wrapping — the builder must match via errors.Is. + return nil, nil, fmt.Errorf("listing grants for %s: %w", r.GetId().GetResource(), err) + } + ret := &resource.SyncOpResults{Annotations: annotations.Annotations{}} + if err == nil && found && entry.ETag == "current" { + ret.Annotations.Update(v2.SourceCacheReplay_builder{ScopeHash: s.scope, Etag: entry.ETag}.Build()) + return nil, ret, nil + } + // Miss (or swallowed deferral): cold row. + ret.Annotations.Update(v2.SourceCacheScope_builder{ScopeHash: s.scope, Etag: "current"}.Build()) + return []*v2.Grant{mkTestGrant(r)}, ret, nil +} + +func mkTestGrant(r *v2.Resource) *v2.Grant { + return v2.Grant_builder{ + Id: "grant-1", + Entitlement: v2.Entitlement_builder{ + Id: fmt.Sprintf("%s:%s:member", r.GetId().GetResourceType(), r.GetId().GetResource()), + Resource: r, + }.Build(), + Principal: r, + }.Build() +} + +func newContinuationTestBuilder(t *testing.T, ts *continuationTestSyncer) *builder { + t.Helper() + cb := &testConnectorBuilderV2Full{resourceSyncers: []ResourceSyncerV2{ts}, hasActionManager: true} + c, err := NewConnector(context.Background(), cb) + require.NoError(t, err) + b, ok := c.(*builder) + require.True(t, ok) + return b +} + +func continuationGrantsRequest(annos annotations.Annotations) *v2.GrantsServiceListGrantsRequest { + return v2.GrantsServiceListGrantsRequest_builder{ + Resource: v2.Resource_builder{ + Id: v2.ResourceId_builder{ResourceType: "group", Resource: "g1"}.Build(), + }.Build(), + Annotations: annos, + }.Build() +} + +func TestContinuation_DeferThenAnswer(t *testing.T) { + ctx := context.Background() + ts := &continuationTestSyncer{testResourceSyncerV2Simple: testResourceSyncerV2Simple{resourceType: "group"}, scope: "groups/g1/members"} + b := newContinuationTestBuilder(t, ts) + + // Phase 1: offer only → the handler's deferred lookup becomes an ask. + offer := annotations.New(&v2.SourceCacheLookupOffer{}) + resp, err := b.ListGrants(ctx, continuationGrantsRequest(offer)) + require.NoError(t, err, "a deferral is a protocol turn, not a failure") + require.Empty(t, resp.GetList(), "ask response must carry no rows") + require.Empty(t, resp.GetNextPageToken()) + + ask := &v2.SourceCacheLookupAsk{} + respAnnos := annotations.Annotations(resp.GetAnnotations()) + hasAsk, err := respAnnos.Pick(ask) + require.NoError(t, err) + require.True(t, hasAsk) + require.Len(t, ask.GetQueries(), 1) + require.Equal(t, "groups/g1/members", ask.GetQueries()[0].GetScopeHash()) + require.Equal(t, string(sourcecache.RowKindGrants), ask.GetQueries()[0].GetRowKind()) + + // Phase 2: same request + answers → the SAME connector code replays. + answers := annotations.New(&v2.SourceCacheLookupOffer{}) + answers.Update(sourcecache.AnswersProto([]sourcecache.Answer{ + {Query: sourcecache.Query{RowKind: sourcecache.RowKindGrants, ScopeHash: "groups/g1/members"}, Found: true, ETag: "current"}, + })) + resp2, err := b.ListGrants(ctx, continuationGrantsRequest(answers)) + require.NoError(t, err) + replay := &v2.SourceCacheReplay{} + resp2Annos := annotations.Annotations(resp2.GetAnnotations()) + hasReplay, err := resp2Annos.Pick(replay) + require.NoError(t, err) + require.True(t, hasReplay, "phase 2 must reach the connector's replay branch") + require.Empty(t, resp2.GetList()) + + // A not-found answer is a MISS, not a deferral: cold fetch. + miss := annotations.New(&v2.SourceCacheLookupOffer{}) + miss.Update(sourcecache.AnswersProto([]sourcecache.Answer{ + {Query: sourcecache.Query{RowKind: sourcecache.RowKindGrants, ScopeHash: "groups/g1/members"}, Found: false}, + })) + resp3, err := b.ListGrants(ctx, continuationGrantsRequest(miss)) + require.NoError(t, err) + require.Len(t, resp3.GetList(), 1, "not-found answer must fall through to cold fetch") +} + +func TestContinuation_LookupManyBatchesOneAsk(t *testing.T) { + ctx := context.Background() + ts := &continuationTestSyncer{testResourceSyncerV2Simple: testResourceSyncerV2Simple{resourceType: "group"}, scope: "chunk", batch: true} + b := newContinuationTestBuilder(t, ts) + + resp, err := b.ListGrants(ctx, continuationGrantsRequest(annotations.New(&v2.SourceCacheLookupOffer{}))) + require.NoError(t, err) + ask := &v2.SourceCacheLookupAsk{} + respAnnos := annotations.Annotations(resp.GetAnnotations()) + hasAsk, err := respAnnos.Pick(ask) + require.NoError(t, err) + require.True(t, hasAsk) + require.Len(t, ask.GetQueries(), 3, "LookupMany must collect the whole batch into ONE ask") + + var answers []sourcecache.Answer + for _, q := range ask.GetQueries() { + answers = append(answers, sourcecache.Answer{ + Query: sourcecache.Query{RowKind: sourcecache.RowKind(q.GetRowKind()), ScopeHash: q.GetScopeHash()}, + Found: true, ETag: "current", + }) + } + req := annotations.New(&v2.SourceCacheLookupOffer{}) + req.Update(sourcecache.AnswersProto(answers)) + resp2, err := b.ListGrants(ctx, continuationGrantsRequest(req)) + require.NoError(t, err) + resp2Annos := annotations.Annotations(resp2.GetAnnotations()) + require.True(t, resp2Annos.Contains(&v2.SourceCacheReplay{})) +} + +func TestContinuation_SwallowedDeferralFailsLoudly(t *testing.T) { + ctx := context.Background() + ts := &continuationTestSyncer{testResourceSyncerV2Simple: testResourceSyncerV2Simple{resourceType: "group"}, scope: "s", swallow: true} + b := newContinuationTestBuilder(t, ts) + + _, err := b.ListGrants(ctx, continuationGrantsRequest(annotations.New(&v2.SourceCacheLookupOffer{}))) + require.Error(t, err) + require.Contains(t, err.Error(), "swallowed", "a swallowed ErrLookupDeferred must be caught in-process, not at the production bounce cap") +} + +func TestContinuation_NoOfferMeansNoopLookup(t *testing.T) { + ctx := context.Background() + ts := &continuationTestSyncer{testResourceSyncerV2Simple: testResourceSyncerV2Simple{resourceType: "group"}, scope: "s"} + b := newContinuationTestBuilder(t, ts) + + // No offer, no answers: lookup misses (NoopLookup), connector fetches + // cold, no ask, no error — exactly today's behavior. + resp, err := b.ListGrants(ctx, continuationGrantsRequest(nil)) + require.NoError(t, err) + require.Len(t, resp.GetList(), 1) + respAnnos := annotations.Annotations(resp.GetAnnotations()) + require.False(t, respAnnos.Contains(&v2.SourceCacheLookupAsk{})) +} + +func TestContinuation_DirectLookupWins(t *testing.T) { + ctx := context.Background() + ts := &continuationTestSyncer{testResourceSyncerV2Simple: testResourceSyncerV2Simple{resourceType: "group"}, scope: "s"} + cb := &testConnectorBuilderV2Full{resourceSyncers: []ResourceSyncerV2{ts}, hasActionManager: true} + c, err := NewConnector(context.Background(), cb, WithSourceCache(staticTestLookup{etag: "current"})) + require.NoError(t, err) + b, ok := c.(*builder) + require.True(t, ok) + + // Offer present AND a direct lookup installed: the direct lookup wins, + // the handler resolves in one phase, and no ask is emitted. + resp, err := b.ListGrants(ctx, continuationGrantsRequest(annotations.New(&v2.SourceCacheLookupOffer{}))) + require.NoError(t, err) + respAnnos := annotations.Annotations(resp.GetAnnotations()) + require.False(t, respAnnos.Contains(&v2.SourceCacheLookupAsk{})) + require.True(t, respAnnos.Contains(&v2.SourceCacheReplay{})) +} + +type staticTestLookup struct{ etag string } + +func (s staticTestLookup) LookupPreviousSourceCache(context.Context, sourcecache.RowKind, string) (sourcecache.Entry, bool, error) { + return sourcecache.Entry{ETag: s.etag}, true, nil +} diff --git a/pkg/connectorbuilder/resource_syncer.go b/pkg/connectorbuilder/resource_syncer.go index 0360c3059..6e776850b 100644 --- a/pkg/connectorbuilder/resource_syncer.go +++ b/pkg/connectorbuilder/resource_syncer.go @@ -2,12 +2,14 @@ package connectorbuilder import ( "context" + "errors" "fmt" v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" "github.com/conductorone/baton-sdk/pkg/annotations" "github.com/conductorone/baton-sdk/pkg/pagination" "github.com/conductorone/baton-sdk/pkg/session" + "github.com/conductorone/baton-sdk/pkg/sourcecache" "github.com/conductorone/baton-sdk/pkg/types/resource" "github.com/conductorone/baton-sdk/pkg/types/tasks" "github.com/conductorone/baton-sdk/pkg/uotel" @@ -65,6 +67,114 @@ type StaticEntitlementSyncerV2 interface { StaticEntitlements(ctx context.Context, opts resource.SyncOpAttrs) ([]*v2.Entitlement, *resource.SyncOpResults, error) } +// TypeScopedGrantsSyncer is the grants-phase analogue of +// StaticEntitlementSyncerV2: the connector enumerates grants for a WHOLE +// resource type instead of being called once per resource. Implement it on +// a resource syncer whose ResourceType carries the v2.TypeScopedGrants +// annotation; the syncer then issues ListGrants calls with an empty +// resource id, which the builder routes here. +// +// The first call of a sync arrives with an empty page token (the planning +// call); the connector may answer with rows directly and/or spawn +// additional independent cursors by attaching a v2.SpawnCursors annotation +// whose page tokens are delivered back through opts.PageToken, one action +// each. Source-cache scope/replay/tombstone annotations work exactly as on +// per-resource grants pages. +type TypeScopedGrantsSyncer interface { + GrantsForResourceType(ctx context.Context, resourceTypeID string, opts resource.SyncOpAttrs) ([]*v2.Grant, *resource.SyncOpResults, error) +} + +// syncOpAttrs assembles the per-call SyncOpAttrs handed to V2 resource +// syncers. SourceCache is never nil: when the runner supplied no lookup +// (source cache disabled/degraded) connectors see NoopLookup and every +// lookup misses. Session is likewise never a nil-backed wrapper: without a +// configured store, connectors see the erroring no-op store instead of a +// panic on first use. +func (b *builder) syncOpAttrs(activeSyncID string, token pagination.Token) resource.SyncOpAttrs { + sc := b.sourceCacheLookup() + if sc == nil { + sc = sourcecache.NoopLookup{} + } + ss := b.sessionStore + if ss == nil { + ss = &session.NoOpSessionStore{} + } + return resource.SyncOpAttrs{ + SyncID: activeSyncID, + PageToken: token, + Session: WithSyncId(ss, activeSyncID), + SourceCache: sc, + } +} + +// continuationOpAttrs builds SyncOpAttrs for a list RPC, selecting the +// source-cache lookup by topology: +// +// - a runner-supplied lookup (in-process interface, subprocess loopback +// gRPC) always wins: those topologies answer lookups directly and +// never bounce; +// - otherwise, when the request carries SourceCacheLookupOffer (and/or +// SourceCacheLookupAnswers from a previous bounce), a per-request +// ContinuationLookup is installed — it serves answered scopes and +// defers the rest via ErrLookupDeferred; +// - otherwise NoopLookup (today's behavior). +// +// The returned ContinuationLookup is nil when the continuation is not in +// play for this request. +func (b *builder) continuationOpAttrs(activeSyncID string, token pagination.Token, reqAnnos annotations.Annotations) (resource.SyncOpAttrs, *sourcecache.ContinuationLookup, error) { + attrs := b.syncOpAttrs(activeSyncID, token) + if b.sourceCacheLookup() != nil { + return attrs, nil, nil + } + answersMsg := &v2.SourceCacheLookupAnswers{} + hasAnswers, err := reqAnnos.Pick(answersMsg) + if err != nil { + return attrs, nil, fmt.Errorf("error parsing source-cache lookup answers annotation: %w", err) + } + if !hasAnswers && !reqAnnos.Contains(&v2.SourceCacheLookupOffer{}) { + return attrs, nil, nil + } + var answers []sourcecache.Answer + if hasAnswers { + answers, err = sourcecache.AnswersFromProto(answersMsg) + if err != nil { + return attrs, nil, err + } + } + cl := sourcecache.NewContinuationLookup(answers) + attrs.SourceCache = cl + return attrs, cl, nil +} + +// continuationOutcome inspects a list handler's result when a +// ContinuationLookup was installed. +// +// Returns (askAnnos, deferred, misuseErr): +// - deferred=true: the handler deferred on recorded scopes. Respond with +// askAnnos (the SourceCacheLookupAsk), NO rows, NO next page token, +// and no failure metrics — this is a protocol turn, not a failure. +// - misuseErr != nil: the handler swallowed a deferred lookup (scopes +// were recorded but no ErrLookupDeferred propagated). Loud failure: +// silently continuing past a deferral re-asks forever and dies at the +// bounce cap in production, so it must fail here, visibly. +// - all-zero: the handler resolved normally; use its result as-is. +func continuationOutcome(cl *sourcecache.ContinuationLookup, handlerErr error) (annotations.Annotations, bool, error) { + if cl == nil { + return nil, false, nil + } + asked := cl.Asked() + if errors.Is(handlerErr, sourcecache.ErrLookupDeferred) && len(asked) > 0 { + annos := annotations.Annotations{} + annos.Update(sourcecache.AskProto(asked)) + return annos, true, nil + } + if handlerErr == nil && len(asked) > 0 { + return nil, false, status.Errorf(codes.Internal, + "connector swallowed a deferred source-cache lookup (%d scope(s) asked, no error propagated): ErrLookupDeferred must be returned — wrap with %%w, never swallow", len(asked)) + } + return nil, false, nil +} + // ResourceTargetedSyncer extends ResourceSyncer to add capabilities for directly syncing an individual resource // // Implementing this interface indicates the connector supports calling "get" on a resource @@ -131,12 +241,26 @@ func (b *builder) ListResources(ctx context.Context, request *v2.ResourcesServic Size: int(request.GetPageSize()), Token: request.GetPageToken(), } - opts := resource.SyncOpAttrs{SyncID: request.GetActiveSyncId(), PageToken: token, Session: WithSyncId(b.sessionStore, request.GetActiveSyncId())} + opts, contLookup, err := b.continuationOpAttrs(request.GetActiveSyncId(), token, annotations.Annotations(request.GetAnnotations())) + if err != nil { + b.m.RecordTaskFailure(ctx, tt, b.nowFunc().Sub(start), err) + return nil, err + } out, retOptions, err := rb.List(ctx, request.GetParentResourceId(), opts) if retOptions == nil { retOptions = &resource.SyncOpResults{} } + askAnnos, deferred, misuseErr := continuationOutcome(contLookup, err) + if misuseErr != nil { + b.m.RecordTaskFailure(ctx, tt, b.nowFunc().Sub(start), misuseErr) + return nil, misuseErr + } + if deferred { + b.m.RecordTaskSuccess(ctx, tt, b.nowFunc().Sub(start)) + return v2.ResourcesServiceListResourcesResponse_builder{Annotations: askAnnos}.Build(), nil + } + resp := v2.ResourcesServiceListResourcesResponse_builder{ List: out, NextPageToken: retOptions.NextPageToken, @@ -220,7 +344,7 @@ func (b *builder) ListStaticEntitlements(ctx context.Context, request *v2.Entitl Size: int(request.GetPageSize()), Token: request.GetPageToken(), } - opts := resource.SyncOpAttrs{SyncID: request.GetActiveSyncId(), PageToken: token, Session: WithSyncId(b.sessionStore, request.GetActiveSyncId())} + opts := b.syncOpAttrs(request.GetActiveSyncId(), token) out, retOptions, err := rbse.StaticEntitlements(ctx, opts) if retOptions == nil { retOptions = &resource.SyncOpResults{} @@ -269,12 +393,26 @@ func (b *builder) ListEntitlements(ctx context.Context, request *v2.Entitlements Size: int(request.GetPageSize()), Token: request.GetPageToken(), } - opts := resource.SyncOpAttrs{SyncID: request.GetActiveSyncId(), PageToken: token, Session: WithSyncId(b.sessionStore, request.GetActiveSyncId())} + opts, contLookup, err := b.continuationOpAttrs(request.GetActiveSyncId(), token, annotations.Annotations(request.GetAnnotations())) + if err != nil { + b.m.RecordTaskFailure(ctx, tt, b.nowFunc().Sub(start), err) + return nil, err + } out, retOptions, err := rb.Entitlements(ctx, request.GetResource(), opts) if retOptions == nil { retOptions = &resource.SyncOpResults{} } + askAnnos, deferred, misuseErr := continuationOutcome(contLookup, err) + if misuseErr != nil { + b.m.RecordTaskFailure(ctx, tt, b.nowFunc().Sub(start), misuseErr) + return nil, misuseErr + } + if deferred { + b.m.RecordTaskSuccess(ctx, tt, b.nowFunc().Sub(start)) + return v2.EntitlementsServiceListEntitlementsResponse_builder{Annotations: askAnnos}.Build(), nil + } + resp := v2.EntitlementsServiceListEntitlementsResponse_builder{ List: out, NextPageToken: retOptions.NextPageToken, @@ -327,12 +465,47 @@ func (b *builder) ListGrants(ctx context.Context, request *v2.GrantsServiceListG Size: int(request.GetPageSize()), Token: request.GetPageToken(), } - opts := resource.SyncOpAttrs{SyncID: request.GetActiveSyncId(), PageToken: token, Session: WithSyncId(b.sessionStore, request.GetActiveSyncId())} - out, retOptions, err := rb.Grants(ctx, request.GetResource(), opts) + reqAnnos := annotations.Annotations(request.GetAnnotations()) + opts, contLookup, err := b.continuationOpAttrs(request.GetActiveSyncId(), token, reqAnnos) + if err != nil { + b.m.RecordTaskFailure(ctx, tt, b.nowFunc().Sub(start), err) + return nil, err + } + + typeScoped := reqAnnos.Contains(&v2.TypeScopedGrants{}) + + var out []*v2.Grant + var retOptions *resource.SyncOpResults + if typeScoped { + // Type-scoped grants call (see v2.TypeScopedGrants): the request + // annotation is the routing marker (the resource is a + // self-referential {type, type} stub to satisfy wire validation). + // Only legal against a syncer that opted in. + tsgs, tsOk := rb.(TypeScopedGrantsSyncer) + if !tsOk { + err = status.Errorf(codes.InvalidArgument, + "error: type-scoped list grants for resource type %s, but its syncer does not implement TypeScopedGrantsSyncer", rid.GetResourceType()) + b.m.RecordTaskFailure(ctx, tt, b.nowFunc().Sub(start), err) + return nil, err + } + out, retOptions, err = tsgs.GrantsForResourceType(ctx, rid.GetResourceType(), opts) + } else { + out, retOptions, err = rb.Grants(ctx, request.GetResource(), opts) + } if retOptions == nil { retOptions = &resource.SyncOpResults{} } + askAnnos, deferred, misuseErr := continuationOutcome(contLookup, err) + if misuseErr != nil { + b.m.RecordTaskFailure(ctx, tt, b.nowFunc().Sub(start), misuseErr) + return nil, misuseErr + } + if deferred { + b.m.RecordTaskSuccess(ctx, tt, b.nowFunc().Sub(start)) + return v2.GrantsServiceListGrantsResponse_builder{Annotations: askAnnos}.Build(), nil + } + resp := v2.GrantsServiceListGrantsResponse_builder{ List: out, Annotations: appendSessionUsage(retOptions.Annotations, sessionUsage), diff --git a/pkg/connectorbuilder/source_cache_slot_test.go b/pkg/connectorbuilder/source_cache_slot_test.go new file mode 100644 index 000000000..1dbf78775 --- /dev/null +++ b/pkg/connectorbuilder/source_cache_slot_test.go @@ -0,0 +1,56 @@ +package connectorbuilder + +// The builder holds ONE mutable source-cache lookup slot shared by every +// sync against it (see SetSourceCache). The runner supports concurrent +// tasks, so installs/clears can race list RPCs reading the slot: the slot +// accesses must be synchronized, and an install over an existing lookup +// (two live syncs cross-wiring each other) must be loudly visible. + +import ( + "context" + "sync" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/conductorone/baton-sdk/pkg/sourcecache" +) + +// TestSetSourceCacheConcurrentWithListCalls hammers SetSourceCache +// (install/clear cycles, as concurrent syncs would) while list RPCs read +// the slot. Run under -race: unsynchronized slot access is a data race +// even when every interleaving happens to behave. +func TestSetSourceCacheConcurrentWithListCalls(t *testing.T) { + ctx := context.Background() + ts := &continuationTestSyncer{ + testResourceSyncerV2Simple: testResourceSyncerV2Simple{resourceType: "group"}, + scope: "groups/g1/members", + } + b := newContinuationTestBuilder(t, ts) + + stop := make(chan struct{}) + var wg sync.WaitGroup + wg.Add(1) + go func() { + defer wg.Done() + for i := 0; ; i++ { + select { + case <-stop: + return + default: + } + if i%2 == 0 { + b.SetSourceCache(ctx, sourcecache.NoopLookup{}) + } else { + b.SetSourceCache(ctx, nil) + } + } + }() + + for i := 0; i < 200; i++ { + _, err := b.ListGrants(ctx, continuationGrantsRequest(nil)) + require.NoError(t, err) + } + close(stop) + wg.Wait() +} diff --git a/pkg/connectorrunner/runner.go b/pkg/connectorrunner/runner.go index c404401b0..4db9cf18a 100644 --- a/pkg/connectorrunner/runner.go +++ b/pkg/connectorrunner/runner.go @@ -426,6 +426,7 @@ type runnerConfig struct { workerCount int targetedSyncResourceIDs []string externalResourceC1Z string + previousSyncC1Z string externalResourceEntitlementIdFilter string keepPreviousSyncC1ZCapable bool keepPreviousSyncC1ZEnabled bool @@ -760,6 +761,13 @@ func WithExternalResourceC1Z(externalResourceC1Z string) Option { } } +func WithPreviousSyncC1Z(previousSyncC1Z string) Option { + return func(ctx context.Context, cfg *runnerConfig) error { + cfg.previousSyncC1Z = previousSyncC1Z + return nil + } +} + func WithExternalResourceEntitlementFilter(entitlementId string) Option { return func(ctx context.Context, cfg *runnerConfig) error { cfg.externalResourceEntitlementIdFilter = entitlementId @@ -802,6 +810,25 @@ func WithKeepPreviousSyncC1ZRuntimeOptIn() Option { } } +// DeclaresPreviousSyncCapability reports whether opts include the connector +// author's replay declaration (WithKeepPreviousSyncC1Z). Used at +// command-definition time to decide whether the replay CLI flags +// (--previous-sync-c1z / --keep-previous-sync-c1z) appear in help: they are +// hidden for the overwhelming majority of connectors that don't support +// replay, and surfaced only for the ones whose author baked the capability +// into their RunConnector options. Options are applied to a scratch config; +// they are plain setters, so this is side-effect free. +func DeclaresPreviousSyncCapability(ctx context.Context, opts ...Option) bool { + cfg := &runnerConfig{} + for _, o := range opts { + if o == nil { + continue + } + _ = o(ctx, cfg) + } + return cfg.keepPreviousSyncC1ZCapable +} + func WithDiffSyncs(c1zPath string, baseSyncID string, newSyncID string) Option { return func(ctx context.Context, cfg *runnerConfig) error { cfg.onDemand = true @@ -973,7 +1000,15 @@ func NewConnectorRunner(ctx context.Context, c types.ConnectorServer, opts ...Op wrapperOpts = append(wrapperOpts, connector.WithTargetedSyncResources(cfg.targetedSyncResourceIDs)) } - if cfg.sessionStoreEnabled { + // The parent control-plane listener serves BOTH BatonSessionService and + // BatonSourceCacheService (internal/connector.runServer). Source-cache + // replay therefore needs the listener even when the connector never uses + // sessions: without it the subprocess connector's lookup degrades to + // NoopLookup, every scope misses, and every sync runs a full cold + // enumeration despite a valid --previous-sync-c1z. Start it whenever + // replay could be in play: the author declared the capability + // (WithKeepPreviousSyncC1Z) or a previous-sync c1z was configured. + if cfg.sessionStoreEnabled || cfg.keepPreviousSyncC1ZCapable || cfg.previousSyncC1Z != "" { wrapperOpts = append(wrapperOpts, connector.WithSessionStoreEnabled()) } @@ -1081,6 +1116,7 @@ func NewConnectorRunner(ctx context.Context, c types.ConnectorServer, opts ...Op tm, err = local.NewSyncer(ctx, cfg.c1zPath, local.WithTmpDir(cfg.tempDir), local.WithExternalResourceC1Z(cfg.externalResourceC1Z), + local.WithPreviousSyncC1Z(cfg.previousSyncC1Z), local.WithExternalResourceEntitlementIdFilter(cfg.externalResourceEntitlementIdFilter), local.WithTargetedSyncResources(resources), local.WithSkipEntitlementsAndGrants(cfg.skipEntitlementsAndGrants), diff --git a/pkg/dotc1z/c1zstore/grant_store.go b/pkg/dotc1z/c1zstore/grant_store.go index 55f972387..1e8dd0c03 100644 --- a/pkg/dotc1z/c1zstore/grant_store.go +++ b/pkg/dotc1z/c1zstore/grant_store.go @@ -145,4 +145,13 @@ type GrantAnnotation struct { PrincipalResourceTypeID string PrincipalResourceID string NeedsExpansion bool + + // SourceScopeHash is the grant row's source-cache scope stamp ("" for + // unstamped rows, and always "" on engines without source-cache + // support). The external-principal post-processing step writes its + // transformed grants under the SOURCE grant's scope so a future + // sync's replay of that scope carries them forward — the source + // grant itself is deleted by that step, and a scope left holding + // neither source nor transformed rows would silently replay empty. + SourceScopeHash string } diff --git a/pkg/dotc1z/c1zstore/sync_meta.go b/pkg/dotc1z/c1zstore/sync_meta.go index 90af84b4b..2904bbd16 100644 --- a/pkg/dotc1z/c1zstore/sync_meta.go +++ b/pkg/dotc1z/c1zstore/sync_meta.go @@ -57,5 +57,14 @@ type SyncRun struct { ParentSyncID string LinkedSyncID string SupportsDiff bool - Stats *reader_v2.SyncStats + // Compacted marks a sync produced by compaction rather than a real + // connector run. Compacted artifacts are keep-newer upsert merges, so + // no input's source-cache validators (etags) describe their contents: + // the syncer refuses to use a compacted sync as a replay source + // (degrades to a cold sync), and orchestrators choosing which + // artifact to materialize as the previous sync should read this flag. + // Always false on the SQLite engine (v1 files cannot be replay + // sources regardless). + Compacted bool + Stats *reader_v2.SyncStats } diff --git a/pkg/dotc1z/engine/pebble/adapter.go b/pkg/dotc1z/engine/pebble/adapter.go index 353eebe38..a5cfd3665 100644 --- a/pkg/dotc1z/engine/pebble/adapter.go +++ b/pkg/dotc1z/engine/pebble/adapter.go @@ -21,6 +21,7 @@ import ( v3 "github.com/conductorone/baton-sdk/pb/c1/storage/v3" "github.com/conductorone/baton-sdk/pkg/connectorstore" "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" + "github.com/conductorone/baton-sdk/pkg/sourcecache" ) // Adapter wraps an *Engine and implements connectorstore.Writer @@ -276,15 +277,12 @@ func (a *Adapter) CheckpointSync(ctx context.Context, syncToken string) error { if err != nil { return err } - updated := v3.SyncRunRecord_builder{ - SyncId: existing.GetSyncId(), - Type: existing.GetType(), - ParentSyncId: existing.GetParentSyncId(), - StartedAt: existing.GetStartedAt(), - EndedAt: existing.GetEndedAt(), - SyncToken: syncToken, - }.Build() - return a.engine.PutSyncRunRecord(ctx, updated) + // Mutate the loaded record rather than rebuilding it field-by-field: + // a rebuild silently drops every field it doesn't name (this used to + // erase compaction provenance when the compactor's grant-expansion + // pass checkpointed over a compacted-marked sync). + existing.SetSyncToken(syncToken) + return a.engine.PutSyncRunRecord(ctx, existing) } // EndSync stamps the open sync_run's ended_at and detaches it. After @@ -367,15 +365,12 @@ func (e *Engine) endSyncFinalize(ctx context.Context, existing *v3.SyncRunRecord return fmt.Errorf("EndSync: repair grant digests: %w", err) } } - updated := v3.SyncRunRecord_builder{ - SyncId: existing.GetSyncId(), - Type: existing.GetType(), - ParentSyncId: existing.GetParentSyncId(), - StartedAt: existing.GetStartedAt(), - EndedAt: timestamppb.Now(), - SyncToken: existing.GetSyncToken(), - }.Build() - if err := e.PutSyncRunRecord(ctx, updated); err != nil { + // Mutate the loaded record rather than rebuilding it field-by-field — + // a rebuild drops every field it doesn't name (supports_diff, + // linked_sync_id, compacted), which erased compaction provenance and + // diff markers when a sync re-ended after expansion or resume. + existing.SetEndedAt(timestamppb.Now()) + if err := e.PutSyncRunRecord(ctx, existing); err != nil { return err } // Populate the stats sidecar BEFORE the durability flush. Stats @@ -422,12 +417,27 @@ func (a *Adapter) PutGrants(ctx context.Context, grants ...*v2.Grant) error { return ErrNoCurrentSync } records := translateGrants(syncID, grants) + stampSourceScope(ctx, records, func(r *v3.GrantRecord, s string) { r.SetSourceScopeHash(s) }) if err := a.engine.PutGrantRecords(ctx, records...); err != nil { return fmt.Errorf("PutGrants: %w", err) } return nil } +// stampSourceScope stamps the source-cache scope hash carried by ctx +// (sourcecache.WithScope) onto freshly translated records. The syncer +// sets the scope around a page's store writes when the page carried a +// SourceCacheScope annotation; everything else writes unstamped rows. +func stampSourceScope[T any](ctx context.Context, records []T, set func(T, string)) { + scope := sourcecache.ScopeFromContext(ctx) + if scope == "" { + return + } + for _, r := range records { + set(r, scope) + } +} + // UnsafePutUniqueGrants writes grants on the trusted-import path: records // are encoded in parallel and written unconditionally, with no read-before-write // and no dedup pass. Do not use it for live connector output. The destination @@ -585,6 +595,7 @@ func (a *Adapter) PutResources(ctx context.Context, resources ...*v2.Resource) e } records = append(records, rec) } + stampSourceScope(ctx, records, func(r *v3.ResourceRecord, s string) { r.SetSourceScopeHash(s) }) if err := a.engine.PutResourceRecords(ctx, records...); err != nil { return fmt.Errorf("PutResources: %w", err) } @@ -612,6 +623,7 @@ func (a *Adapter) PutEntitlements(ctx context.Context, entitlements ...*v2.Entit } records = append(records, rec) } + stampSourceScope(ctx, records, func(r *v3.EntitlementRecord, s string) { r.SetSourceScopeHash(s) }) if err := a.engine.PutEntitlementRecords(ctx, records...); err != nil { return fmt.Errorf("PutEntitlements: %w", err) } diff --git a/pkg/dotc1z/engine/pebble/adapter_grants_store.go b/pkg/dotc1z/engine/pebble/adapter_grants_store.go index 30736c59e..6cff1955b 100644 --- a/pkg/dotc1z/engine/pebble/adapter_grants_store.go +++ b/pkg/dotc1z/engine/pebble/adapter_grants_store.go @@ -220,6 +220,10 @@ func (g pebbleGrantStore) translateExpanded(syncID string, grants []*v2.Grant) [ // because the caller left a residual GrantExpandable annotation. newRec.SetExpansion(nil) newRec.SetNeedsExpansion(false) + // Same shape for the source-cache scope stamp: existing records get + // their prior stamp restored in PutExpandedGrantRecords; brand-new + // expander-derived rows are never part of a source scope. + newRec.SetSourceScopeHash("") merged = append(merged, newRec) } return merged @@ -327,6 +331,7 @@ func (g pebbleGrantStore) ListWithAnnotationsPage(ctx context.Context, pageToken PrincipalResourceTypeID: princ.GetResourceTypeId(), PrincipalResourceID: princ.GetResourceId(), NeedsExpansion: rec.GetNeedsExpansion(), + SourceScopeHash: rec.GetSourceScopeHash(), }) } return rows, next, nil @@ -372,6 +377,7 @@ func (g pebbleGrantStore) ListWithAnnotationsForResourcePage( PrincipalResourceTypeID: rec.GetPrincipal().GetResourceTypeId(), PrincipalResourceID: rec.GetPrincipal().GetResourceId(), NeedsExpansion: rec.GetNeedsExpansion(), + SourceScopeHash: rec.GetSourceScopeHash(), }) } return rows, next, nil diff --git a/pkg/dotc1z/engine/pebble/adapter_sync_meta.go b/pkg/dotc1z/engine/pebble/adapter_sync_meta.go index 5dc736644..ddaec85a4 100644 --- a/pkg/dotc1z/engine/pebble/adapter_sync_meta.go +++ b/pkg/dotc1z/engine/pebble/adapter_sync_meta.go @@ -115,6 +115,7 @@ func syncRunRecordToExported(r *v3.SyncRunRecord) *c1zstore.SyncRun { ParentSyncID: r.GetParentSyncId(), LinkedSyncID: r.GetLinkedSyncId(), SupportsDiff: r.GetSupportsDiff(), + Compacted: r.GetCompacted(), } if t := r.GetStartedAt(); t != nil { tt := t.AsTime() @@ -149,6 +150,7 @@ func (a *Adapter) sortedSyncRuns(ctx context.Context) ([]c1zstore.SyncRun, error ParentSyncID: r.GetParentSyncId(), SupportsDiff: r.GetSupportsDiff(), LinkedSyncID: r.GetLinkedSyncId(), + Compacted: r.GetCompacted(), } if t := r.GetStartedAt(); t != nil { tt := t.AsTime() diff --git a/pkg/dotc1z/engine/pebble/bulk_import.go b/pkg/dotc1z/engine/pebble/bulk_import.go index ce280cebb..570135f07 100644 --- a/pkg/dotc1z/engine/pebble/bulk_import.go +++ b/pkg/dotc1z/engine/pebble/bulk_import.go @@ -57,6 +57,7 @@ const bulkSpillBufferSize = 1 << 20 var grantIndexFamilies = []byte{ idxGrantByPrincipal, idxGrantByNeedsExpansion, + idxGrantBySourceScope, } // bulkSSTWriter builds one SST file for a single disjoint key bucket. @@ -695,6 +696,7 @@ func (b *BulkSyncImport) Finish(ctx context.Context) error { // paths so overwrites of imported identities clean up index entries. _ = b.e.takeFreshGrantsEmpty() _ = b.e.takeFreshResourcesEmpty() + _ = b.e.takeFreshEntitlementsEmpty() return nil }) if err != nil { diff --git a/pkg/dotc1z/engine/pebble/cleanup.go b/pkg/dotc1z/engine/pebble/cleanup.go index 66f30a06a..b07c98ba3 100644 --- a/pkg/dotc1z/engine/pebble/cleanup.go +++ b/pkg/dotc1z/engine/pebble/cleanup.go @@ -37,6 +37,10 @@ func scopedRanges() [][2][]byte { {GrantByNeedsExpansionLowerBound(), GrantByNeedsExpansionUpperBound()}, {GrantByEntPrincHashLowerBound(), GrantByEntPrincHashUpperBound()}, {DigestLowerBound(), DigestUpperBound()}, + {GrantBySourceScopeLowerBound(), GrantBySourceScopeUpperBound()}, + {EntitlementBySourceScopeLowerBound(), EntitlementBySourceScopeUpperBound()}, + {ResourceBySourceScopeLowerBound(), ResourceBySourceScopeUpperBound()}, + {SourceCacheEntryLowerBound(), SourceCacheEntryUpperBound()}, {encodeAssetPrefix(), upperBoundOf(encodeAssetPrefix())}, // Stats sidecar — single key; the half-open range shape // contains exactly that one key. diff --git a/pkg/dotc1z/engine/pebble/engine.go b/pkg/dotc1z/engine/pebble/engine.go index 166104e0e..e35db586e 100644 --- a/pkg/dotc1z/engine/pebble/engine.go +++ b/pkg/dotc1z/engine/pebble/engine.go @@ -46,7 +46,7 @@ type Engine struct { // can skip the read-before-write index-cleanup path because this // sync_id is guaranteed to be empty. freshSync bool - // freshGrantsEmpty / freshResourcesEmpty + // freshGrantsEmpty / freshResourcesEmpty / freshEntitlementsEmpty // are one-shot bits guarded by currentSyncMu. MarkFreshSync sets // each to true; the first PutXxxRecords call of the fresh sync // reads the value via takeFreshXxxEmpty() which returns it and @@ -54,8 +54,9 @@ type Engine struct { // call only" — subsequent calls in the same fresh sync must // still read-before-write to clean up cross-call duplicate index // entries. - freshGrantsEmpty bool - freshResourcesEmpty bool + freshGrantsEmpty bool + freshResourcesEmpty bool + freshEntitlementsEmpty bool // writeWG tracks in-flight writes. Incremented at the start of // every Writer method, decremented in defer. @@ -342,6 +343,7 @@ func (e *Engine) SetCurrentSync(syncID string) error { e.freshSync = false e.freshGrantsEmpty = false e.freshResourcesEmpty = false + e.freshEntitlementsEmpty = false e.currentSyncMu.Unlock() // Binding a sync means more writes are coming; leave the sealed state // and resume compactions so L0 keeps draining (see seal). @@ -420,6 +422,7 @@ func (e *Engine) MarkFreshSync(syncID string) error { e.freshSync = true e.freshGrantsEmpty = true e.freshResourcesEmpty = true + e.freshEntitlementsEmpty = true e.currentSyncMu.Unlock() // A fresh sync writes heavily; leave the sealed state and resume // compactions so L0 keeps draining (see seal). @@ -437,6 +440,7 @@ func (e *Engine) clearCurrentSync() { e.freshSync = false e.freshGrantsEmpty = false e.freshResourcesEmpty = false + e.freshEntitlementsEmpty = false e.currentSyncMu.Unlock() } @@ -481,6 +485,16 @@ func (e *Engine) takeFreshResourcesEmpty() bool { return true } +func (e *Engine) takeFreshEntitlementsEmpty() bool { + e.currentSyncMu.Lock() + defer e.currentSyncMu.Unlock() + if !e.freshEntitlementsEmpty { + return false + } + e.freshEntitlementsEmpty = false + return true +} + // EndFreshSync clears the fresh-sync flag and flushes the memtable // + fsyncs the WAL so the data written during the sync is on disk // before the caller returns. Called by Adapter.EndSync. diff --git a/pkg/dotc1z/engine/pebble/entitlements.go b/pkg/dotc1z/engine/pebble/entitlements.go index 2182c3a2d..3cc37a557 100644 --- a/pkg/dotc1z/engine/pebble/entitlements.go +++ b/pkg/dotc1z/engine/pebble/entitlements.go @@ -20,9 +20,16 @@ func (e *Engine) PutEntitlementRecord(ctx context.Context, r *v3.EntitlementReco // PutEntitlementRecords writes N entitlements by structured primary key. // The identity key is a pure function of the record (it contains the raw -// external id), so overwrites are idempotent and no read-before-write or -// index cleanup is needed; a within-call dedup pre-pass keeps last-wins -// semantics for same-identity duplicates in one batch. +// external id), so overwrites of the PRIMARY row are idempotent; a +// within-call dedup pre-pass keeps last-wins semantics for same-identity +// duplicates in one batch. The by_source_scope index (the only +// entitlement secondary index) is NOT idempotent across scope changes, so +// a read-before-write Get cleans a differing old stamp — mirroring +// PutGrantRecords/PutResourceRecords — gated off only on the provably +// empty first call of a fresh sync. Without this, a same-identity rewrite +// under a new scope leaves a stale index entry AND the old scope's +// manifest entry survives, so the NEXT sync's replay of the old scope +// copies zero rows where the manifest promised content. func (e *Engine) PutEntitlementRecords(ctx context.Context, records ...*v3.EntitlementRecord) error { if len(records) == 0 { return nil @@ -35,6 +42,7 @@ func (e *Engine) PutEntitlementRecords(ctx context.Context, records ...*v3.Entit defer priBatch.Close() fresh := e.IsFreshSync() + skipGet := e.takeFreshEntitlementsEmpty() type dedupKey struct { id entitlementIdentity @@ -72,9 +80,35 @@ func (e *Engine) PutEntitlementRecords(ctx context.Context, records ...*v3.Entit if err != nil { return err } + newScope := r.GetSourceScopeHash() + if !skipGet { + oldVal, closer, getErr := e.db.Get(key) + switch { + case getErr == nil: + oldScope, scanErr := scanEntitlementSourceScopeRaw(oldVal) + closer.Close() + if scanErr != nil { + return scanErr + } + if oldScope != "" && oldScope != newScope { + if err := priBatch.Delete(encodeEntitlementBySourceScopeIndexKey(oldScope, id), nil); err != nil { + return err + } + } + case errors.Is(getErr, pebble.ErrNotFound): + // no prior record — write unconditionally + default: + return fmt.Errorf("PutEntitlementRecords: get old: %w", getErr) + } + } if err := priBatch.Set(key, val, nil); err != nil { return err } + if newScope != "" { + if err := priBatch.Set(encodeEntitlementBySourceScopeIndexKey(newScope, id), nil, nil); err != nil { + return err + } + } } opts := writeOpts(e.opts.durability) if fresh { @@ -122,6 +156,22 @@ func (e *Engine) DeleteEntitlementRecord(ctx context.Context, externalID string) key := encodeEntitlementIdentityKey(id) batch := e.db.NewBatch() defer batch.Close() + // Clean up the source-scope index entry (the only entitlement + // secondary index) so a replayed scope can't resurrect the row. + if oldVal, closer, getErr := e.db.Get(key); getErr == nil { + oldScope, scanErr := scanEntitlementSourceScopeRaw(oldVal) + closer.Close() + if scanErr != nil { + return scanErr + } + if oldScope != "" { + if err := batch.Delete(encodeEntitlementBySourceScopeIndexKey(oldScope, id), nil); err != nil { + return err + } + } + } else if !errors.Is(getErr, pebble.ErrNotFound) { + return getErr + } if err := batch.Delete(key, nil); err != nil { return err } diff --git a/pkg/dotc1z/engine/pebble/grants.go b/pkg/dotc1z/engine/pebble/grants.go index 1abf1d3d8..f14c1f7c1 100644 --- a/pkg/dotc1z/engine/pebble/grants.go +++ b/pkg/dotc1z/engine/pebble/grants.go @@ -263,6 +263,12 @@ func (e *Engine) PutExpandedGrantRecords(ctx context.Context, records []*v3.Gran r.SetExpansion(prior.GetExpansion()) r.SetNeedsExpansion(prior.GetNeedsExpansion()) r.SetDiscoveredAt(prior.GetDiscoveredAt()) + // Preserve the source-cache scope stamp exactly like the + // expansion side-state: the expander rewrites existing + // direct grants to bake in Sources, and clobbering the + // stamp here would silently drop every expander-touched + // grant from the next sync's replay of its scope. + r.SetSourceScopeHash(prior.GetSourceScopeHash()) var err error idxScratch, err = e.deleteGrantIndexesScratch(idxBatch, ext, oldVal, idxScratch) if err != nil { @@ -746,14 +752,17 @@ func (e *Engine) putSynthesizedGrantContributionsBatch(ctx context.Context, reco if err != nil { return err } - // sources (field 9) is the record's highest field, so appending - // it after the base marshal matches the deterministic byte order. + // sources (field 9) is the highest field this record carries, so + // appending it after the base marshal matches the deterministic + // byte order. (source_scope_hash is field 10, but synthesized + // grants never carry a scope stamp — fillSynthGrantRecord leaves + // it unset — so field 9 stays last on the wire.) val, srcScratch = appendGrantSourcesWire(val, srcScratch, rec.sources) valScratch = val if err := priBatch.Set(keyScratch, val, nil); err != nil { return err } - idxScratch, err = e.writeGrantIndexesForIdentityScratch(idxBatch, rec.id, false, idxScratch) + idxScratch, err = e.writeGrantIndexesForIdentityScratch(idxBatch, rec.id, false, "", idxScratch) if err != nil { return err } @@ -986,11 +995,14 @@ func grantIndexKeys(r *v3.GrantRecord) [][]byte { if err != nil { return nil } - keys := make([][]byte, 0, 2) + keys := make([][]byte, 0, 3) keys = append(keys, encodeGrantByPrincipalIdentityIndexKey(id)) if r.GetNeedsExpansion() { keys = append(keys, encodeGrantByNeedsExpansionIdentityIndexKey(id)) } + if sh := r.GetSourceScopeHash(); sh != "" { + keys = append(keys, encodeGrantBySourceScopeIndexKey(sh, id)) + } return keys } @@ -1038,7 +1050,7 @@ func (e *Engine) writeGrantIndexesScratch(batch *pebble.Batch, r *v3.GrantRecord if err != nil { return scratch, err } - return e.writeGrantIndexesForIdentityScratch(batch, id, r.GetNeedsExpansion(), scratch) + return e.writeGrantIndexesForIdentityScratch(batch, id, r.GetNeedsExpansion(), r.GetSourceScopeHash(), scratch) } // markDeferredIdxPending arms the deferred by_principal rebuild, durably. @@ -1081,7 +1093,10 @@ func (e *Engine) clearDeferredIdxPending() error { // for one grant. by_principal is never written inline: it is scattered // relative to the entitlement-first write order, so it is always rebuilt as // one sorted SST at EndSync (deferredIdxPending → BuildDeferredGrantIndexes). -func (e *Engine) writeGrantIndexesForIdentityScratch(batch *pebble.Batch, id grantIdentity, needsExpansion bool, scratch []byte) ([]byte, error) { +// by_needs_expansion and by_source_scope are written inline: both share the +// entitlement-first ordering of the primary keyspace, so their writes stay +// sorted. +func (e *Engine) writeGrantIndexesForIdentityScratch(batch *pebble.Batch, id grantIdentity, needsExpansion bool, sourceScopeHash string, scratch []byte) ([]byte, error) { if err := e.markDeferredIdxPending(); err != nil { return scratch, err } @@ -1096,6 +1111,12 @@ func (e *Engine) writeGrantIndexesForIdentityScratch(batch *pebble.Batch, id gra if err := e.stageGrantDigestInvalidation(batch, id.entitlement); err != nil { return scratch, err } + if sourceScopeHash != "" { + scratch = appendGrantBySourceScopeIndexKey(scratch[:0], sourceScopeHash, id) + if err := batch.Set(scratch, nil, nil); err != nil { + return scratch, err + } + } return scratch, nil } @@ -1108,7 +1129,7 @@ func (e *Engine) writeGrantIndexesForIdentityScratch(batch *pebble.Batch, id gra // which also clears any stale entries an overwrite would have left. Returns // the (possibly grown) scratch buffer. func (e *Engine) deleteGrantIndexesScratch(batch *pebble.Batch, externalID string, value, scratch []byte) ([]byte, error) { - entRT, entRID, entID, principalRT, principalID, _, err := scanGrantIndexFieldsRaw(value) + entRT, entRID, entID, principalRT, principalID, _, sourceScopeHash, err := scanGrantIndexFieldsRaw(value) if err != nil { return scratch, err } @@ -1129,6 +1150,12 @@ func (e *Engine) deleteGrantIndexesScratch(batch *pebble.Batch, externalID strin if err := e.stageGrantDigestInvalidation(batch, id.entitlement); err != nil { return scratch, err } + if sourceScopeHash != "" { + scratch = appendGrantBySourceScopeIndexKey(scratch[:0], sourceScopeHash, id) + if err := batch.Delete(scratch, nil); err != nil { + return scratch, err + } + } return scratch, nil } diff --git a/pkg/dotc1z/engine/pebble/grants_synth_encode.go b/pkg/dotc1z/engine/pebble/grants_synth_encode.go index b398712fd..a818dcf7e 100644 --- a/pkg/dotc1z/engine/pebble/grants_synth_encode.go +++ b/pkg/dotc1z/engine/pebble/grants_synth_encode.go @@ -30,11 +30,19 @@ var expandedGrantImmutableAnnotations = []*anypb.Any{expandedGrantImmutableAnnot // appendSynthGrantExternalIDWire) and sources (field 9, appended by // appendGrantSourcesWire). Expansion/NeedsExpansion stay zero for // synthesized grants. +// +// SourceScopeHash is cleared unconditionally, for two reasons: synthesized +// (expander-derived) grants are never part of a source-cache scope, and — +// load-bearing for appendGrantSourcesWire — an empty proto3 scalar emits no +// bytes, which keeps sources (field 9) the highest field on the wire so the +// hand encoder's append-after-base-marshal stays canonical. See +// TestGrantSourcesWireSchemaPin. func fillSynthGrantRecord(r *v3.GrantRecord, rec *synthesizedGrantRecord, now *timestamppb.Timestamp) { r.SetEntitlement(rec.entitlement) r.SetPrincipal(rec.principal) r.SetAnnotations(expandedGrantImmutableAnnotations) r.SetDiscoveredAt(now) + r.SetSourceScopeHash("") } // grantExternalIDFieldTag is the wire tag for GrantRecord.external_id (2). diff --git a/pkg/dotc1z/engine/pebble/grants_synth_encode_test.go b/pkg/dotc1z/engine/pebble/grants_synth_encode_test.go index a128d1746..bcb99caba 100644 --- a/pkg/dotc1z/engine/pebble/grants_synth_encode_test.go +++ b/pkg/dotc1z/engine/pebble/grants_synth_encode_test.go @@ -45,8 +45,19 @@ func TestGrantSourcesWireSchemaPin(t *testing.T) { maxNum = n } } - require.Equal(t, protoreflect.FieldNumber(9), maxNum, - "GrantRecord gained a field numbered above sources(9): appendGrantSourcesWire appends sources last and would emit non-canonical field order") + // source_scope_hash (10) sorts above sources (9), which would break the + // append-after-base-marshal canonical order IF it were ever populated on + // a synthesized record. It never is: fillSynthGrantRecord clears it + // unconditionally, and an empty proto3 scalar emits no bytes, so sources + // stays the highest field on the wire. Any field beyond 10 (or a + // non-scalar 10) requires revisiting appendGrantSourcesWire. + require.Equal(t, protoreflect.FieldNumber(10), maxNum, + "GrantRecord gained a field beyond source_scope_hash(10): decide whether appendGrantSourcesWire's append-last invariant still holds") + ssh := fields.ByNumber(10) + require.NotNil(t, ssh) + require.Equal(t, protoreflect.Name("source_scope_hash"), ssh.Name(), "field 10 is no longer source_scope_hash") + require.Equal(t, protoreflect.StringKind, ssh.Kind(), + "source_scope_hash is no longer a string scalar; the empty-value-emits-nothing guarantee appendGrantSourcesWire relies on may not hold") src := fields.ByName("sources") require.NotNil(t, src, "GrantRecord.sources missing") @@ -238,6 +249,12 @@ func TestFillSynthGrantRecordSchemaPin(t *testing.T) { "7:needs_expansion:bool", "8:annotations:message", "9:sources:message", + // source_scope_hash is part of the synthesized write path only as an + // unconditional CLEAR (expander-derived grants are never part of a + // source-cache scope); fillSynthGrantRecord sets it to "" to honor + // the reused-struct invariant and appendGrantSourcesWire's + // highest-populated-field assumption. + "10:source_scope_hash:string", }, protoFields, "GrantRecord shape changed: decide whether fillSynthGrantRecord must set the new/changed field") recType := reflect.TypeOf(synthesizedGrantRecord{}) diff --git a/pkg/dotc1z/engine/pebble/id_index_migration.go b/pkg/dotc1z/engine/pebble/id_index_migration.go index 019aa7150..c592f89fd 100644 --- a/pkg/dotc1z/engine/pebble/id_index_migration.go +++ b/pkg/dotc1z/engine/pebble/id_index_migration.go @@ -213,7 +213,7 @@ func (e *Engine) emitStructuredGrantMigration(ctx context.Context, primary *spil lastLog = now } } - entRT, entRID, entID, principalRT, principalID, _, err := scanGrantIndexFieldsRaw(iter.Value()) + entRT, entRID, entID, principalRT, principalID, _, _, err := scanGrantIndexFieldsRaw(iter.Value()) if err != nil { return rows, fmt.Errorf("id-index migration: scan grant: %w", err) } diff --git a/pkg/dotc1z/engine/pebble/if_newer.go b/pkg/dotc1z/engine/pebble/if_newer.go index 1b20dcc81..14951cd86 100644 --- a/pkg/dotc1z/engine/pebble/if_newer.go +++ b/pkg/dotc1z/engine/pebble/if_newer.go @@ -188,7 +188,16 @@ func (e *Engine) PutEntitlementRecordsIfNewer(ctx context.Context, records ...*v closer.Close() continue } + oldScope, scanErr := scanEntitlementSourceScopeRaw(oldVal) closer.Close() + if scanErr != nil { + return scanErr + } + if oldScope != "" && oldScope != r.GetSourceScopeHash() { + if err := batch.Delete(encodeEntitlementBySourceScopeIndexKey(oldScope, id), nil); err != nil { + return err + } + } case errors.Is(getErr, pebble.ErrNotFound): default: return fmt.Errorf("PutEntitlementRecordsIfNewer: get: %w", getErr) @@ -200,6 +209,11 @@ func (e *Engine) PutEntitlementRecordsIfNewer(ctx context.Context, records ...*v if err := batch.Set(key, val, nil); err != nil { return err } + if sh := r.GetSourceScopeHash(); sh != "" { + if err := batch.Set(encodeEntitlementBySourceScopeIndexKey(sh, id), nil, nil); err != nil { + return err + } + } written++ } if written == 0 { diff --git a/pkg/dotc1z/engine/pebble/inspect_diff_test.go b/pkg/dotc1z/engine/pebble/inspect_diff_test.go index c31aa4117..39484c8b3 100644 --- a/pkg/dotc1z/engine/pebble/inspect_diff_test.go +++ b/pkg/dotc1z/engine/pebble/inspect_diff_test.go @@ -543,7 +543,7 @@ func grantIdentityFromByEntitlementKey(key []byte, layout string) (string, bool) } func grantIdentityFromLegacyGrantValue(value []byte) (string, bool, error) { - entRT, entRID, entID, principalRT, principalID, _, err := scanGrantIndexFieldsRaw(value) + entRT, entRID, entID, principalRT, principalID, _, _, err := scanGrantIndexFieldsRaw(value) if err != nil { return "", false, err } diff --git a/pkg/dotc1z/engine/pebble/keys.go b/pkg/dotc1z/engine/pebble/keys.go index 440793d5e..c71800ef8 100644 --- a/pkg/dotc1z/engine/pebble/keys.go +++ b/pkg/dotc1z/engine/pebble/keys.go @@ -64,7 +64,16 @@ const ( typeCounter byte = 0x08 typeSession byte = 0x09 typeDigest byte = 0x0A - typeEngineMeta byte = 0xFF + // typeSourceCache holds one SourceCacheEntryRecord per + // (row_kind, scope_hash) — the source-cache replay manifest (see + // proto/c1/connector/v2/annotation_source_cache.proto). Placed + // after typeSession so the clone path's counter/session excise + // span ([typeCounter, upperBoundOf(typeSession))) leaves it in the + // clone: a cloned sync artifact stays usable as a replay source. + // ResetForNewSync's [typeResourceType, typeEngineMeta) span wipes + // it with the rest of the sync-scoped data. + typeSourceCache byte = 0x0B + typeEngineMeta byte = 0xFF ) // Index-discriminator bytes (second byte after typeIndex). One byte @@ -86,6 +95,13 @@ const ( // (typeDigest) folds over; built ONLY by the seal-time deferred // pass, never maintained inline. See digest.go and grant_digest.go. idxGrantByEntitlementPrincipalHash byte = 0x08 + // by_source_scope families: partial indexes over records whose + // source_scope_hash is non-empty (source-cache replay). Tails are + // identity tuples per the keys.go convention, so replay derives + // each primary key from the index key without touching the record. + idxGrantBySourceScope byte = 0x09 + idxEntitlementBySourceScope byte = 0x0A + idxResourceBySourceScope byte = 0x0B ) // --- Grant --- @@ -399,6 +415,120 @@ func DigestUpperBound() []byte { return upperBoundOf(DigestLowerBound()) } +// --- Source-cache (by_source_scope indexes + entry keyspace) --- + +// encodeGrantBySourceScopeIndexKey is the partial by_source_scope index +// over grants whose SourceScopeHash is non-empty: +// +// v3 | typeIndex | idxGrantBySourceScope | 0x00 | +// scope_hash | 0x00 | +// ent_rt | 0x00 | ent_rid | 0x00 | ent_flag | 0x00 | ent_tail | 0x00 | +// principal_rt | 0x00 | principal_id +// +// The tail after scope_hash is the grant identity tuple — byte-identical +// to the grant primary key's tail — so replay derives the primary key from +// the index key alone. Paired with encodeGrantBySourceScopePrefix +// (by-value prefix, with trailing sep). +func encodeGrantBySourceScopeIndexKey(scopeHash string, id grantIdentity) []byte { + return appendGrantBySourceScopeIndexKey(make([]byte, 0, 128), scopeHash, id) +} + +func appendGrantBySourceScopeIndexKey(dst []byte, scopeHash string, id grantIdentity) []byte { + dst = append(dst, versionV3, typeIndex, idxGrantBySourceScope) + dst = codec.AppendTupleSeparator(dst) + return codec.AppendTupleStrings( + dst, + scopeHash, + id.entitlement.resourceTypeID, + id.entitlement.resourceID, + id.entitlement.flagComponent(), + id.entitlement.tail, + id.principalTypeID, + id.principalID, + ) +} + +// encodeGrantBySourceScopePrefix is the by-value prefix for "all grants +// stamped with this scope hash". Trailing sep is load-bearing — see the +// keys.go convention doc. +func encodeGrantBySourceScopePrefix(scopeHash string) []byte { + buf := make([]byte, 0, 32+len(scopeHash)) + buf = append(buf, versionV3, typeIndex, idxGrantBySourceScope) + buf = codec.AppendTupleSeparator(buf) + buf = codec.AppendTupleStrings(buf, scopeHash) + return codec.AppendTupleSeparator(buf) +} + +// encodeEntitlementBySourceScopeIndexKey: +// +// v3 | typeIndex | idxEntitlementBySourceScope | 0x00 | +// scope_hash | 0x00 | rt | 0x00 | rid | 0x00 | flag | 0x00 | tail +// +// Tail is the entitlement identity tuple, byte-identical to the +// entitlement primary key's tail. Paired with +// encodeEntitlementBySourceScopePrefix. +func encodeEntitlementBySourceScopeIndexKey(scopeHash string, id entitlementIdentity) []byte { + return appendEntitlementBySourceScopeIndexKey(make([]byte, 0, 128), scopeHash, id) +} + +func appendEntitlementBySourceScopeIndexKey(dst []byte, scopeHash string, id entitlementIdentity) []byte { + dst = append(dst, versionV3, typeIndex, idxEntitlementBySourceScope) + dst = codec.AppendTupleSeparator(dst) + return codec.AppendTupleStrings(dst, scopeHash, id.resourceTypeID, id.resourceID, id.flagComponent(), id.tail) +} + +func encodeEntitlementBySourceScopePrefix(scopeHash string) []byte { + buf := make([]byte, 0, 32+len(scopeHash)) + buf = append(buf, versionV3, typeIndex, idxEntitlementBySourceScope) + buf = codec.AppendTupleSeparator(buf) + buf = codec.AppendTupleStrings(buf, scopeHash) + return codec.AppendTupleSeparator(buf) +} + +// encodeResourceBySourceScopeIndexKey: +// +// v3 | typeIndex | idxResourceBySourceScope | 0x00 | +// scope_hash | 0x00 | resource_type_id | 0x00 | resource_id +// +// Tail is the resource primary tuple. Paired with +// encodeResourceBySourceScopePrefix. +func encodeResourceBySourceScopeIndexKey(scopeHash, resourceTypeID, resourceID string) []byte { + return appendResourceBySourceScopeIndexKey(make([]byte, 0, 96), scopeHash, resourceTypeID, resourceID) +} + +func appendResourceBySourceScopeIndexKey(dst []byte, scopeHash, resourceTypeID, resourceID string) []byte { + dst = append(dst, versionV3, typeIndex, idxResourceBySourceScope) + dst = codec.AppendTupleSeparator(dst) + return codec.AppendTupleStrings(dst, scopeHash, resourceTypeID, resourceID) +} + +func encodeResourceBySourceScopePrefix(scopeHash string) []byte { + buf := make([]byte, 0, 32+len(scopeHash)) + buf = append(buf, versionV3, typeIndex, idxResourceBySourceScope) + buf = codec.AppendTupleSeparator(buf) + buf = codec.AppendTupleStrings(buf, scopeHash) + return codec.AppendTupleSeparator(buf) +} + +// encodeSourceCacheEntryKey is the primary key for one source-cache +// manifest entry: +// +// v3 | typeSourceCache | 0x00 | row_kind | 0x00 | scope_hash +// +// Paired with encodeSourceCachePrefix (by-type prefix). +func encodeSourceCacheEntryKey(rowKind, scopeHash string) []byte { + buf := make([]byte, 0, 32+len(rowKind)+len(scopeHash)) + buf = append(buf, versionV3, typeSourceCache) + buf = codec.AppendTupleSeparator(buf) + return codec.AppendTupleStrings(buf, rowKind, scopeHash) +} + +// encodeSourceCachePrefix is the by-type prefix for all source-cache +// entries. +func encodeSourceCachePrefix() []byte { + return []byte{versionV3, typeSourceCache} +} + // --- ResourceType --- // encodeResourceTypeKey returns the primary key for a resource_type: @@ -627,6 +757,28 @@ func GrantByEntitlementResourceUpperBound() []byte { return upperBoundOf(GrantByEntitlementResourceLowerBound()) } +func GrantBySourceScopeLowerBound() []byte { + return []byte{versionV3, typeIndex, idxGrantBySourceScope} +} +func GrantBySourceScopeUpperBound() []byte { return upperBoundOf(GrantBySourceScopeLowerBound()) } + +func EntitlementBySourceScopeLowerBound() []byte { + return []byte{versionV3, typeIndex, idxEntitlementBySourceScope} +} +func EntitlementBySourceScopeUpperBound() []byte { + return upperBoundOf(EntitlementBySourceScopeLowerBound()) +} + +func ResourceBySourceScopeLowerBound() []byte { + return []byte{versionV3, typeIndex, idxResourceBySourceScope} +} +func ResourceBySourceScopeUpperBound() []byte { + return upperBoundOf(ResourceBySourceScopeLowerBound()) +} + +func SourceCacheEntryLowerBound() []byte { return encodeSourceCachePrefix() } +func SourceCacheEntryUpperBound() []byte { return upperBoundOf(encodeSourceCachePrefix()) } + func AssetLowerBound() []byte { return encodeAssetPrefix() } func AssetUpperBound() []byte { return upperBoundOf(encodeAssetPrefix()) } diff --git a/pkg/dotc1z/engine/pebble/lookup.go b/pkg/dotc1z/engine/pebble/lookup.go index 2b70e59fe..282c3a9c2 100644 --- a/pkg/dotc1z/engine/pebble/lookup.go +++ b/pkg/dotc1z/engine/pebble/lookup.go @@ -265,6 +265,28 @@ func (e *Engine) grantPrimaryPrefixNonEmpty(prefix []byte) (bool, error) { // the row instead of the concat). Exactly one hit wins; zero is // pebble.ErrNotFound; several is ErrAmbiguousExternalID. func (e *Engine) resolveGrantIdentityByExternalID(ctx context.Context, grantID string) (grantIdentity, error) { + id, err := e.resolveGrantIdentityByCandidates(ctx, grantID) + if err == nil || !errors.Is(err, errNoGrantCandidateHits) { + return id, err + } + // Candidate probing proved nothing either way: the id may still be a + // connector-custom STORED external id (with or without colons), which + // only the O(all grants) scan can find. Interactive edges pay it; + // bounded callers (DeleteGrantRecordBounded) stop before this line. + return e.scanGrantIdentityByStoredExternalID(ctx, grantID) +} + +// errNoGrantCandidateHits reports that combinatorial candidate probing +// completed without a hit — distinct from pebble.ErrNotFound, which the +// full resolution reserves for "provably absent after the scan of last +// resort". Internal to the resolution layer. +var errNoGrantCandidateHits = errors.New("pebble: no grant candidate parse hit") + +// resolveGrantIdentityByCandidates is the bounded stage of grant-id +// resolution: combinatorial concat splits probed with point Gets, no +// keyspace scan. Returns errNoGrantCandidateHits when the shape yields no +// candidates (fewer than two colons) or every candidate missed. +func (e *Engine) resolveGrantIdentityByCandidates(ctx context.Context, grantID string) (grantIdentity, error) { var colons []int for i := 0; i < len(grantID); i++ { if grantID[i] == ':' { @@ -272,10 +294,8 @@ func (e *Engine) resolveGrantIdentityByExternalID(ctx context.Context, grantID s } } if len(colons) < 2 { - // No concat shape to split: connector-custom ids (SQLite keyed rows - // by these, and provisioner revokes address grants with them) are - // findable only by their STORED external id. - return e.scanGrantIdentityByStoredExternalID(ctx, grantID) + // No concat shape to split: findable only by STORED external id. + return grantIdentity{}, errNoGrantCandidateHits } if len(colons) > maxBareIDColons { return grantIdentity{}, fmt.Errorf("%w: grant id has %d colons; too complex to resolve safely by string", ErrAmbiguousExternalID, len(colons)) @@ -378,9 +398,7 @@ func (e *Engine) resolveGrantIdentityByExternalID(ctx context.Context, grantID s } switch len(hits) { case 0: - // Every concat split missed: the id may still be a connector-custom - // STORED external id that merely contains colons. - return e.scanGrantIdentityByStoredExternalID(ctx, grantID) + return grantIdentity{}, errNoGrantCandidateHits case 1: return hits[0], nil default: diff --git a/pkg/dotc1z/engine/pebble/raw_records.go b/pkg/dotc1z/engine/pebble/raw_records.go index b56c4c439..6b29395c9 100644 --- a/pkg/dotc1z/engine/pebble/raw_records.go +++ b/pkg/dotc1z/engine/pebble/raw_records.go @@ -3,11 +3,15 @@ package pebble import ( "fmt" "math" + "strings" "time" "github.com/cockroachdb/pebble/v2" "google.golang.org/protobuf/encoding/protowire" + "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/timestamppb" + + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" ) const ( @@ -110,18 +114,25 @@ func rawTimestampNanos(value []byte) (int64, error) { } func (e *Engine) deleteResourceIndexesRaw(batch *pebble.Batch, resourceTypeID string, resourceID string, value []byte) error { - parentRT, parentID, err := scanResourceParentRaw(value) + parentRT, parentID, sourceScopeHash, err := scanResourceIndexFieldsRaw(value) if err != nil { return err } - if parentID == "" { - return nil + if parentID != "" { + if err := batch.Delete(encodeResourceByParentIndexKey(parentRT, parentID, resourceTypeID, resourceID), nil); err != nil { + return err + } + } + if sourceScopeHash != "" { + if err := batch.Delete(encodeResourceBySourceScopeIndexKey(sourceScopeHash, resourceTypeID, resourceID), nil); err != nil { + return err + } } - return batch.Delete(encodeResourceByParentIndexKey(parentRT, parentID, resourceTypeID, resourceID), nil) + return nil } func (e *Engine) deleteGrantIndexesRaw(batch *pebble.Batch, externalID string, value []byte) error { - entRT, entRID, entID, principalRT, principalID, _, err := scanGrantIndexFieldsRaw(value) + entRT, entRID, entID, principalRT, principalID, _, sourceScopeHash, err := scanGrantIndexFieldsRaw(value) if err != nil { return err } @@ -142,6 +153,11 @@ func (e *Engine) deleteGrantIndexesRaw(batch *pebble.Batch, externalID string, v if err := e.stageGrantDigestInvalidation(batch, id.entitlement); err != nil { return err } + if sourceScopeHash != "" { + if err := batch.Delete(encodeGrantBySourceScopeIndexKey(sourceScopeHash, id), nil); err != nil { + return err + } + } return batch.Delete(encodeGrantByNeedsExpansionIdentityIndexKey(id), nil) } @@ -319,37 +335,83 @@ func scanEntitlementResourceTypeRaw(value []byte) ([]byte, error) { // occurrence of the target field, matching scanGrantIndexFieldsRaw and // approximating proto merge semantics. Values written by this SDK carry // at most one occurrence, so this only matters for foreign writers. -func scanResourceParentRaw(value []byte) (string, string, error) { - var rt, id string +// scanResourceIndexFieldsRaw extracts the parent ref (field 6) and +// source_scope_hash (field 9) from a marshaled ResourceRecord — the +// fields that key resource secondary indexes. +func scanResourceIndexFieldsRaw(value []byte) (string, string, string, error) { + var rt, id, sourceScopeHash string for len(value) > 0 { num, typ, n := protowire.ConsumeTag(value) if n < 0 { - return "", "", protowire.ParseError(n) + return "", "", "", protowire.ParseError(n) + } + value = value[n:] + switch num { + case 6: + if typ != protowire.BytesType { + return "", "", "", fmt.Errorf("raw record: resource parent has wire type %v", typ) + } + msg, n := protowire.ConsumeBytes(value) + if n < 0 { + return "", "", "", protowire.ParseError(n) + } + var err error + rt, id, err = scanResourceRefRaw(msg) + if err != nil { + return "", "", "", err + } + value = value[n:] + case 9: + if typ != protowire.BytesType { + return "", "", "", fmt.Errorf("raw record: resource source_scope_hash has wire type %v", typ) + } + s, n := protowire.ConsumeBytes(value) + if n < 0 { + return "", "", "", protowire.ParseError(n) + } + sourceScopeHash = string(s) + value = value[n:] + default: + n = protowire.ConsumeFieldValue(num, typ, value) + if n < 0 { + return "", "", "", protowire.ParseError(n) + } + value = value[n:] + } + } + return rt, id, sourceScopeHash, nil +} + +// scanEntitlementSourceScopeRaw extracts source_scope_hash (field 11) +// from a marshaled EntitlementRecord. Keys the entitlement +// by_source_scope index — the only entitlement secondary index. +func scanEntitlementSourceScopeRaw(value []byte) (string, error) { + var sourceScopeHash string + for len(value) > 0 { + num, typ, n := protowire.ConsumeTag(value) + if n < 0 { + return "", protowire.ParseError(n) } value = value[n:] - if num != 6 { + if num != 11 { n = protowire.ConsumeFieldValue(num, typ, value) if n < 0 { - return "", "", protowire.ParseError(n) + return "", protowire.ParseError(n) } value = value[n:] continue } if typ != protowire.BytesType { - return "", "", fmt.Errorf("raw record: resource parent has wire type %v", typ) + return "", fmt.Errorf("raw record: entitlement source_scope_hash has wire type %v", typ) } - msg, n := protowire.ConsumeBytes(value) + s, n := protowire.ConsumeBytes(value) if n < 0 { - return "", "", protowire.ParseError(n) - } - var err error - rt, id, err = scanResourceRefRaw(msg) - if err != nil { - return "", "", err + return "", protowire.ParseError(n) } + sourceScopeHash = string(s) value = value[n:] } - return rt, id, nil + return sourceScopeHash, nil } func scanEntitlementResourceRaw(value []byte) (string, string, error) { @@ -429,63 +491,73 @@ func scanEntitlementIdentityFieldsRaw(value []byte) (string, string, string, err return rt, id, externalID, nil } -func scanGrantIndexFieldsRaw(value []byte) (string, string, string, string, string, bool, error) { - var entRT, entRID, entID, principalRT, principalID string +func scanGrantIndexFieldsRaw(value []byte) (string, string, string, string, string, bool, string, error) { + var entRT, entRID, entID, principalRT, principalID, sourceScopeHash string var needsExpansion bool for len(value) > 0 { num, typ, n := protowire.ConsumeTag(value) if n < 0 { - return "", "", "", "", "", false, protowire.ParseError(n) + return "", "", "", "", "", false, "", protowire.ParseError(n) } value = value[n:] switch num { case 3: if typ != protowire.BytesType { - return "", "", "", "", "", false, fmt.Errorf("raw record: grant entitlement has wire type %v", typ) + return "", "", "", "", "", false, "", fmt.Errorf("raw record: grant entitlement has wire type %v", typ) } msg, n := protowire.ConsumeBytes(value) if n < 0 { - return "", "", "", "", "", false, protowire.ParseError(n) + return "", "", "", "", "", false, "", protowire.ParseError(n) } var err error entRT, entRID, entID, err = scanEntitlementRefRaw(msg) if err != nil { - return "", "", "", "", "", false, err + return "", "", "", "", "", false, "", err } value = value[n:] case 4: if typ != protowire.BytesType { - return "", "", "", "", "", false, fmt.Errorf("raw record: grant principal has wire type %v", typ) + return "", "", "", "", "", false, "", fmt.Errorf("raw record: grant principal has wire type %v", typ) } msg, n := protowire.ConsumeBytes(value) if n < 0 { - return "", "", "", "", "", false, protowire.ParseError(n) + return "", "", "", "", "", false, "", protowire.ParseError(n) } var err error principalRT, principalID, err = scanPrincipalRefRaw(msg) if err != nil { - return "", "", "", "", "", false, err + return "", "", "", "", "", false, "", err } value = value[n:] case 7: if typ != protowire.VarintType { - return "", "", "", "", "", false, fmt.Errorf("raw record: grant needs_expansion has wire type %v", typ) + return "", "", "", "", "", false, "", fmt.Errorf("raw record: grant needs_expansion has wire type %v", typ) } v, n := protowire.ConsumeVarint(value) if n < 0 { - return "", "", "", "", "", false, protowire.ParseError(n) + return "", "", "", "", "", false, "", protowire.ParseError(n) } needsExpansion = v != 0 value = value[n:] + case 10: + if typ != protowire.BytesType { + return "", "", "", "", "", false, "", fmt.Errorf("raw record: grant source_scope_hash has wire type %v", typ) + } + s, n := protowire.ConsumeBytes(value) + if n < 0 { + return "", "", "", "", "", false, "", protowire.ParseError(n) + } + sourceScopeHash = string(s) + value = value[n:] default: n = protowire.ConsumeFieldValue(num, typ, value) if n < 0 { - return "", "", "", "", "", false, protowire.ParseError(n) + return "", "", "", "", "", false, "", protowire.ParseError(n) } value = value[n:] } } - return entRT, entRID, entID, principalRT, principalID, needsExpansion, nil + return entRT, entRID, entID, principalRT, principalID, needsExpansion, sourceScopeHash, nil } // scanGrantNeedsExpansionRaw extracts only the needs_expansion flag @@ -592,3 +664,173 @@ func scanResourceRefRawBytes(value []byte, set func(protowire.Number, []byte)) e } return nil } + +// Annotation message names (the tail of an Any type_url) that drive +// replay side effects. Matched on the segment after the last '/' so +// non-default type-url prefixes still resolve. +const ( + anyTypeChildResourceType = "c1.connector.v2.ChildResourceType" + anyTypeInsertResourceGrants = "c1.connector.v2.InsertResourceGrants" + anyTypeExternalResourceMatch = "c1.connector.v2.ExternalResourceMatch" + anyTypeExternalResourceMatchAll = "c1.connector.v2.ExternalResourceMatchAll" + anyTypeExternalResourceMatchID = "c1.connector.v2.ExternalResourceMatchID" +) + +// scanAnnotationAnysRaw walks a record's repeated-Any annotations field +// (fieldNum) without unmarshaling the record, invoking visit with each +// Any's message type name and value bytes. Used by the source-cache +// replay paths to detect side-effect annotations (ChildResourceType, +// InsertResourceGrants, ExternalResourceMatch*) on raw-copied rows. +func scanAnnotationAnysRaw(value []byte, fieldNum protowire.Number, visit func(typeName string, payload []byte) error) error { + for len(value) > 0 { + num, typ, n := protowire.ConsumeTag(value) + if n < 0 { + return protowire.ParseError(n) + } + value = value[n:] + if num != fieldNum { + n = protowire.ConsumeFieldValue(num, typ, value) + if n < 0 { + return protowire.ParseError(n) + } + value = value[n:] + continue + } + if typ != protowire.BytesType { + return fmt.Errorf("raw record: annotations field has wire type %v", typ) + } + anyMsg, n := protowire.ConsumeBytes(value) + if n < 0 { + return protowire.ParseError(n) + } + value = value[n:] + + var typeURL string + var payload []byte + rest := anyMsg + for len(rest) > 0 { + anum, atyp, an := protowire.ConsumeTag(rest) + if an < 0 { + return protowire.ParseError(an) + } + rest = rest[an:] + if atyp != protowire.BytesType { + an = protowire.ConsumeFieldValue(anum, atyp, rest) + if an < 0 { + return protowire.ParseError(an) + } + rest = rest[an:] + continue + } + b, an := protowire.ConsumeBytes(rest) + if an < 0 { + return protowire.ParseError(an) + } + switch anum { + case 1: + typeURL = string(b) + case 2: + payload = b + default: + } + rest = rest[an:] + } + typeName := typeURL + if i := strings.LastIndexByte(typeURL, '/'); i >= 0 { + typeName = typeURL[i+1:] + } + if typeName == "" { + continue + } + if err := visit(typeName, payload); err != nil { + return err + } + } + return nil +} + +// scanResourceChildTypeIDsRaw returns the resource_type_ids named by +// ChildResourceType annotations on a marshaled ResourceRecord (field 7). +// Nil for the common case of a resource with no child types. +func scanResourceChildTypeIDsRaw(value []byte) ([]string, error) { + var out []string + err := scanAnnotationAnysRaw(value, 7, func(typeName string, payload []byte) error { + if typeName != anyTypeChildResourceType { + return nil + } + crt := &v2.ChildResourceType{} + if err := proto.Unmarshal(payload, crt); err != nil { + return fmt.Errorf("raw record: unmarshal ChildResourceType annotation: %w", err) + } + if id := crt.GetResourceTypeId(); id != "" { + out = append(out, id) + } + return nil + }) + if err != nil { + return nil, err + } + return out, nil +} + +const anyTypeEntitlementExclusionGroup = "c1.connector.v2.EntitlementExclusionGroup" + +// scanEntitlementExclusionGroupsRaw returns the exclusion-group +// annotations on a marshaled EntitlementRecord (annotations field 7), +// paired with the row's identity so the syncer can validate them exactly +// like fresh response rows. Nil for the common unannotated case, with no +// identity scan performed. +func scanEntitlementExclusionGroupsRaw(value []byte) ([]ReplayedExclusionGroup, error) { + var groups []*v2.EntitlementExclusionGroup + err := scanAnnotationAnysRaw(value, 7, func(typeName string, payload []byte) error { + if typeName != anyTypeEntitlementExclusionGroup { + return nil + } + eg := &v2.EntitlementExclusionGroup{} + if err := proto.Unmarshal(payload, eg); err != nil { + return fmt.Errorf("raw record: unmarshal EntitlementExclusionGroup annotation: %w", err) + } + groups = append(groups, eg) + return nil + }) + if err != nil { + return nil, err + } + if len(groups) == 0 { + return nil, nil + } + rt, _, externalID, err := scanEntitlementIdentityFieldsRaw(value) + if err != nil { + return nil, err + } + out := make([]ReplayedExclusionGroup, 0, len(groups)) + for _, eg := range groups { + out = append(out, ReplayedExclusionGroup{ + EntitlementID: externalID, + ResourceTypeID: rt, + Group: eg, + }) + } + return out, nil +} + +// grantSourceCacheFlags reports which replay side-effect annotations a +// marshaled GrantRecord carries (annotations field 8). +type grantSourceCacheFlags struct { + insertResourceGrants bool + externalResourceMatch bool +} + +func scanGrantSourceCacheFlagsRaw(value []byte) (grantSourceCacheFlags, error) { + var f grantSourceCacheFlags + err := scanAnnotationAnysRaw(value, 8, func(typeName string, _ []byte) error { + switch typeName { + case anyTypeInsertResourceGrants: + f.insertResourceGrants = true + case anyTypeExternalResourceMatch, anyTypeExternalResourceMatchAll, anyTypeExternalResourceMatchID: + f.externalResourceMatch = true + } + return nil + }) + return f, err +} diff --git a/pkg/dotc1z/engine/pebble/resources.go b/pkg/dotc1z/engine/pebble/resources.go index 564a0430a..0f0d820cb 100644 --- a/pkg/dotc1z/engine/pebble/resources.go +++ b/pkg/dotc1z/engine/pebble/resources.go @@ -151,15 +151,22 @@ func (e *Engine) DeleteResourceRecord(ctx context.Context, resourceTypeID, resou } func (e *Engine) writeResourceIndexes(batch *pebble.Batch, r *v3.ResourceRecord) error { - parent := r.GetParent() - if parent == nil || parent.GetResourceId() == "" { - return nil + if parent := r.GetParent(); parent != nil && parent.GetResourceId() != "" { + k := encodeResourceByParentIndexKey( + parent.GetResourceTypeId(), parent.GetResourceId(), + r.GetResourceTypeId(), r.GetResourceId(), + ) + if err := batch.Set(k, nil, nil); err != nil { + return err + } + } + if sh := r.GetSourceScopeHash(); sh != "" { + k := encodeResourceBySourceScopeIndexKey(sh, r.GetResourceTypeId(), r.GetResourceId()) + if err := batch.Set(k, nil, nil); err != nil { + return err + } } - k := encodeResourceByParentIndexKey( - parent.GetResourceTypeId(), parent.GetResourceId(), - r.GetResourceTypeId(), r.GetResourceId(), - ) - return batch.Set(k, nil, nil) + return nil } func (e *Engine) IterateResources(ctx context.Context, yield func(*v3.ResourceRecord) bool) error { diff --git a/pkg/dotc1z/engine/pebble/source_cache.go b/pkg/dotc1z/engine/pebble/source_cache.go new file mode 100644 index 000000000..733f326b9 --- /dev/null +++ b/pkg/dotc1z/engine/pebble/source_cache.go @@ -0,0 +1,1173 @@ +package pebble + +import ( + "context" + "errors" + "fmt" + + "github.com/cockroachdb/pebble/v2" + "google.golang.org/protobuf/encoding/protowire" + "google.golang.org/protobuf/types/known/timestamppb" + + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + v3 "github.com/conductorone/baton-sdk/pb/c1/storage/v3" + "github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/codec" +) + +// Source-cache replay, engine side. +// +// The typeSourceCache keyspace holds one SourceCacheEntryRecord per +// (row_kind, scope_hash): the opaque upstream validator (etag / delta +// token) the sync recorded for that scope. Rows produced under a scope +// are stamped with source_scope_hash and indexed under the +// by_source_scope families, whose tails are identity tuples — so a +// replay derives every primary key from the index key alone and copies +// raw values across files without a proto unmarshal. +// +// The previous sync lives in a separate read-only engine (a Pebble c1z +// holds exactly one sync); replay copies from prev into the receiver. + +// replayBatchRows bounds how many rows accumulate in one pebble.Batch +// before an intermediate commit. Replay of a delta-query collection can +// be the whole previous row set, so the batch must not grow unbounded. +const replayBatchRows = 10_000 + +// ReplayedParentResource identifies one replayed resource whose +// annotations carry ChildResourceType — the syncer must schedule child +// SyncResources actions for it, since a replayed page returns no rows +// and the response-row path that normally schedules children never +// sees these parents. +type ReplayedParentResource struct { + ResourceTypeID string + ResourceID string + ChildTypeIDs []string +} + +// ReplayedExclusionGroup is one exclusion-group annotation found on a +// replayed entitlement row, paired with the row's identity so the syncer +// can run the same invariant validation fresh response rows get. +type ReplayedExclusionGroup struct { + EntitlementID string + ResourceTypeID string + Group *v2.EntitlementExclusionGroup +} + +// SourceCacheReplayResult reports what one scope's replay copied. +type SourceCacheReplayResult struct { + Rows int64 + // NeedsExpansion is true when at least one copied grant row carried + // needs_expansion. The syncer must arm grant expansion in this case: + // replayed pages never pass GrantExpandable-annotated rows through + // the syncer's connector-response path, which is otherwise the only + // thing that enables the expansion phase. + NeedsExpansion bool + // StaleSkipped counts index entries under the scope that did NOT + // yield a copied row: the primary was missing, or its value stamp + // named a different scope. This is the discriminator between "scope + // legitimately empty" (index prefix empty, StaleSkipped == 0) and + // "scope's rows were clobbered without index cleanup" (index says + // rows existed, none survived the stamp check). The syncer fails a + // replay that copies zero rows while StaleSkipped > 0. + StaleSkipped int64 + // ChildResources lists replayed parents carrying ChildResourceType + // annotations (resources replay only). See ReplayedParentResource. + ChildResources []ReplayedParentResource + // HasExternalResourceGrants is true when at least one copied grant + // row carries an ExternalResourceMatch* annotation (grants replay + // only). The syncer must arm external-resource-grant processing — + // same rationale as NeedsExpansion: response rows are the only + // other place the flag is set, and a replayed page has none. + HasExternalResourceGrants bool + // RelatedResourceRows counts resource rows copied from prev because + // a replayed grant carried InsertResourceGrants (grants replay + // only). Those resources exist ONLY as grant-driven insertions — + // no ListResources response ever returns them — so the replay must + // recreate them or the current sync holds grants whose resources + // do not exist. + RelatedResourceRows int64 + // ExclusionGroups lists exclusion-group annotations found on + // replayed entitlement rows (entitlements replay only). The syncer + // must run the same invariant validation fresh rows get (one + // default per group, single resource type, size cap) — validation + // runs on response rows, which a replayed page has none of. + ExclusionGroups []ReplayedExclusionGroup +} + +// SourceCacheManifestSnapshot returns every manifest entry as +// "row_kind\x00scope_hash" -> etag. Inspection surface: the +// replay-equivalence harness compares a warm sync's manifest against a +// cold sync's (they must be identical — same scopes, same validators), +// catching phantom or missing entries that are invisible through the v2 +// read API but poison FUTURE syncs' replays. +func (e *Engine) SourceCacheManifestSnapshot(ctx context.Context) (map[string]string, error) { + iter, err := e.db.NewIter(&pebble.IterOptions{ + LowerBound: encodeSourceCachePrefix(), + UpperBound: upperBoundOf(encodeSourceCachePrefix()), + }) + if err != nil { + return nil, err + } + defer iter.Close() + out := map[string]string{} + for iter.First(); iter.Valid(); iter.Next() { + if err := ctx.Err(); err != nil { + return nil, err + } + rec := &v3.SourceCacheEntryRecord{} + if err := unmarshalRecord(iter.Value(), rec); err != nil { + return nil, fmt.Errorf("SourceCacheManifestSnapshot: unmarshal: %w", err) + } + out[rec.GetRowKind()+"\x00"+rec.GetScopeHash()] = rec.GetEtag() + } + return out, iter.Error() +} + +// SourceCacheScopeIndexSnapshot returns, per row kind, scope_hash -> the +// number of by_source_scope index entries. Same inspection rationale as +// SourceCacheManifestSnapshot: stamp/index divergence between a warm and +// a cold sync is invisible to v2 reads until a later sync replays from +// the damaged file. +func (e *Engine) SourceCacheScopeIndexSnapshot(ctx context.Context) (map[string]map[string]int, error) { + out := map[string]map[string]int{} + families := []struct { + kind string + lower []byte + }{ + {"resources", ResourceBySourceScopeLowerBound()}, + {"entitlements", EntitlementBySourceScopeLowerBound()}, + {"grants", GrantBySourceScopeLowerBound()}, + } + for _, fam := range families { + prefix := codec.AppendTupleSeparator(append([]byte{}, fam.lower...)) + iter, err := e.db.NewIter(&pebble.IterOptions{ + LowerBound: prefix, + UpperBound: upperBoundOf(fam.lower), + }) + if err != nil { + return nil, err + } + counts := map[string]int{} + for iter.First(); iter.Valid(); iter.Next() { + if err := ctx.Err(); err != nil { + iter.Close() + return nil, err + } + tail := iter.Key()[len(prefix):] + scope, _, ok := codec.DecodeTupleStringAlias(tail, 0) + if !ok { + iter.Close() + return nil, fmt.Errorf("SourceCacheScopeIndexSnapshot: malformed %s index key %x", fam.kind, iter.Key()) + } + counts[string(scope)]++ + } + if err := iter.Error(); err != nil { + iter.Close() + return nil, err + } + iter.Close() + out[fam.kind] = counts + } + return out, nil +} + +// ClearSourceCacheEntries removes every source-cache manifest entry. +// Compaction calls this on fold outputs: a keep-newer upsert merge is not +// a faithful snapshot of ANY input sync (base rows the newer sync deleted +// survive), so no input's validator may be allowed to 304-validate the +// merged row set. With the manifest empty, every lookup against the +// artifact misses and the next sync fetches fresh — cold-correct. The +// scope stamps and scope-index entries on rows are left in place: replay +// only reads them after a manifest hit, which can no longer happen. +func (e *Engine) ClearSourceCacheEntries(ctx context.Context) error { + return e.withWrite(func() error { + if err := e.requireCurrentSync(); err != nil { + return err + } + return e.db.DeleteRange(SourceCacheEntryLowerBound(), SourceCacheEntryUpperBound(), writeOpts(e.opts.durability)) + }) +} + +// PutSourceCacheEntry writes the manifest entry for (rowKind, scopeHash). +// Zero-row scopes still get entries — the validator must survive to the +// next sync even when the scope produced no rows. +func (e *Engine) PutSourceCacheEntry(ctx context.Context, rowKind, scopeHash, etag string) error { + return e.withWrite(func() error { + if err := e.requireCurrentSync(); err != nil { + return err + } + rec := &v3.SourceCacheEntryRecord{} + rec.SetRowKind(rowKind) + rec.SetScopeHash(scopeHash) + rec.SetEtag(etag) + rec.SetDiscoveredAt(timestamppb.Now()) + val, err := marshalRecord(rec) + if err != nil { + return err + } + opts := writeOpts(e.opts.durability) + if e.IsFreshSync() { + opts = pebble.NoSync + } + return e.db.Set(encodeSourceCacheEntryKey(rowKind, scopeHash), val, opts) + }) +} + +// GetSourceCacheEntry returns the manifest entry for (rowKind, scopeHash), +// or pebble.ErrNotFound. +func (e *Engine) GetSourceCacheEntry(ctx context.Context, rowKind, scopeHash string) (*v3.SourceCacheEntryRecord, error) { + val, closer, err := e.db.Get(encodeSourceCacheEntryKey(rowKind, scopeHash)) + if err != nil { + return nil, err + } + defer closer.Close() + rec := &v3.SourceCacheEntryRecord{} + if err := unmarshalRecord(val, rec); err != nil { + return nil, fmt.Errorf("GetSourceCacheEntry: unmarshal: %w", err) + } + return rec, nil +} + +// DeleteGrantRecordBounded deletes a grant by canonical public id WITHOUT +// the O(all grants) stored-external-id scan fallback that the interactive +// DeleteGrantRecord path is allowed to take. Used by the source-cache +// tombstone path, where a mass-removal round would otherwise pay a full +// keyspace scan PER already-absent id. +// +// Consequence, by design: a grant stored under a connector-CUSTOM id (one +// that isn't the SDK concat shape) is unreachable here and the delete +// no-ops. Connectors with custom grant ids must use principal-scoped +// tombstones (SourceCacheScope.deleted_principal_ids) instead — documented +// in the annotation proto. +func (e *Engine) DeleteGrantRecordBounded(ctx context.Context, externalID string) error { + return e.withWrite(func() error { + id, err := e.resolveGrantIdentityByCandidates(ctx, externalID) + if err != nil { + if errors.Is(err, errNoGrantCandidateHits) || errors.Is(err, pebble.ErrNotFound) { + return nil // absent (or custom-id) — tombstone no-op + } + return err + } + return e.deleteGrantByIdentityLocked(id) + }) +} + +// DeleteGrantsByPrincipalsInScope deletes every grant row in the CURRENT +// store stamped with scopeHash whose principal id is in principalIDs — +// the engine side of principal-scoped delta tombstones +// (SourceCacheScope.deleted_principal_ids). +// +// One prefix scan of the scope's by_source_scope index resolves +// everything: the index tail IS the grant identity, so the primary key +// and every secondary index key for a match are constructible from the +// index key alone — no value reads, no string resolution, no guessing. +// A principal with no rows in the scope is a no-op (providers tombstone +// objects the client never synced). Deleting a missing secondary index +// entry is a pebble no-op, which covers the mixed inline/deferred +// by_principal state mid-sync. +// +// Complexity: O(scope size) tuple-walks per call regardless of tombstone +// count — callers batch a page's tombstones into one call. +func (e *Engine) DeleteGrantsByPrincipalsInScope(ctx context.Context, scopeHash string, principalIDs map[string]struct{}) (int64, error) { + if len(principalIDs) == 0 { + return 0, nil + } + prefix := encodeGrantBySourceScopePrefix(scopeHash) + var deleted int64 + + err := e.withWrite(func() error { + if err := e.requireCurrentSync(); err != nil { + return err + } + iter, err := e.db.NewIter(&pebble.IterOptions{ + LowerBound: prefix, + UpperBound: upperBoundOf(prefix), + }) + if err != nil { + return err + } + defer iter.Close() + + opts := writeOpts(e.opts.durability) + if e.IsFreshSync() { + opts = pebble.NoSync + } + batch := e.db.NewBatch() + defer func() { _ = batch.Close() }() + + for iter.First(); iter.Valid(); iter.Next() { + if err := ctx.Err(); err != nil { + return err + } + key := iter.Key() + tail := key[len(prefix):] + // Tail layout: ent_rt | ent_rid | flag | ent_tail | prin_rt | prin_id + // (identical to the grant primary key tail; decoder shared with + // the primary-prefix scan paths in grants.go). + id, ok := decodeGrantIdentityTail(key, prefix) + if !ok { + continue // malformed index key — defensive skip + } + if _, hit := principalIDs[id.principalID]; !hit { + continue + } + // Primary key = grant header + the identity tail verbatim. + priKey := make([]byte, 0, 3+len(tail)) + priKey = append(priKey, versionV3, typeGrant) + priKey = codec.AppendTupleSeparator(priKey) + priKey = append(priKey, tail...) + if err := batch.Delete(priKey, nil); err != nil { + return err + } + if err := batch.Delete(encodeGrantByPrincipalIdentityIndexKey(id), nil); err != nil { + return err + } + if err := batch.Delete(encodeGrantByNeedsExpansionIdentityIndexKey(id), nil); err != nil { + return err + } + // The scope index entry itself (the key under the iterator — + // safe: the iterator reads a snapshot). + if err := batch.Delete(key, nil); err != nil { + return err + } + deleted++ + } + if err := iter.Error(); err != nil { + return err + } + if err := batch.Commit(opts); err != nil { + return err + } + return nil + }) + if err != nil { + return 0, err + } + return deleted, nil +} + +// DeleteGrantsByExternalIDsInScope deletes every grant row in the CURRENT +// store stamped with scopeHash whose STORED grant id (external id, which +// may be a connector-custom shape) is in ids. One scan of the scope's +// index, loading each candidate's primary row to compare the stored id — +// bounded by the scope's row count, never the whole keyspace. This is the +// tombstone path for connectors with custom grant ids whose scopes span +// multiple resources (so principal-scoped deletes would over-delete). +func (e *Engine) DeleteGrantsByExternalIDsInScope(ctx context.Context, scopeHash string, ids map[string]struct{}) (int64, error) { + if len(ids) == 0 { + return 0, nil + } + prefix := encodeGrantBySourceScopePrefix(scopeHash) + var deleted int64 + + err := e.withWrite(func() error { + if err := e.requireCurrentSync(); err != nil { + return err + } + iter, err := e.db.NewIter(&pebble.IterOptions{ + LowerBound: prefix, + UpperBound: upperBoundOf(prefix), + }) + if err != nil { + return err + } + defer iter.Close() + + opts := writeOpts(e.opts.durability) + if e.IsFreshSync() { + opts = pebble.NoSync + } + batch := e.db.NewBatch() + defer func() { _ = batch.Close() }() + + for iter.First(); iter.Valid(); iter.Next() { + if err := ctx.Err(); err != nil { + return err + } + key := iter.Key() + tail := key[len(prefix):] + id, ok := decodeGrantIdentityTail(key, prefix) + if !ok { + continue // malformed index key — defensive skip + } + // Primary key = grant header + the identity tail verbatim. + priKey := make([]byte, 0, 3+len(tail)) + priKey = append(priKey, versionV3, typeGrant) + priKey = codec.AppendTupleSeparator(priKey) + priKey = append(priKey, tail...) + + val, closer, err := e.db.Get(priKey) + if err != nil { + if errors.Is(err, pebble.ErrNotFound) { + continue // index ahead of primary — defensive skip + } + return err + } + rec := &v3.GrantRecord{} + uerr := unmarshalRecord(val, rec) + _ = closer.Close() + if uerr != nil { + return fmt.Errorf("DeleteGrantsByExternalIDsInScope: unmarshal: %w", uerr) + } + if _, hit := ids[rec.GetExternalId()]; !hit { + continue + } + if err := batch.Delete(priKey, nil); err != nil { + return err + } + if err := batch.Delete(encodeGrantByPrincipalIdentityIndexKey(id), nil); err != nil { + return err + } + if err := batch.Delete(encodeGrantByNeedsExpansionIdentityIndexKey(id), nil); err != nil { + return err + } + if err := batch.Delete(key, nil); err != nil { + return err + } + deleted++ + } + if err := iter.Error(); err != nil { + return err + } + if err := batch.Commit(opts); err != nil { + return err + } + return nil + }) + if err != nil { + return 0, err + } + return deleted, nil +} + +// DeleteResourcesByIDsInScope deletes every resource row in the CURRENT +// store stamped with scopeHash whose resource id is in resourceIDs (any +// resource type) — principal-scoped tombstones for RowKindResources. +func (e *Engine) DeleteResourcesByIDsInScope(ctx context.Context, scopeHash string, resourceIDs map[string]struct{}) (int64, error) { + if len(resourceIDs) == 0 { + return 0, nil + } + prefix := encodeResourceBySourceScopePrefix(scopeHash) + var deleted int64 + + err := e.withWrite(func() error { + if err := e.requireCurrentSync(); err != nil { + return err + } + iter, err := e.db.NewIter(&pebble.IterOptions{ + LowerBound: prefix, + UpperBound: upperBoundOf(prefix), + }) + if err != nil { + return err + } + defer iter.Close() + + opts := writeOpts(e.opts.durability) + if e.IsFreshSync() { + opts = pebble.NoSync + } + batch := e.db.NewBatch() + defer func() { _ = batch.Close() }() + + for iter.First(); iter.Valid(); iter.Next() { + if err := ctx.Err(); err != nil { + return err + } + key := iter.Key() + tail := key[len(prefix):] + // Tail layout: resource_type_id | resource_id. + rtBytes, next, ok := codec.DecodeTupleStringAlias(tail, 0) + if !ok || next >= len(tail) { + continue + } + ridBytes, _, ok := codec.DecodeTupleStringAlias(tail, next+1) + if !ok { + continue + } + if _, hit := resourceIDs[string(ridBytes)]; !hit { + continue + } + rt, rid := string(rtBytes), string(ridBytes) + priKey := make([]byte, 0, 3+len(tail)) + priKey = append(priKey, versionV3, typeResource) + priKey = codec.AppendTupleSeparator(priKey) + priKey = append(priKey, tail...) + // by_parent cleanup needs the parent ref from the value. + if val, closer, getErr := e.db.Get(priKey); getErr == nil { + err := e.deleteResourceIndexesRaw(batch, rt, rid, val) + closer.Close() + if err != nil { + return err + } + } else if !errors.Is(getErr, pebble.ErrNotFound) { + return getErr + } + if err := batch.Delete(priKey, nil); err != nil { + return err + } + // deleteResourceIndexesRaw already covered the scope entry + // (source_scope_hash is in the value), but delete the iterated + // key too in case the value read missed (orphan entry). + if err := batch.Delete(key, nil); err != nil { + return err + } + deleted++ + } + if err := iter.Error(); err != nil { + return err + } + return batch.Commit(opts) + }) + if err != nil { + return 0, err + } + return deleted, nil +} + +// grantValueHasSourcesRaw reports whether a marshaled GrantRecord carries +// at least one sources entry (field 9), without unmarshaling. +func grantValueHasSourcesRaw(value []byte) (bool, error) { + for len(value) > 0 { + num, typ, n := protowire.ConsumeTag(value) + if n < 0 { + return false, protowire.ParseError(n) + } + value = value[n:] + if num == 9 { + return true, nil + } + n = protowire.ConsumeFieldValue(num, typ, value) + if n < 0 { + return false, protowire.ParseError(n) + } + value = value[n:] + } + return false, nil +} + +// stripExpanderSourcesRaw clears a replayed grant's Sources map when it is +// expander-written, so the current sync's expansion recomputes it from +// true state instead of inheriting contributions that may have been +// removed upstream. Classification mirrors RollbackExpansion: a Sources +// map containing a self-source entry (keyed by the grant's own entitlement +// id) was written by the expander; one without a self-source is +// connector-set public data and is preserved. Returns (newValue, true) when +// the record was rewritten, (nil, false) when the original bytes should be +// copied verbatim. +func stripExpanderSourcesRaw(value []byte, ownEntitlementID string) ([]byte, bool, error) { + r := &v3.GrantRecord{} + if err := unmarshalRecord(value, r); err != nil { + return nil, false, fmt.Errorf("source cache replay: unmarshal grant for sources strip: %w", err) + } + sources := r.GetSources() + if len(sources) == 0 { + return nil, false, nil + } + if _, hasSelf := sources[ownEntitlementID]; !hasSelf { + // No self-source: connector-set Sources. Preserve verbatim. + return nil, false, nil + } + r.SetSources(nil) + stripped, err := marshalRecord(r) + if err != nil { + return nil, false, fmt.Errorf("source cache replay: re-marshal grant after sources strip: %w", err) + } + return stripped, true, nil +} + +// decodeResourcePrimaryTail decodes (resource_type_id, resource_id) +// from a resource primary key (v3 | typeResource | 0x00 | rt | 0x00 | rid). +func decodeResourcePrimaryTail(priKey []byte) (string, string, error) { + const headerLen = 3 // versionV3, typeResource, separator + if len(priKey) <= headerLen { + return "", "", fmt.Errorf("source cache replay: malformed resource primary key %x", priKey) + } + tail := priKey[headerLen:] + rtBytes, next, err := codec.DecodeTupleStringTo(nil, tail, 0) + if err != nil { + return "", "", err + } + if next >= len(tail) { + return "", "", fmt.Errorf("source cache replay: resource primary key missing resource_id: %x", priKey) + } + ridBytes, _, err := codec.DecodeTupleStringTo(nil, tail, next+1) + if err != nil { + return "", "", err + } + return string(rtBytes), string(ridBytes), nil +} + +// replayPrimaryFromIndexKey derives a record's primary key from its +// by_source_scope index key. The index prefix (header|0x00|scope|0x00) +// is followed by exactly the identity tuple that forms the primary +// key's tail, so the primary is header' + 0x00-separated remainder. +func replayPrimaryFromIndexKey(indexKey, indexPrefix []byte, primaryHeader [2]byte) ([]byte, error) { + if len(indexKey) <= len(indexPrefix) { + return nil, fmt.Errorf("source cache replay: malformed index key %x", indexKey) + } + tail := indexKey[len(indexPrefix):] + key := make([]byte, 0, 3+len(tail)) + key = append(key, primaryHeader[0], primaryHeader[1], 0x00) + return append(key, tail...), nil +} + +// ReplaySourceCacheGrants copies every grant stamped with scopeHash from +// prev into the receiver: raw primary copy plus index synthesis from the +// raw value (principal, needs_expansion, source-scope families). Mirrors +// PutGrantRecords' read-before-write index cleanup when the receiver +// already holds a record at the same identity. +func (e *Engine) ReplaySourceCacheGrants(ctx context.Context, prev *Engine, scopeHash string) (SourceCacheReplayResult, error) { + var res SourceCacheReplayResult + prefix := encodeGrantBySourceScopePrefix(scopeHash) + primaryHeader := [2]byte{versionV3, typeGrant} + + err := e.withWrite(func() error { + if err := e.requireCurrentSync(); err != nil { + return err + } + iter, err := prev.db.NewIter(&pebble.IterOptions{ + LowerBound: prefix, + UpperBound: upperBoundOf(prefix), + }) + if err != nil { + return err + } + defer iter.Close() + + opts := writeOpts(e.opts.durability) + if e.IsFreshSync() { + opts = pebble.NoSync + } + batch := e.db.NewBatch() + defer func() { _ = batch.Close() }() + rowsInBatch := 0 + + // Consumed in a defer keyed on rows whose commit LANDED, not on + // success: a replay that fails after an intermediate commit + // (large scope, error or cancellation mid-copy) has already + // populated the keyspace, and while the failing action unwinds, + // concurrently draining workers can still write — a first + // PutGrantRecords taking the empty-keyspace fast path over + // partially replayed identities would skip index cleanup. + // + // sawRelatedResource: InsertResourceGrants copies write resource + // rows too, so the resources empty-keyspace fast path must also + // be disarmed. Related copies always ride a batch that carries + // at least their own grant row, so committedRows > 0 covers + // "any batch that may hold a resource write landed". + committedRows := 0 + sawRelatedResource := false + defer func() { + if committedRows > 0 { + _ = e.takeFreshGrantsEmpty() + if sawRelatedResource { + _ = e.takeFreshResourcesEmpty() + } + } + }() + // relatedCopied dedupes InsertResourceGrants resource copies: + // many grants in one scope typically reference few resources. + var relatedCopied map[string]struct{} + + for iter.First(); iter.Valid(); iter.Next() { + if err := ctx.Err(); err != nil { + return err + } + priKey, err := replayPrimaryFromIndexKey(iter.Key(), prefix, primaryHeader) + if err != nil { + return err + } + val, closer, getErr := prev.db.Get(priKey) + if getErr != nil { + if errors.Is(getErr, pebble.ErrNotFound) { + // Orphan index entry in the previous file — skip, + // matching the defensive-skip semantic of the other + // index read paths. + res.StaleSkipped++ + continue + } + return fmt.Errorf("source cache replay: get prev grant: %w", getErr) + } + + entRT, entRID, entID, principalRT, principalID, needsExpansion, srcScope, scanErr := scanGrantIndexFieldsRaw(val) + if scanErr != nil { + closer.Close() + return scanErr + } + // Stale-index defense: only copy rows whose VALUE stamp matches + // the queried scope. An index entry pointing at a row stamped + // differently (or not at all) is left over from a path that + // replaced the row without cleaning the index — e.g. a fold + // compaction predating the source-cache bucket plans, or an + // in-sync same-identity rewrite under a different scope. Copying + // it would inject rows upstream never returned for this scope. + if srcScope != scopeHash { + closer.Close() + res.StaleSkipped++ + continue + } + + // Clean up index entries for any record the current sync + // already wrote at this identity (same discipline as + // PutGrantRecords' read-before-write). + if oldVal, oldCloser, oldErr := e.db.Get(priKey); oldErr == nil { + if err := e.deleteGrantIndexesRaw(batch, "", oldVal); err != nil { + oldCloser.Close() + closer.Close() + return err + } + oldCloser.Close() + } else if !errors.Is(oldErr, pebble.ErrNotFound) { + closer.Close() + return fmt.Errorf("source cache replay: get current grant: %w", oldErr) + } + + // Replay-equivalence: a cached sync must reproduce what a full + // resync would produce. The one field where a verbatim copy + // diverges is expander-written Sources — the previous sync's + // expansion baked contributions into direct grants, and + // re-expansion only ADDS, so a contribution removed this sync + // (via a delta tombstone or a refetched page) would survive + // forever. Strip expander-written Sources so the current sync's + // expansion recomputes them from true state; connector-set + // Sources (no self-source entry — same classification as + // RollbackExpansion) are connector data and are preserved + // verbatim. The probe is a cheap protowire scan; the vast + // majority of rows carry no Sources and stay on the raw-copy + // path. + writeVal := val + if hasSources, probeErr := grantValueHasSourcesRaw(val); probeErr != nil { + closer.Close() + return probeErr + } else if hasSources { + stripped, strippedOK, stripErr := stripExpanderSourcesRaw(val, entID) + if stripErr != nil { + closer.Close() + return stripErr + } + if strippedOK { + writeVal = stripped + } + } + // Replay side-effect annotations: response rows are the only + // other place these are detected, and a replayed page has + // none — see SourceCacheReplayResult field docs. + flags, flagsErr := scanGrantSourceCacheFlagsRaw(val) + if flagsErr != nil { + closer.Close() + return flagsErr + } + if flags.externalResourceMatch { + res.HasExternalResourceGrants = true + } + if flags.insertResourceGrants && entRT != "" && entRID != "" { + resKey := encodeResourceKey(entRT, entRID) + if _, done := relatedCopied[string(resKey)]; !done { + if relatedCopied == nil { + relatedCopied = make(map[string]struct{}) + } + relatedCopied[string(resKey)] = struct{}{} + copied, relErr := e.replayRelatedResourceRaw(batch, prev, resKey, entRT, entRID) + if relErr != nil { + closer.Close() + return relErr + } + if copied { + res.RelatedResourceRows++ + sawRelatedResource = true + } + } + } + + if err := batch.Set(priKey, writeVal, nil); err != nil { + closer.Close() + return err + } + closer.Close() + + if entID != "" && entRT != "" && entRID != "" && principalRT != "" && principalID != "" { + id := grantIdentity{ + entitlement: entitlementIdentityFromParts(entRT, entRID, entID), + principalTypeID: principalRT, + principalID: principalID, + } + if _, err := e.writeGrantIndexesForIdentityScratch(batch, id, needsExpansion, srcScope, nil); err != nil { + return err + } + } + if needsExpansion { + res.NeedsExpansion = true + } + res.Rows++ + rowsInBatch++ + if rowsInBatch >= replayBatchRows { + if err := batch.Commit(opts); err != nil { + return err + } + committedRows += rowsInBatch + _ = batch.Close() + batch = e.db.NewBatch() + rowsInBatch = 0 + } + } + if err := iter.Error(); err != nil { + return err + } + if err := batch.Commit(opts); err != nil { + return err + } + // Replay populated the fresh sync's grant keyspace directly. The + // first overlay PutGrantRecords must therefore perform its normal + // read-before-write index cleanup, rather than claiming the + // keyspace is still empty and leaving replayed index entries stale + // (consumed by the defer above). + committedRows += rowsInBatch + return nil + }) + if err != nil { + return SourceCacheReplayResult{}, err + } + return res, nil +} + +// replayRelatedResourceRaw copies one resource row from prev into batch +// (primary + secondary indexes), byte-verbatim — the engine side of +// recreating InsertResourceGrants-inserted resources when their grants +// page replays. Returns false when prev holds no such row: the previous +// sync's output didn't have it either, so there is nothing to preserve. +func (e *Engine) replayRelatedResourceRaw(batch *pebble.Batch, prev *Engine, priKey []byte, rt, rid string) (bool, error) { + val, closer, err := prev.db.Get(priKey) + if err != nil { + if errors.Is(err, pebble.ErrNotFound) { + return false, nil + } + return false, fmt.Errorf("source cache replay: get prev related resource: %w", err) + } + defer closer.Close() + + parentRT, parentID, srcScope, err := scanResourceIndexFieldsRaw(val) + if err != nil { + return false, err + } + // Clean up index entries for any record the current sync already + // wrote at this identity (same discipline as the resource replay). + if oldVal, oldCloser, oldErr := e.db.Get(priKey); oldErr == nil { + err := e.deleteResourceIndexesRaw(batch, rt, rid, oldVal) + oldCloser.Close() + if err != nil { + return false, err + } + } else if !errors.Is(oldErr, pebble.ErrNotFound) { + return false, fmt.Errorf("source cache replay: get current related resource: %w", oldErr) + } + if err := batch.Set(priKey, val, nil); err != nil { + return false, err + } + if parentID != "" { + if err := batch.Set(encodeResourceByParentIndexKey(parentRT, parentID, rt, rid), nil, nil); err != nil { + return false, err + } + } + if srcScope != "" { + if err := batch.Set(encodeResourceBySourceScopeIndexKey(srcScope, rt, rid), nil, nil); err != nil { + return false, err + } + } + return true, nil +} + +// ReplaySourceCacheEntitlements copies every entitlement stamped with +// scopeHash from prev into the receiver. +func (e *Engine) ReplaySourceCacheEntitlements(ctx context.Context, prev *Engine, scopeHash string) (SourceCacheReplayResult, error) { + var res SourceCacheReplayResult + prefix := encodeEntitlementBySourceScopePrefix(scopeHash) + primaryHeader := [2]byte{versionV3, typeEntitlement} + + err := e.withWrite(func() error { + if err := e.requireCurrentSync(); err != nil { + return err + } + iter, err := prev.db.NewIter(&pebble.IterOptions{ + LowerBound: prefix, + UpperBound: upperBoundOf(prefix), + }) + if err != nil { + return err + } + defer iter.Close() + + opts := writeOpts(e.opts.durability) + if e.IsFreshSync() { + opts = pebble.NoSync + } + batch := e.db.NewBatch() + defer func() { _ = batch.Close() }() + rowsInBatch := 0 + + // Keyed on rows whose commit LANDED, not on success (see the + // grants replay): a partial replay has already mutated the + // entitlement keyspace, so the bare-id lookup map must be + // invalidated and the empty-keyspace fast path disarmed even when + // the replay itself fails — draining workers can still resolve + // tombstones and write entitlements while the failure unwinds. + committedRows := 0 + defer func() { + if committedRows > 0 { + e.noteEntitlementKeyspaceWrite() + _ = e.takeFreshEntitlementsEmpty() + } + }() + + for iter.First(); iter.Valid(); iter.Next() { + if err := ctx.Err(); err != nil { + return err + } + priKey, err := replayPrimaryFromIndexKey(iter.Key(), prefix, primaryHeader) + if err != nil { + return err + } + val, closer, getErr := prev.db.Get(priKey) + if getErr != nil { + if errors.Is(getErr, pebble.ErrNotFound) { + res.StaleSkipped++ + continue + } + return fmt.Errorf("source cache replay: get prev entitlement: %w", getErr) + } + // Stale-index defense: only copy rows whose VALUE stamp matches + // the queried scope (see the grants replay for rationale). + prevScope, prevScanErr := scanEntitlementSourceScopeRaw(val) + if prevScanErr != nil { + closer.Close() + return prevScanErr + } + if prevScope != scopeHash { + closer.Close() + res.StaleSkipped++ + continue + } + // Exclusion-group annotations must reach the syncer's + // validator — see SourceCacheReplayResult.ExclusionGroups. + egs, egErr := scanEntitlementExclusionGroupsRaw(val) + if egErr != nil { + closer.Close() + return egErr + } + res.ExclusionGroups = append(res.ExclusionGroups, egs...) + // The replayed row's source-scope index key is byte-identical + // to prev's — the only entitlement secondary index. Clean up a + // differing stamp on any record the current sync already wrote + // at this identity. + if oldVal, oldCloser, oldErr := e.db.Get(priKey); oldErr == nil { + oldScope, scanErr := scanEntitlementSourceScopeRaw(oldVal) + oldCloser.Close() + if scanErr != nil { + closer.Close() + return scanErr + } + if oldScope != "" && oldScope != scopeHash { + // The old index key's tail equals the primary key's + // tail (identity tuple), so rebuild it byte-wise. + oldIdxKey := make([]byte, 0, 8+len(oldScope)+len(priKey)) + oldIdxKey = append(oldIdxKey, versionV3, typeIndex, idxEntitlementBySourceScope) + oldIdxKey = codec.AppendTupleSeparator(oldIdxKey) + oldIdxKey = codec.AppendTupleStrings(oldIdxKey, oldScope) + oldIdxKey = codec.AppendTupleSeparator(oldIdxKey) + oldIdxKey = append(oldIdxKey, priKey[3:]...) + if err := batch.Delete(oldIdxKey, nil); err != nil { + closer.Close() + return err + } + } + } else if !errors.Is(oldErr, pebble.ErrNotFound) { + closer.Close() + return fmt.Errorf("source cache replay: get current entitlement: %w", oldErr) + } + if err := batch.Set(priKey, val, nil); err != nil { + closer.Close() + return err + } + closer.Close() + if err := batch.Set(iter.Key(), nil, nil); err != nil { + return err + } + res.Rows++ + rowsInBatch++ + if rowsInBatch >= replayBatchRows { + if err := batch.Commit(opts); err != nil { + return err + } + committedRows += rowsInBatch + _ = batch.Close() + batch = e.db.NewBatch() + rowsInBatch = 0 + } + } + if err := iter.Error(); err != nil { + return err + } + if err := batch.Commit(opts); err != nil { + return err + } + // The defer above invalidates the lazy bare-id lookup map (see + // lookup.go — later tombstones would silently miss replayed rows + // otherwise) and disarms the empty-keyspace fast path for the + // first overlay PutEntitlementRecords. The bump happens strictly + // AFTER the last commit that landed: a map build racing the + // replay records the pre-bump generation and is correctly + // invalidated; bumping earlier would let such a build record the + // fresh generation against partially-replayed state. + committedRows += rowsInBatch + return nil + }) + if err != nil { + return SourceCacheReplayResult{}, err + } + return res, nil +} + +// ReplaySourceCacheResources copies every resource stamped with scopeHash +// from prev into the receiver, synthesizing by_parent and by_source_scope +// index entries from the raw value. +func (e *Engine) ReplaySourceCacheResources(ctx context.Context, prev *Engine, scopeHash string) (SourceCacheReplayResult, error) { + var res SourceCacheReplayResult + prefix := encodeResourceBySourceScopePrefix(scopeHash) + primaryHeader := [2]byte{versionV3, typeResource} + + err := e.withWrite(func() error { + if err := e.requireCurrentSync(); err != nil { + return err + } + iter, err := prev.db.NewIter(&pebble.IterOptions{ + LowerBound: prefix, + UpperBound: upperBoundOf(prefix), + }) + if err != nil { + return err + } + defer iter.Close() + + opts := writeOpts(e.opts.durability) + if e.IsFreshSync() { + opts = pebble.NoSync + } + batch := e.db.NewBatch() + defer func() { _ = batch.Close() }() + rowsInBatch := 0 + + // Keyed on rows whose commit LANDED, not on success — see the + // grants replay for rationale. + committedRows := 0 + defer func() { + if committedRows > 0 { + _ = e.takeFreshResourcesEmpty() + } + }() + + for iter.First(); iter.Valid(); iter.Next() { + if err := ctx.Err(); err != nil { + return err + } + priKey, err := replayPrimaryFromIndexKey(iter.Key(), prefix, primaryHeader) + if err != nil { + return err + } + val, closer, getErr := prev.db.Get(priKey) + if getErr != nil { + if errors.Is(getErr, pebble.ErrNotFound) { + res.StaleSkipped++ + continue + } + return fmt.Errorf("source cache replay: get prev resource: %w", getErr) + } + + rt, rid, decodeErr := decodeResourcePrimaryTail(priKey) + if decodeErr != nil { + closer.Close() + return decodeErr + } + parentRT, parentID, srcScope, scanErr := scanResourceIndexFieldsRaw(val) + if scanErr != nil { + closer.Close() + return scanErr + } + // Stale-index defense: only copy rows whose VALUE stamp matches + // the queried scope (see the grants replay for rationale). + if srcScope != scopeHash { + closer.Close() + res.StaleSkipped++ + continue + } + // Child scheduling is response-row-driven in the syncer; a + // replayed parent never reaches that path, so surface its + // child types for the syncer to schedule. + childTypeIDs, childErr := scanResourceChildTypeIDsRaw(val) + if childErr != nil { + closer.Close() + return childErr + } + if len(childTypeIDs) > 0 { + res.ChildResources = append(res.ChildResources, ReplayedParentResource{ + ResourceTypeID: rt, + ResourceID: rid, + ChildTypeIDs: childTypeIDs, + }) + } + if oldVal, oldCloser, oldErr := e.db.Get(priKey); oldErr == nil { + if err := e.deleteResourceIndexesRaw(batch, rt, rid, oldVal); err != nil { + oldCloser.Close() + closer.Close() + return err + } + oldCloser.Close() + } else if !errors.Is(oldErr, pebble.ErrNotFound) { + closer.Close() + return fmt.Errorf("source cache replay: get current resource: %w", oldErr) + } + + if err := batch.Set(priKey, val, nil); err != nil { + closer.Close() + return err + } + closer.Close() + + if parentID != "" { + if err := batch.Set(encodeResourceByParentIndexKey(parentRT, parentID, rt, rid), nil, nil); err != nil { + return err + } + } + if srcScope != "" { + if err := batch.Set(encodeResourceBySourceScopeIndexKey(srcScope, rt, rid), nil, nil); err != nil { + return err + } + } + res.Rows++ + rowsInBatch++ + if rowsInBatch >= replayBatchRows { + if err := batch.Commit(opts); err != nil { + return err + } + committedRows += rowsInBatch + _ = batch.Close() + batch = e.db.NewBatch() + rowsInBatch = 0 + } + } + if err := iter.Error(); err != nil { + return err + } + if err := batch.Commit(opts); err != nil { + return err + } + // See grant replay above: direct replay writes mean the first + // overlay PutResourceRecords must not use the empty-keyspace + // fast-path, or old by_parent/by_source_scope entries survive + // (consumed by the defer above). + committedRows += rowsInBatch + return nil + }) + if err != nil { + return SourceCacheReplayResult{}, err + } + return res, nil +} diff --git a/pkg/dotc1z/engine/pebble/source_cache_engine_test.go b/pkg/dotc1z/engine/pebble/source_cache_engine_test.go new file mode 100644 index 000000000..d275b6464 --- /dev/null +++ b/pkg/dotc1z/engine/pebble/source_cache_engine_test.go @@ -0,0 +1,529 @@ +package pebble + +import ( + "context" + "testing" + + "github.com/cockroachdb/pebble/v2" + "github.com/stretchr/testify/require" + + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + v3 "github.com/conductorone/baton-sdk/pb/c1/storage/v3" + "github.com/conductorone/baton-sdk/pkg/annotations" + "github.com/conductorone/baton-sdk/pkg/connectorstore" + "github.com/conductorone/baton-sdk/pkg/sourcecache" + batonGrant "github.com/conductorone/baton-sdk/pkg/types/grant" +) + +var ( + scopeA = sourcecache.HashScope("https://example.test/teams/1/members?page=1") + scopeB = sourcecache.HashScope("https://example.test/teams/2/members?page=1") +) + +func scGrant(entName, principalID string, expandable bool) *v2.Grant { + // Public ids are the raw ":"-join (an external contract, never parsed + // for identity — see identity.go). + canonicalEntID := "group:g1:" + entName + ent := v2.Entitlement_builder{ + Id: canonicalEntID, + Resource: v2.Resource_builder{ + Id: v2.ResourceId_builder{ResourceType: "group", Resource: "g1"}.Build(), + }.Build(), + }.Build() + principal := v2.Resource_builder{ + Id: v2.ResourceId_builder{ResourceType: "user", Resource: principalID}.Build(), + }.Build() + var annos annotations.Annotations + if expandable { + annos.Update(v2.GrantExpandable_builder{EntitlementIds: []string{canonicalEntID}}.Build()) + } + return v2.Grant_builder{ + Id: batonGrant.NewGrantID(principal, ent), + Entitlement: ent, + Principal: principal, + Annotations: annos, + }.Build() +} + +func TestSourceCacheEntryCRUD(t *testing.T) { + ctx := context.Background() + a := newAdapter(t) + _, err := a.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + + e := a.PebbleEngine() + _, err = e.GetSourceCacheEntry(ctx, "grants", scopeA) + require.ErrorIs(t, err, pebble.ErrNotFound) + + require.NoError(t, e.PutSourceCacheEntry(ctx, "grants", scopeA, `W/"etag-1"`)) + rec, err := e.GetSourceCacheEntry(ctx, "grants", scopeA) + require.NoError(t, err) + require.Equal(t, `W/"etag-1"`, rec.GetEtag()) + require.NotNil(t, rec.GetDiscoveredAt()) + + // Row-kind partition: the same scope hash under a different kind misses. + _, err = e.GetSourceCacheEntry(ctx, "resources", scopeA) + require.ErrorIs(t, err, pebble.ErrNotFound) + + // Rotation overwrites in place (delta tokens rotate every round). + require.NoError(t, e.PutSourceCacheEntry(ctx, "grants", scopeA, "delta-token-2")) + rec, err = e.GetSourceCacheEntry(ctx, "grants", scopeA) + require.NoError(t, err) + require.Equal(t, "delta-token-2", rec.GetEtag()) +} + +// TestSourceCacheReplayGrantsAcrossEngines is the core replay contract: +// grants stamped with a scope in file A are copied — rows, indexes, and +// needs_expansion reporting — into file B, and unstamped/other-scope rows +// stay behind. +func TestSourceCacheReplayGrantsAcrossEngines(t *testing.T) { + ctx := context.Background() + + prev := newAdapter(t) + _, err := prev.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + + gExpandable := scGrant("member", "alice", true) + gPlain := scGrant("member", "bob", false) + gOtherScope := scGrant("member", "carol", false) + gUnstamped := scGrant("member", "dave", false) + + scopedCtx := sourcecache.WithScope(ctx, scopeA) + require.NoError(t, prev.PutGrants(scopedCtx, gExpandable, gPlain)) + require.NoError(t, prev.PutGrants(sourcecache.WithScope(ctx, scopeB), gOtherScope)) + require.NoError(t, prev.PutGrants(ctx, gUnstamped)) + require.NoError(t, prev.PebbleEngine().PutSourceCacheEntry(ctx, "grants", scopeA, "etag-a")) + + cur := newAdapter(t) + _, err = cur.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + + res, err := cur.PebbleEngine().ReplaySourceCacheGrants(ctx, prev.PebbleEngine(), scopeA) + require.NoError(t, err) + require.Equal(t, int64(2), res.Rows) + require.True(t, res.NeedsExpansion, "replay must report the expandable grant so the syncer can arm expansion") + + // Replayed rows are readable by ID and carry their expansion flag. + got, err := cur.PebbleEngine().GetGrantRecord(ctx, gExpandable.GetId()) + require.NoError(t, err) + require.True(t, got.GetNeedsExpansion()) + require.Equal(t, scopeA, got.GetSourceScopeHash()) + _, err = cur.PebbleEngine().GetGrantRecord(ctx, gPlain.GetId()) + require.NoError(t, err) + + // Rows outside the scope are not copied. + _, err = cur.PebbleEngine().GetGrantRecord(ctx, gOtherScope.GetId()) + require.ErrorIs(t, err, pebble.ErrNotFound) + _, err = cur.PebbleEngine().GetGrantRecord(ctx, gUnstamped.GetId()) + require.ErrorIs(t, err, pebble.ErrNotFound) + + // The by_source_scope index came across too: a second replay FROM the + // current store (as next sync's previous) finds the same rows. + next := newAdapter(t) + _, err = next.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + res2, err := next.PebbleEngine().ReplaySourceCacheGrants(ctx, cur.PebbleEngine(), scopeA) + require.NoError(t, err) + require.Equal(t, int64(2), res2.Rows) +} + +// TestSourceCacheReplayIsIdempotent pins double-execution safety: a +// resumed sync can re-run a replay page whose first execution already +// committed (the checkpoint races the crash), so replaying the same scope +// twice into the same store must converge to the same state — same rows, +// same count reported, no error. +func TestSourceCacheReplayIsIdempotent(t *testing.T) { + ctx := context.Background() + + prev := newAdapter(t) + _, err := prev.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + g1 := scGrant("member", "alice", true) + g2 := scGrant("member", "bob", false) + require.NoError(t, prev.PutGrants(sourcecache.WithScope(ctx, scopeA), g1, g2)) + + cur := newAdapter(t) + _, err = cur.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + + res1, err := cur.PebbleEngine().ReplaySourceCacheGrants(ctx, prev.PebbleEngine(), scopeA) + require.NoError(t, err) + require.Equal(t, int64(2), res1.Rows) + + res2, err := cur.PebbleEngine().ReplaySourceCacheGrants(ctx, prev.PebbleEngine(), scopeA) + require.NoError(t, err) + require.Equal(t, int64(2), res2.Rows, "second replay converges, not errors") + require.Equal(t, res1.NeedsExpansion, res2.NeedsExpansion) + + // State after double replay is exactly the two rows, once each. + var seen int + require.NoError(t, cur.PebbleEngine().IterateGrants(ctx, func(*v3.GrantRecord) bool { + seen++ + return true + })) + require.Equal(t, 2, seen) +} + +// TestSourceCacheReplayIgnoresStaleIndexEntries pins the stale-index +// defense: an index entry whose target row is stamped with a DIFFERENT +// scope (left behind by a path that replaced rows without cleaning the +// index — e.g. a fold compaction predating the source-cache bucket plans) +// must not be copied. Replaying it would inject rows upstream never +// returned for the queried scope. +func TestSourceCacheReplayIgnoresStaleIndexEntries(t *testing.T) { + ctx := context.Background() + + prev := newAdapter(t) + _, err := prev.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + + // Row genuinely stamped with scopeB... + g := scGrant("member", "alice", false) + require.NoError(t, prev.PutGrants(sourcecache.WithScope(ctx, scopeB), g)) + + // ...plus a stale index entry claiming it belongs to scopeA. + rec, err := prev.PebbleEngine().GetGrantRecord(ctx, g.GetId()) + require.NoError(t, err) + staleIdx := replayTestGrantScopeIndexKey(t, scopeA, rec) + require.NoError(t, prev.PebbleEngine().DB().Set(staleIdx, nil, nil)) + + cur := newAdapter(t) + _, err = cur.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + + res, err := cur.PebbleEngine().ReplaySourceCacheGrants(ctx, prev.PebbleEngine(), scopeA) + require.NoError(t, err) + require.Zero(t, res.Rows, "a row stamped with a different scope must not be replayed via a stale index entry") + require.Equal(t, int64(1), res.StaleSkipped, + "the skipped stale entry must be reported — it is the syncer's only way to tell 'legitimately empty scope' from 'scope contents clobbered without index cleanup'") + _, err = cur.PebbleEngine().GetGrantRecord(ctx, g.GetId()) + require.ErrorIs(t, err, pebble.ErrNotFound) + + // The genuine scope still replays, with no staleness reported. + res, err = cur.PebbleEngine().ReplaySourceCacheGrants(ctx, prev.PebbleEngine(), scopeB) + require.NoError(t, err) + require.Equal(t, int64(1), res.Rows) + require.Zero(t, res.StaleSkipped) +} + +// TestEntitlementRestampCleansOldScopeIndex pins the read-before-write +// cleanup on the entitlement put path: rewriting the SAME entitlement +// identity under a NEW scope must delete the old scope's by_source_scope +// entry. Without the cleanup, the old scope's index still claims the row +// while its value stamp names the new scope — the next sync's replay of +// the old scope then copies zero rows against a manifest entry that +// promised content (Rows == 0, StaleSkipped > 0), which the syncer treats +// as data loss and fails. +func TestEntitlementRestampCleansOldScopeIndex(t *testing.T) { + ctx := context.Background() + + prev := newAdapter(t) + _, err := prev.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + + res1 := v2.Resource_builder{ + Id: v2.ResourceId_builder{ResourceType: "group", Resource: "g1"}.Build(), + }.Build() + entID := "group:g1:member" + ent1 := v2.Entitlement_builder{Id: entID, Resource: res1, DisplayName: "member"}.Build() + + // First put stamps scopeA (and consumes the fresh-empty fast path); + // the rewrite stamps scopeB and must clean scopeA's index entry. + require.NoError(t, prev.PutEntitlements(sourcecache.WithScope(ctx, scopeA), ent1)) + ent1b := v2.Entitlement_builder{Id: entID, Resource: res1, DisplayName: "member (renamed)"}.Build() + require.NoError(t, prev.PutEntitlements(sourcecache.WithScope(ctx, scopeB), ent1b)) + + cur := newAdapter(t) + _, err = cur.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + + // Old scope: legitimately empty now — no rows AND no stale entries. + resA, err := cur.PebbleEngine().ReplaySourceCacheEntitlements(ctx, prev.PebbleEngine(), scopeA) + require.NoError(t, err) + require.Zero(t, resA.Rows) + require.Zero(t, resA.StaleSkipped, "re-stamp must delete the old scope's index entry, not leave it stale") + + // New scope: exactly the rewritten row. + resB, err := cur.PebbleEngine().ReplaySourceCacheEntitlements(ctx, prev.PebbleEngine(), scopeB) + require.NoError(t, err) + require.Equal(t, int64(1), resB.Rows) + require.Zero(t, resB.StaleSkipped) + got, err := cur.PebbleEngine().GetEntitlementRecord(ctx, entID) + require.NoError(t, err) + require.Equal(t, scopeB, got.GetSourceScopeHash()) + require.Equal(t, "member (renamed)", got.GetDisplayName()) +} + +// replayTestGrantScopeIndexKey builds a by_source_scope index key for the +// given scope pointing at rec's identity, bypassing the write path — the +// test plants it as a stale entry. +func replayTestGrantScopeIndexKey(t *testing.T, scopeHash string, rec *v3.GrantRecord) []byte { + t.Helper() + id, err := grantIdentityFromRecord(rec) + require.NoError(t, err) + return encodeGrantBySourceScopeIndexKey(scopeHash, id) +} + +// TestDeleteGrantsByPrincipalsInScope pins the principal-scoped tombstone +// path: bare principal ids kill exactly their rows within the scope — +// other scopes' rows for the same principal survive, unknown principals +// no-op, and the deleted rows' index entries (including the scope index) +// go with them. +func TestDeleteGrantsByPrincipalsInScope(t *testing.T) { + ctx := context.Background() + a := newAdapter(t) + _, err := a.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + + gAlice := scGrant("member", "alice", false) + gBob := scGrant("member", "bob", true) + require.NoError(t, a.PutGrants(sourcecache.WithScope(ctx, scopeA), gAlice, gBob)) + // Same principal, different scope — must survive an A-scoped delete. + gAliceOther := scGrant("owner", "alice", false) + require.NoError(t, a.PutGrants(sourcecache.WithScope(ctx, scopeB), gAliceOther)) + + deleted, err := a.PebbleEngine().DeleteGrantsByPrincipalsInScope(ctx, scopeA, map[string]struct{}{ + "alice": {}, + "bob": {}, + "unknown": {}, // tombstone for a principal never synced — no-op + }) + require.NoError(t, err) + require.Equal(t, int64(2), deleted) + + _, err = a.PebbleEngine().GetGrantRecord(ctx, gAlice.GetId()) + require.ErrorIs(t, err, pebble.ErrNotFound) + _, err = a.PebbleEngine().GetGrantRecord(ctx, gBob.GetId()) + require.ErrorIs(t, err, pebble.ErrNotFound) + _, err = a.PebbleEngine().GetGrantRecord(ctx, gAliceOther.GetId()) + require.NoError(t, err, "same principal in a different scope must survive") + + // Scope index entries went with the rows: a replay of scopeA from this + // store copies nothing. + cur := newAdapter(t) + _, err = cur.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + res, err := cur.PebbleEngine().ReplaySourceCacheGrants(ctx, a.PebbleEngine(), scopeA) + require.NoError(t, err) + require.Zero(t, res.Rows) +} + +// TestDeleteResourcesByIDsInScope pins the resources analog: bare object +// ids, any resource type, scope-relative. +func TestDeleteResourcesByIDsInScope(t *testing.T) { + ctx := context.Background() + a := newAdapter(t) + _, err := a.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + + u1 := v2.Resource_builder{Id: v2.ResourceId_builder{ResourceType: "user", Resource: "u1"}.Build(), DisplayName: "U1"}.Build() + u2 := v2.Resource_builder{Id: v2.ResourceId_builder{ResourceType: "user", Resource: "u2"}.Build(), DisplayName: "U2"}.Build() + require.NoError(t, a.PutResources(sourcecache.WithScope(ctx, scopeA), u1, u2)) + u3 := v2.Resource_builder{Id: v2.ResourceId_builder{ResourceType: "user", Resource: "u3"}.Build(), DisplayName: "U3"}.Build() + require.NoError(t, a.PutResources(sourcecache.WithScope(ctx, scopeB), u3)) + + deleted, err := a.PebbleEngine().DeleteResourcesByIDsInScope(ctx, scopeA, map[string]struct{}{ + "u1": {}, "u3": {}, // u3 is in scope B — must not die from an A-scoped tombstone + }) + require.NoError(t, err) + require.Equal(t, int64(1), deleted) + + _, err = a.PebbleEngine().GetResourceRecord(ctx, "user", "u1") + require.ErrorIs(t, err, pebble.ErrNotFound) + _, err = a.PebbleEngine().GetResourceRecord(ctx, "user", "u2") + require.NoError(t, err) + _, err = a.PebbleEngine().GetResourceRecord(ctx, "user", "u3") + require.NoError(t, err, "tombstone must be scope-relative") +} + +// TestDeleteGrantRecordBounded pins the no-scan contract: SDK-shaped ids +// resolve and delete; absent ids no-op without error (and without the +// O(all grants) fallback scan, by construction). +func TestDeleteGrantRecordBounded(t *testing.T) { + ctx := context.Background() + a := newAdapter(t) + _, err := a.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + + g := scGrant("member", "alice", false) + require.NoError(t, a.PutGrants(ctx, g)) + + // Absent id (well-formed, no row): no-op, no error. + require.NoError(t, a.PebbleEngine().DeleteGrantRecordBounded(ctx, "group:g1:custom:member:user:ghost")) + _, err = a.PebbleEngine().GetGrantRecord(ctx, g.GetId()) + require.NoError(t, err) + + // Present id: deleted. + require.NoError(t, a.PebbleEngine().DeleteGrantRecordBounded(ctx, g.GetId())) + _, err = a.PebbleEngine().GetGrantRecord(ctx, g.GetId()) + require.ErrorIs(t, err, pebble.ErrNotFound) +} + +func TestSourceCacheReplayResourcesAndEntitlements(t *testing.T) { + ctx := context.Background() + + prev := newAdapter(t) + _, err := prev.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + + res1 := v2.Resource_builder{ + Id: v2.ResourceId_builder{ResourceType: "group", Resource: "g1"}.Build(), + DisplayName: "Group One", + }.Build() + entID := "group:g1:member" + ent1 := v2.Entitlement_builder{Id: entID, Resource: res1, DisplayName: "member"}.Build() + + scoped := sourcecache.WithScope(ctx, scopeA) + require.NoError(t, prev.PutResources(scoped, res1)) + require.NoError(t, prev.PutEntitlements(scoped, ent1)) + + cur := newAdapter(t) + _, err = cur.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + + rres, err := cur.PebbleEngine().ReplaySourceCacheResources(ctx, prev.PebbleEngine(), scopeA) + require.NoError(t, err) + require.Equal(t, int64(1), rres.Rows) + eres, err := cur.PebbleEngine().ReplaySourceCacheEntitlements(ctx, prev.PebbleEngine(), scopeA) + require.NoError(t, err) + require.Equal(t, int64(1), eres.Rows) + + gotRes, err := cur.PebbleEngine().GetResourceRecord(ctx, "group", "g1") + require.NoError(t, err) + require.Equal(t, "Group One", gotRes.GetDisplayName()) + require.Equal(t, scopeA, gotRes.GetSourceScopeHash()) + + gotEnt, err := cur.PebbleEngine().GetEntitlementRecord(ctx, entID) + require.NoError(t, err) + require.Equal(t, scopeA, gotEnt.GetSourceScopeHash()) +} + +// TestReplayInvalidatesEntitlementIDLookup pins the keyspace-generation +// contract for the replay writer: replay mutates the entitlement primary +// keyspace, so a bare-id lookup map built BEFORE the replay must be +// invalidated by it. If replay skips noteEntitlementKeyspaceWrite, the +// stale map hides replayed rows from resolveEntitlementIdentityByExternalID +// and a delta tombstone (DeleteEntitlementRecord) silently no-ops — a row +// upstream reported deleted survives the sync. +func TestReplayInvalidatesEntitlementIDLookup(t *testing.T) { + ctx := context.Background() + + prev := newAdapter(t) + _, err := prev.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + + res1 := v2.Resource_builder{ + Id: v2.ResourceId_builder{ResourceType: "group", Resource: "g1"}.Build(), + DisplayName: "Group One", + }.Build() + entID := "group:g1:member" + ent1 := v2.Entitlement_builder{Id: entID, Resource: res1, DisplayName: "member"}.Build() + require.NoError(t, prev.PutEntitlements(sourcecache.WithScope(ctx, scopeA), ent1)) + + cur := newAdapter(t) + _, err = cur.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + + // Force the lazy bare-id map to build BEFORE the replay (empty + // keyspace → entID resolves to nothing). + _, err = cur.PebbleEngine().GetEntitlementRecord(ctx, entID) + require.Error(t, err, "sanity: entitlement must not resolve before replay") + + rres, err := cur.PebbleEngine().ReplaySourceCacheEntitlements(ctx, prev.PebbleEngine(), scopeA) + require.NoError(t, err) + require.Equal(t, int64(1), rres.Rows) + + // The replayed row must be visible through the bare-id lookup (the + // map built above is stale and must have been invalidated) ... + gotEnt, err := cur.PebbleEngine().GetEntitlementRecord(ctx, entID) + require.NoError(t, err, "replayed entitlement must resolve by external id after a pre-replay map build") + require.Equal(t, scopeA, gotEnt.GetSourceScopeHash()) + + // ... and, the actual production stake: a delta tombstone against the + // replayed row must DELETE it, not silently no-op. + require.NoError(t, cur.PebbleEngine().DeleteEntitlementRecord(ctx, entID)) + _, err = cur.PebbleEngine().GetEntitlementRecord(ctx, entID) + require.Error(t, err, "tombstoned replayed entitlement must be gone") +} + +// TestStoreExpandedGrantsPreservesSourceScope pins the write-path contract +// that keeps replay correct across expansion: when the expander rewrites an +// existing direct grant (to bake in Sources), the record's source scope +// stamp — and therefore its membership in the next sync's replay of that +// scope — must survive, exactly like expansion/needs_expansion do. +func TestStoreExpandedGrantsPreservesSourceScope(t *testing.T) { + ctx := context.Background() + + a := newAdapter(t) + _, err := a.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + + g := scGrant("member", "alice", true) + require.NoError(t, a.PutGrants(sourcecache.WithScope(ctx, scopeA), g)) + + // Expander-style rewrite: same grant, Sources baked in, no scope ctx. + rewrite := scGrant("member", "alice", false) + rewrite.SetSources(v2.GrantSources_builder{ + Sources: map[string]*v2.GrantSources_GrantSource{ + g.GetEntitlement().GetId(): {}, + }, + }.Build()) + require.NoError(t, a.Grants().StoreExpandedGrants(ctx, rewrite)) + + got, err := a.PebbleEngine().GetGrantRecord(ctx, g.GetId()) + require.NoError(t, err) + require.Equal(t, scopeA, got.GetSourceScopeHash(), "expander rewrite clobbered the source scope stamp") + require.True(t, got.GetNeedsExpansion(), "expansion side-state must be preserved too") + + // And the index survives: a replay from this store still finds the row. + cur := newAdapter(t) + _, err = cur.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + res, err := cur.PebbleEngine().ReplaySourceCacheGrants(ctx, a.PebbleEngine(), scopeA) + require.NoError(t, err) + require.Equal(t, int64(1), res.Rows) + replayed, err := cur.PebbleEngine().GetGrantRecord(ctx, g.GetId()) + require.NoError(t, err) + // Replay-equivalence: expander-written Sources (self-source present) + // are stripped so the current sync's expansion recomputes them from + // true state — a full resync would produce Sources reflecting current + // membership, so a cached sync must too. needs_expansion survived the + // copy (asserted via the replay result above), so expansion re-adds + // them. + require.Empty(t, replayed.GetSources(), "expander-written Sources must be stripped on replay for full-resync equivalence") + require.True(t, replayed.GetNeedsExpansion()) +} + +// TestSourceCacheReplayPreservesConnectorSetSources pins the other half of +// the sources-strip classification: a Sources map WITHOUT a self-source +// entry is connector-set public data (the connector emitted it on the +// ordinary path) and must survive replay byte-for-byte — never stripped. +func TestSourceCacheReplayPreservesConnectorSetSources(t *testing.T) { + ctx := context.Background() + + prev := newAdapter(t) + _, err := prev.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + + g := scGrant("member", "alice", false) + // Connector-set provenance: keyed by a FOREIGN entitlement, no + // self-source. This is public v2.Grant data, not expander output. + g.SetSources(v2.GrantSources_builder{ + Sources: map[string]*v2.GrantSources_GrantSource{ + "group:other:member": {}, + }, + }.Build()) + require.NoError(t, prev.PutGrants(sourcecache.WithScope(ctx, scopeA), g)) + + cur := newAdapter(t) + _, err = cur.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + res, err := cur.PebbleEngine().ReplaySourceCacheGrants(ctx, prev.PebbleEngine(), scopeA) + require.NoError(t, err) + require.Equal(t, int64(1), res.Rows) + + replayed, err := cur.PebbleEngine().GetGrantRecord(ctx, g.GetId()) + require.NoError(t, err) + require.Len(t, replayed.GetSources(), 1, "connector-set Sources must survive replay") + _, ok := replayed.GetSources()["group:other:member"] + require.True(t, ok) +} diff --git a/pkg/dotc1z/source_cache.go b/pkg/dotc1z/source_cache.go new file mode 100644 index 000000000..80171981f --- /dev/null +++ b/pkg/dotc1z/source_cache.go @@ -0,0 +1,213 @@ +package dotc1z + +import ( + "context" + "errors" + "fmt" + + cdbpebble "github.com/cockroachdb/pebble/v2" + + "github.com/conductorone/baton-sdk/pkg/bid" + "github.com/conductorone/baton-sdk/pkg/connectorstore" + "github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble" + "github.com/conductorone/baton-sdk/pkg/sourcecache" +) + +// SourceCacheReplayResult reports what one scope's replay copied. +type SourceCacheReplayResult = pebble.SourceCacheReplayResult + +// SourceCacheStore is the optional store capability backing source-cache +// replay (see proto/c1/connector/v2/annotation_source_cache.proto). It is +// implemented ONLY by the Pebble engine; the syncer type-asserts for it and +// treats a store without it as "source cache unsupported" (no-op lookup, +// no replay). It is deliberately NOT part of c1zstore.Store. +type SourceCacheStore interface { + // LookupSourceCacheEntry returns this store's manifest entry for + // (kind, scopeHash). Backs the connector-facing lookup when this + // store is the previous sync. + LookupSourceCacheEntry(ctx context.Context, kind sourcecache.RowKind, scopeHash string) (sourcecache.Entry, bool, error) + + // PutSourceCacheEntry writes the current sync's manifest entry for + // (kind, scopeHash). Zero-row scopes still get entries. + PutSourceCacheEntry(ctx context.Context, kind sourcecache.RowKind, scopeHash string, etag string) error + + // ReplaySourceCache copies every row stamped with scopeHash from prev + // (the previous sync's store, opened read-only) into this store. prev + // must be a Pebble store. Does NOT write the manifest entry — the + // caller writes it after the scope's overlay/deletes complete, so a + // failed replay can't leave a phantom hit for the next sync. + ReplaySourceCache(ctx context.Context, prev connectorstore.Reader, kind sourcecache.RowKind, scopeHash string) (SourceCacheReplayResult, error) + + // DeleteSourceCacheRows removes rows by public canonical ID from the + // current sync, after replay + overlay (delta-query tombstones). + // ID formats per kind: grants and entitlements use their canonical + // IDs; resources use Baton resource BIDs ("bid:r:..."). + // + // Grant resolution is BOUNDED: candidate probing only, never the + // O(all grants) stored-external-id scan. Grants stored under + // connector-custom ids are unreachable here (delete no-ops); such + // connectors use DeleteSourceCacheRowsInScope instead. + DeleteSourceCacheRows(ctx context.Context, kind sourcecache.RowKind, ids []string) error + + // DeleteSourceCacheRowsInScope removes rows stamped with scopeHash by + // bare object id — grants by principal id (no principal type, no + // canonical-id reconstruction), resources by resource id (any type). + // One index scan of the scope per call; a page's tombstones are + // batched into one call. Ids with no matching rows are no-ops. + // Not supported for entitlements. + DeleteSourceCacheRowsInScope(ctx context.Context, kind sourcecache.RowKind, scopeHash string, ids []string) (int64, error) + + // DeleteSourceCacheGrantsByIDInScope removes grant rows stamped with + // scopeHash whose STORED grant id is in ids — works for + // connector-custom grant-id shapes that the global bounded delete + // cannot resolve, and stays bounded by the scope's row count. Ids with + // no matching rows are no-ops. + DeleteSourceCacheGrantsByIDInScope(ctx context.Context, scopeHash string, ids []string) (int64, error) +} + +var _ SourceCacheStore = (*pebbleStore)(nil) + +// sourceCacheEngine recovers the Pebble engine from an arbitrary store, +// nil-safe. Mirrors pebble.AsEngine but accepts any value so the syncer +// can probe its previous-sync reader without caring about its static type. +func sourceCacheEngine(store any) (*pebble.Engine, bool) { + a, ok := store.(interface{ PebbleEngine() *pebble.Engine }) + if !ok { + return nil, false + } + e := a.PebbleEngine() + return e, e != nil +} + +func (s *pebbleStore) LookupSourceCacheEntry(ctx context.Context, kind sourcecache.RowKind, scopeHash string) (sourcecache.Entry, bool, error) { + if err := sourcecache.ValidateRowKind(kind); err != nil { + return sourcecache.Entry{}, false, err + } + rec, err := s.engine.GetSourceCacheEntry(ctx, string(kind), scopeHash) + if err != nil { + if errors.Is(err, cdbpebble.ErrNotFound) { + return sourcecache.Entry{}, false, nil + } + return sourcecache.Entry{}, false, err + } + return sourcecache.Entry{ + ETag: rec.GetEtag(), + DiscoveredAt: rec.GetDiscoveredAt().AsTime(), + }, true, nil +} + +func (s *pebbleStore) PutSourceCacheEntry(ctx context.Context, kind sourcecache.RowKind, scopeHash string, etag string) error { + if err := sourcecache.ValidateRowKind(kind); err != nil { + return err + } + return s.markDirty(s.engine.PutSourceCacheEntry(ctx, string(kind), scopeHash, etag)) +} + +func (s *pebbleStore) ReplaySourceCache(ctx context.Context, prev connectorstore.Reader, kind sourcecache.RowKind, scopeHash string) (SourceCacheReplayResult, error) { + prevEngine, ok := sourceCacheEngine(prev) + if !ok { + return SourceCacheReplayResult{}, errors.New("source cache replay: previous sync store is not a pebble store") + } + var res SourceCacheReplayResult + var err error + switch kind { + case sourcecache.RowKindResources: + res, err = s.engine.ReplaySourceCacheResources(ctx, prevEngine, scopeHash) + case sourcecache.RowKindEntitlements: + res, err = s.engine.ReplaySourceCacheEntitlements(ctx, prevEngine, scopeHash) + case sourcecache.RowKindGrants: + res, err = s.engine.ReplaySourceCacheGrants(ctx, prevEngine, scopeHash) + default: + return SourceCacheReplayResult{}, fmt.Errorf("source cache replay: invalid row kind %q", kind) + } + if err != nil { + return SourceCacheReplayResult{}, err + } + if res.Rows > 0 { + s.MarkDirty() + } + return res, nil +} + +// DeleteSourceCacheRows deletes delta tombstones by public id string. +// +// NOTE on the bare-id lookup safety contract (engine/pebble/lookup.go): +// sync paths normally must not resolve grants by string. This path is a +// deliberate, narrow exception: tombstone ids are strings the connector +// itself emitted for these rows, volumes are delta-sized (not O(rows)), +// and resolution keeps the exactly-one rule — an ambiguous id fails the +// sync loudly rather than guessing a delete, which matches the +// source-cache replay-phase error policy. +func (s *pebbleStore) DeleteSourceCacheRows(ctx context.Context, kind sourcecache.RowKind, ids []string) error { + for _, id := range ids { + switch kind { + case sourcecache.RowKindGrants: + if err := s.markDirty(s.engine.DeleteGrantRecordBounded(ctx, id)); err != nil { + return fmt.Errorf("source cache delete grant %q: %w", id, err) + } + case sourcecache.RowKindEntitlements: + if err := s.markDirty(s.engine.DeleteEntitlementRecord(ctx, id)); err != nil { + return fmt.Errorf("source cache delete entitlement %q: %w", id, err) + } + case sourcecache.RowKindResources: + r, err := bid.ParseResourceBid(id) + if err != nil { + return fmt.Errorf("source cache delete resource: invalid resource bid %q: %w", id, err) + } + rid := r.GetId() + if err := s.markDirty(s.engine.DeleteResourceRecord(ctx, rid.GetResourceType(), rid.GetResource())); err != nil { + return fmt.Errorf("source cache delete resource %q: %w", id, err) + } + default: + return fmt.Errorf("source cache delete: invalid row kind %q", kind) + } + } + return nil +} + +func (s *pebbleStore) DeleteSourceCacheGrantsByIDInScope(ctx context.Context, scopeHash string, ids []string) (int64, error) { + if len(ids) == 0 { + return 0, nil + } + idSet := make(map[string]struct{}, len(ids)) + for _, id := range ids { + idSet[id] = struct{}{} + } + deleted, err := s.engine.DeleteGrantsByExternalIDsInScope(ctx, scopeHash, idSet) + if err != nil { + return 0, fmt.Errorf("source cache grant-id delete for scope %q: %w", scopeHash, err) + } + if deleted > 0 { + s.MarkDirty() + } + return deleted, nil +} + +func (s *pebbleStore) DeleteSourceCacheRowsInScope(ctx context.Context, kind sourcecache.RowKind, scopeHash string, ids []string) (int64, error) { + if len(ids) == 0 { + return 0, nil + } + idSet := make(map[string]struct{}, len(ids)) + for _, id := range ids { + idSet[id] = struct{}{} + } + var deleted int64 + var err error + switch kind { + case sourcecache.RowKindGrants: + deleted, err = s.engine.DeleteGrantsByPrincipalsInScope(ctx, scopeHash, idSet) + case sourcecache.RowKindResources: + deleted, err = s.engine.DeleteResourcesByIDsInScope(ctx, scopeHash, idSet) + case sourcecache.RowKindEntitlements: + return 0, fmt.Errorf("source cache scoped delete: not supported for entitlements") + default: + return 0, fmt.Errorf("source cache scoped delete: invalid row kind %q", kind) + } + if err != nil { + return 0, fmt.Errorf("source cache scoped delete for scope %q: %w", scopeHash, err) + } + if deleted > 0 { + s.MarkDirty() + } + return deleted, nil +} diff --git a/pkg/field/defaults.go b/pkg/field/defaults.go index e45726643..40c4e5511 100644 --- a/pkg/field/defaults.go +++ b/pkg/field/defaults.go @@ -264,6 +264,17 @@ var ( WithDescription("The path to the c1z file to sync external baton resources with"), WithPersistent(true), WithExportTarget(ExportTargetNone)) + // PreviousSyncC1ZField is hidden by default: source-cache replay is + // author-opt-in functionality (connectorrunner.WithKeepPreviousSyncC1Z), + // and non-replaying connectors shouldn't advertise it to operators. + // DefineConfiguration unhides it for connectors that declare the + // capability; the flag still parses everywhere (hidden ≠ disabled) so + // expert/debug use keeps working. + PreviousSyncC1ZField = StringField("previous-sync-c1z", + WithDescription("The path to the previous sync c1z file to use as a source-cache replay input"), + WithPersistent(true), + WithHidden(true), + WithExportTarget(ExportTargetNone)) externalResourceEntitlementIdFilter = StringField("external-resource-entitlement-id-filter", WithDescription("The entitlement that external users, groups must have access to sync external baton resources"), WithPersistent(true), @@ -276,11 +287,14 @@ var ( // connectors whose author also declared ETag-replay support at build // time (connectorrunner.WithKeepPreviousSyncC1Z) — both are // required. Costs one c1z of local disk. + // Hidden by default for the same reason as PreviousSyncC1ZField; unhidden + // when the connector author declares the replay capability. KeepPreviousSyncC1ZField = BoolField("keep-previous-sync-c1z", WithDescription("Keep the previously synced c1z on disk to enable ETag replay across service-mode syncs "+ "(requires a connector that supports ETag replay; costs one c1z of local disk)"), WithDefaultValue(false), WithPersistent(true), + WithHidden(true), WithExportTarget(ExportTargetNone)) LambdaServerClientIDField = StringField("lambda-client-id", WithRequired(true), WithDescription("The oauth client id to use with the configuration endpoint"), @@ -431,6 +445,7 @@ var DefaultFields = append([]SchemaField{ skipEntitlementsAndGrants, skipGrants, externalResourceC1ZField, + PreviousSyncC1ZField, externalResourceEntitlementIdFilter, KeepPreviousSyncC1ZField, diffSyncsField, diff --git a/pkg/field/fields.go b/pkg/field/fields.go index 1107861c1..738d1ca63 100644 --- a/pkg/field/fields.go +++ b/pkg/field/fields.go @@ -126,6 +126,34 @@ func (s SchemaField) ExportAs(et ExportTarget) SchemaField { return c } +// WithConnectorDefault returns a copy of a shared/default SDK field carrying +// a connector-specific default value. Include the copy in the connector's +// field.Configuration Fields — DefineConfiguration replaces the SDK's copy +// with it (same re-export mechanism as ExportAs), so --help, flag parsing, +// and exported config schemas all reflect the connector's default. +// +// Value-resolution precedence is unchanged and sentinel-free: an explicit +// flag beats the environment beats the config file beats this default (the +// default lives on the flag itself, so a user-supplied zero value remains +// distinguishable from "unset"). +// +// Example — a connector whose sync fans out well declares its own worker +// default without hiding the shared flag's semantics: +// +// field.NewConfiguration([]field.SchemaField{ +// field.WithConnectorDefault(field.WorkerCountField, 16), +// ... +// }) +// +// The value's type must match the field's variant (int for IntField, etc.); +// a mismatch fails loudly at startup when the flag is registered. +func WithConnectorDefault[T SchemaTypes](s SchemaField, defaultValue T) SchemaField { + c := s + c.DefaultValue = defaultValue + c.WasReExported = true + return c +} + // Go doesn't allow generic methods on a non-generic struct. func ValidateField[T SchemaTypes](s *SchemaField, value T) (bool, error) { return s.validate(value) diff --git a/pkg/lambda/grpc/continuation_wire_test.go b/pkg/lambda/grpc/continuation_wire_test.go new file mode 100644 index 000000000..e1a7754ad --- /dev/null +++ b/pkg/lambda/grpc/continuation_wire_test.go @@ -0,0 +1,144 @@ +package grpc + +// Wire round-trip pins for the source-cache lookup continuation +// annotations (SourceCacheLookupOffer / Ask / Answers). The protocol rides +// request and response annotations through this transport, so both +// encodings must preserve them for peers that know the types: +// +// - v2 wire frame: lossless binary proto (covered here and by the +// composed lambda-stack e2e in pkg/sync); +// - legacy protojson: known types survive; the recursive annotation +// filter only prunes types missing from the registry — which is the +// designed degradation for old peers (offer stripped → connector +// never defers → cold sync), not a data-integrity risk. + +import ( + "encoding/json" + "testing" + + "github.com/stretchr/testify/require" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/types/known/anypb" + + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + pbtransport "github.com/conductorone/baton-sdk/pb/c1/transport/v1" + "github.com/conductorone/baton-sdk/pkg/annotations" +) + +func continuationRequestPayload(t *testing.T) []byte { + t.Helper() + annos := annotations.New(&v2.SourceCacheLookupOffer{}) + annos.Update(v2.SourceCacheLookupAnswers_builder{ + Answers: []*v2.SourceCacheLookupAnswers_Answer{ + v2.SourceCacheLookupAnswers_Answer_builder{ + RowKind: "grants", + ScopeHash: "groups/g1/members", + Found: true, + Etag: `W/"abc123"`, + }.Build(), + v2.SourceCacheLookupAnswers_Answer_builder{ + RowKind: "grants", + ScopeHash: "groups/g2/members", + Found: false, + }.Build(), + }, + }.Build()) + inner := v2.GrantsServiceListGrantsRequest_builder{ + Resource: v2.Resource_builder{ + Id: v2.ResourceId_builder{ResourceType: "group", Resource: "g1"}.Build(), + }.Build(), + Annotations: annos, + }.Build() + + req, err := NewRequest("/c1.connector.v2.GrantsService/ListGrants", inner, metadata.MD{}) + require.NoError(t, err) + payload, err := json.Marshal(req) + require.NoError(t, err) + return payload +} + +func assertContinuationRequest(t *testing.T, req *Request) { + t.Helper() + inner := &v2.GrantsServiceListGrantsRequest{} + require.NoError(t, req.UnmarshalRequest(inner)) + annos := annotations.Annotations(inner.GetAnnotations()) + require.True(t, annos.Contains(&v2.SourceCacheLookupOffer{}), "offer must survive the wire") + answers := &v2.SourceCacheLookupAnswers{} + ok, err := annos.Pick(answers) + require.NoError(t, err) + require.True(t, ok, "answers must survive the wire") + require.Len(t, answers.GetAnswers(), 2) + require.True(t, answers.GetAnswers()[0].GetFound()) + require.Equal(t, `W/"abc123"`, answers.GetAnswers()[0].GetEtag()) + require.False(t, answers.GetAnswers()[1].GetFound(), "explicit not-found must survive (distinct from absent)") +} + +func TestContinuationAnnotations_RequestRoundTrip_V2Frame(t *testing.T) { + payload := continuationRequestPayload(t) + + decoded := &Request{} + require.NoError(t, json.Unmarshal(payload, decoded)) + require.True(t, decoded.wireV2, "dual-encoded payload must decode via the v2 frame") + assertContinuationRequest(t, decoded) +} + +func TestContinuationAnnotations_RequestRoundTrip_LegacyJSON(t *testing.T) { + payload := continuationRequestPayload(t) + + // Strip the v2 frame fields, forcing the legacy protojson path (an + // old-SDK worker's encoding, or the frame-dropped oversize fallback). + var obj map[string]json.RawMessage + require.NoError(t, json.Unmarshal(payload, &obj)) + delete(obj, "v") + delete(obj, "frame") + legacy, err := json.Marshal(obj) + require.NoError(t, err) + + decoded := &Request{} + require.NoError(t, json.Unmarshal(legacy, decoded)) + require.False(t, decoded.wireV2) + // Types are registered in this process: the annotation filter keeps them. + assertContinuationRequest(t, decoded) +} + +func TestContinuationAnnotations_AskResponseRoundTrip(t *testing.T) { + inner := v2.GrantsServiceListGrantsResponse_builder{ + Annotations: annotations.New(v2.SourceCacheLookupAsk_builder{ + Queries: []*v2.SourceCacheLookupAsk_Query{ + v2.SourceCacheLookupAsk_Query_builder{RowKind: "grants", ScopeHash: "groups/g1/members"}.Build(), + }, + }.Build()), + }.Build() + + // Response path: frame encoding is selected when the request carried a + // frame (wireV2), legacy protojson otherwise. Pin both. + for _, wireV2 := range []bool{true, false} { + anyResp, err := anypb.New(inner) + require.NoError(t, err) + anyStatus, err := anypb.New(status.New(codes.OK, "OK").Proto()) + require.NoError(t, err) + resp := &Response{ + msg: pbtransport.Response_builder{ + Resp: anyResp, + Status: anyStatus, + }.Build(), + wireV2: wireV2, + } + payload, err := json.Marshal(resp) + require.NoError(t, err) + + decoded := &Response{} + require.NoError(t, json.Unmarshal(payload, decoded)) + out := &v2.GrantsServiceListGrantsResponse{} + require.NoError(t, decoded.UnmarshalResponse(out)) + ask := &v2.SourceCacheLookupAsk{} + annos := annotations.Annotations(out.GetAnnotations()) + ok, err := annos.Pick(ask) + require.NoError(t, err) + require.True(t, ok, "ask must survive the wire (wireV2=%v)", wireV2) + require.Len(t, ask.GetQueries(), 1) + require.Equal(t, "groups/g1/members", ask.GetQueries()[0].GetScopeHash()) + } +} diff --git a/pkg/lambda/grpc/wire.go b/pkg/lambda/grpc/wire.go index 8be74902f..78160eaa0 100644 --- a/pkg/lambda/grpc/wire.go +++ b/pkg/lambda/grpc/wire.go @@ -81,8 +81,9 @@ func spliceWireFrame(legacy []byte, msg proto.Message) ([]byte, error) { } var buf bytes.Buffer buf.Grow(len(legacy) + len(suffix)) - buf.Write(legacy[:len(legacy)-1]) - buf.WriteByte(',') - buf.Write(suffix[1:]) + // bytes.Buffer writes are documented to never return an error. + _, _ = buf.Write(legacy[:len(legacy)-1]) + _ = buf.WriteByte(',') + _, _ = buf.Write(suffix[1:]) return buf.Bytes(), nil } diff --git a/pkg/sourcecache/context.go b/pkg/sourcecache/context.go new file mode 100644 index 000000000..266dd5c04 --- /dev/null +++ b/pkg/sourcecache/context.go @@ -0,0 +1,20 @@ +package sourcecache + +import "context" + +type scopeContextKey struct{} + +// WithScope returns a context carrying the source-cache scope hash for +// rows written under it. The syncer wraps a page's store writes in this +// context when the page carried a SourceCacheScope annotation; the Pebble +// write path stamps the record's source_scope_hash from it. +func WithScope(ctx context.Context, scopeHash string) context.Context { + return context.WithValue(ctx, scopeContextKey{}, scopeHash) +} + +// ScopeFromContext returns the scope hash set by WithScope, or "" when +// the context carries none (the common, unstamped case). +func ScopeFromContext(ctx context.Context) string { + s, _ := ctx.Value(scopeContextKey{}).(string) + return s +} diff --git a/pkg/sourcecache/continuation.go b/pkg/sourcecache/continuation.go new file mode 100644 index 000000000..ea647bc8a --- /dev/null +++ b/pkg/sourcecache/continuation.go @@ -0,0 +1,285 @@ +package sourcecache + +// Lookup continuation (ask/answer): the lookup transport for connector +// runtimes that cannot call back to the syncer mid-request (single-shot +// request/response tunnels, e.g. gRPC-over-Lambda). The connector's first +// execution of a page ("phase 1") records the scopes it needs and fails +// with ErrLookupDeferred; the SDK converts that into a +// SourceCacheLookupAsk response; the syncer resolves the queries against +// its local previous-sync store and re-invokes the same request with +// SourceCacheLookupAnswers attached ("phase 2"), where the same connector +// code gets real answers. See docs/tasks/source-cache-lambda-lookup.md. + +import ( + "context" + "errors" + "fmt" + "sync" + + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" +) + +// ErrLookupDeferred is returned by a deferring Lookup (phase 1 of the +// ask/answer continuation) when the answer is not yet available. The SDK +// intercepts it and answers the RPC with a SourceCacheLookupAsk. +// +// Propagation contract for connectors: wrap with %w if you must, NEVER +// swallow. The SDK matches with errors.Is, so idiomatic wrapping +// (fmt.Errorf("listing members: %w", err)) is fine. A connector that +// logs-and-continues past this error re-asks for scopes it already +// "handled" and turns every warm sync into a hard failure at the bounce +// cap. +var ErrLookupDeferred = errors.New("source cache lookup deferred: answer arrives on re-invoke") + +// Query identifies one scope to resolve against the previous sync. +type Query struct { + RowKind RowKind + ScopeHash string +} + +// Answer resolves one Query. Found=false means the previous sync has no +// entry for the scope: fetch fresh. (Distinct from a query that got no +// Answer at all, which means unresolved: ask again.) +type Answer struct { + Query + Found bool + ETag string +} + +// BatchLookup is optionally implemented by Lookup implementations that can +// resolve many scopes in one round trip. Connectors should not type-assert +// for it directly; call LookupMany, which falls back to per-query lookups. +type BatchLookup interface { + LookupPreviousSourceCacheMany(ctx context.Context, queries []Query) ([]Answer, error) +} + +// LookupMany resolves a batch of queries through lookup, using one round +// trip when the implementation supports it (BatchLookup) and a loop of +// single lookups otherwise. This is the topology-uniform batch API: +// in-process and subprocess lookups loop over local/loopback point-reads; +// a deferring lookup collects the whole batch into ONE ask. +// +// The returned answers are exact and complete for the queried set — one +// Answer per Query, order preserved, explicit Found per entry. (A +// deferring lookup returns ErrLookupDeferred instead of answers; phase 2 +// then answers the same calls. A found answer whose etag does not fit the +// transport size budget arrives as not-found — the scope goes cold, which +// is correct, just slower — never as a silent omission.) +func LookupMany(ctx context.Context, lookup Lookup, queries []Query) ([]Answer, error) { + if bl, ok := lookup.(BatchLookup); ok { + return bl.LookupPreviousSourceCacheMany(ctx, queries) + } + answers := make([]Answer, 0, len(queries)) + for _, q := range queries { + entry, found, err := lookup.LookupPreviousSourceCache(ctx, q.RowKind, q.ScopeHash) + if err != nil { + return nil, err + } + a := Answer{Query: q, Found: found} + if found { + a.ETag = entry.ETag + } + answers = append(answers, a) + } + return answers, nil +} + +// ContinuationLookup is the per-request Lookup installed for the +// ask/answer continuation. It serves lookups from the answers delivered on +// the request (phase 2) and defers everything else by recording the query +// and returning ErrLookupDeferred (phase 1, or a phase 2 that queries +// scopes it did not ask for in phase 1 — allowed but wasteful). +// +// It is constructed per RPC by the SDK; connectors only ever see it as +// SyncOpAttrs.SourceCache. +type ContinuationLookup struct { + mu sync.Mutex + answers map[Query]Answer + asked []Query + askedSet map[Query]struct{} +} + +var ( + _ Lookup = (*ContinuationLookup)(nil) + _ BatchLookup = (*ContinuationLookup)(nil) +) + +// NewContinuationLookup builds a ContinuationLookup pre-loaded with the +// request's answers (nil/empty on phase 1). +func NewContinuationLookup(answers []Answer) *ContinuationLookup { + m := make(map[Query]Answer, len(answers)) + for _, a := range answers { + m[a.Query] = a + } + return &ContinuationLookup{ + answers: m, + askedSet: map[Query]struct{}{}, + } +} + +func (c *ContinuationLookup) LookupPreviousSourceCache(_ context.Context, rowKind RowKind, scopeHash string) (Entry, bool, error) { + q := Query{RowKind: rowKind, ScopeHash: scopeHash} + c.mu.Lock() + defer c.mu.Unlock() + if a, ok := c.answers[q]; ok { + if !a.Found { + return Entry{}, false, nil + } + return Entry{ETag: a.ETag}, true, nil + } + c.recordLocked(q) + return Entry{}, false, fmt.Errorf("lookup %s/%s: %w", rowKind, scopeHash, ErrLookupDeferred) +} + +func (c *ContinuationLookup) LookupPreviousSourceCacheMany(_ context.Context, queries []Query) ([]Answer, error) { + c.mu.Lock() + defer c.mu.Unlock() + answers := make([]Answer, 0, len(queries)) + missing := 0 + for _, q := range queries { + a, ok := c.answers[q] + if !ok { + c.recordLocked(q) + missing++ + continue + } + answers = append(answers, a) + } + if missing > 0 { + // Defer the whole batch: phase 2 re-runs the same call with the + // full answer set (already-answered queries remain answered on the + // re-invoked request). + return nil, fmt.Errorf("batch lookup: %d of %d queries unresolved: %w", missing, len(queries), ErrLookupDeferred) + } + return answers, nil +} + +func (c *ContinuationLookup) recordLocked(q Query) { + if _, seen := c.askedSet[q]; seen { + return + } + c.askedSet[q] = struct{}{} + c.asked = append(c.asked, q) +} + +// Asked returns the queries recorded by deferred lookups, deduplicated, in +// first-ask order. Empty means no lookup deferred and the handler's result +// stands. +func (c *ContinuationLookup) Asked() []Query { + c.mu.Lock() + defer c.mu.Unlock() + out := make([]Query, len(c.asked)) + copy(out, c.asked) + return out +} + +// --- proto conversions (shared by connectorbuilder and the syncer) ------ + +// AskProto builds the SourceCacheLookupAsk annotation for a set of +// deferred queries. +func AskProto(queries []Query) *v2.SourceCacheLookupAsk { + qs := make([]*v2.SourceCacheLookupAsk_Query, 0, len(queries)) + for _, q := range queries { + qs = append(qs, v2.SourceCacheLookupAsk_Query_builder{ + RowKind: string(q.RowKind), + ScopeHash: q.ScopeHash, + }.Build()) + } + return v2.SourceCacheLookupAsk_builder{Queries: qs}.Build() +} + +// maxQueriesPerAsk mirrors SourceCacheLookupAsk.queries.max_items. Asks +// arrive in connector response annotations; annotations.Pick unmarshals the +// Any but does not invoke generated protobuf validation, so this boundary +// enforces the resource cap explicitly. +const maxQueriesPerAsk = 4096 + +// MaxLookupBouncesPerRequest bounds consecutive asks for one request +// (same page token; only the answers annotation differs between +// re-invokes). Owned here so the answer-set cap below stays coherent +// with the syncer's bounce loop, which enforces it: answers accumulate +// across bounces, so the most answers one request can legally carry is +// this many asks' worth of queries. +const MaxLookupBouncesPerRequest = 4 + +// QueriesFromProto extracts and validates the queries of an ask. +func QueriesFromProto(ask *v2.SourceCacheLookupAsk) ([]Query, error) { + if len(ask.GetQueries()) == 0 { + // Without this an empty ask would die on the syncer's no-progress + // guard with a misleading "re-asked already-answered scopes" + // message; name the actual bug. + return nil, fmt.Errorf("source cache lookup ask: no queries (a connector must not ask without at least one unresolved scope)") + } + if len(ask.GetQueries()) > maxQueriesPerAsk { + return nil, fmt.Errorf("source cache lookup ask: %d queries (max %d)", len(ask.GetQueries()), maxQueriesPerAsk) + } + out := make([]Query, 0, len(ask.GetQueries())) + for _, q := range ask.GetQueries() { + kind := RowKind(q.GetRowKind()) + if err := ValidateRowKind(kind); err != nil { + return nil, err + } + if err := ValidateScopeHash(q.GetScopeHash()); err != nil { + return nil, err + } + out = append(out, Query{RowKind: kind, ScopeHash: q.GetScopeHash()}) + } + return out, nil +} + +// AnswersProto builds the SourceCacheLookupAnswers annotation. +func AnswersProto(answers []Answer) *v2.SourceCacheLookupAnswers { + as := make([]*v2.SourceCacheLookupAnswers_Answer, 0, len(answers)) + for _, a := range answers { + as = append(as, v2.SourceCacheLookupAnswers_Answer_builder{ + RowKind: string(a.RowKind), + ScopeHash: a.ScopeHash, + Found: a.Found, + Etag: a.ETag, + }.Build()) + } + return v2.SourceCacheLookupAnswers_builder{Answers: as}.Build() +} + +// Wire caps for answer sets. Answers come from the SDK (a trusted +// parent), but the same validation discipline as QueriesFromProto keeps a +// corrupted or hostile transport from smuggling malformed entries into +// lookup results. +// +// maxAnswersPerMessage is NOT one ask's max_items: answers accumulate +// across bounces (every re-invoke carries the union of all resolved +// queries), so the coherent bound is the most a compliant exchange can +// legally produce — MaxLookupBouncesPerRequest maximal asks. Capping at +// one ask's size would hard-fail a legal 4096-scope first ask followed by +// a single late scope on a later bounce. +const ( + maxAnswersPerMessage = maxQueriesPerAsk * MaxLookupBouncesPerRequest + maxAnswerETagLen = 65536 +) + +// AnswersFromProto extracts and validates the answers delivered on a +// request. +func AnswersFromProto(msg *v2.SourceCacheLookupAnswers) ([]Answer, error) { + if len(msg.GetAnswers()) > maxAnswersPerMessage { + return nil, fmt.Errorf("source cache lookup answers: %d entries (max %d)", len(msg.GetAnswers()), maxAnswersPerMessage) + } + out := make([]Answer, 0, len(msg.GetAnswers())) + for _, a := range msg.GetAnswers() { + kind := RowKind(a.GetRowKind()) + if err := ValidateRowKind(kind); err != nil { + return nil, fmt.Errorf("source cache lookup answers: %w", err) + } + if err := ValidateScopeHash(a.GetScopeHash()); err != nil { + return nil, fmt.Errorf("source cache lookup answers: %w", err) + } + if len(a.GetEtag()) > maxAnswerETagLen { + return nil, fmt.Errorf("source cache lookup answers: etag for scope %q is %d bytes (max %d)", a.GetScopeHash(), len(a.GetEtag()), maxAnswerETagLen) + } + out = append(out, Answer{ + Query: Query{RowKind: kind, ScopeHash: a.GetScopeHash()}, + Found: a.GetFound(), + ETag: a.GetEtag(), + }) + } + return out, nil +} diff --git a/pkg/sourcecache/grpc_lookup.go b/pkg/sourcecache/grpc_lookup.go new file mode 100644 index 000000000..4d81eba9f --- /dev/null +++ b/pkg/sourcecache/grpc_lookup.go @@ -0,0 +1,91 @@ +package sourcecache + +import ( + "context" + "sync/atomic" + "time" + + "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" + "go.uber.org/zap" + + v1 "github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1" +) + +// GRPCLookup is the connector-side Lookup implementation that talks to +// BatonSourceCacheService on the parent SDK. +// +// This is deliberately not routed through the session store: session data +// passes through the connector's local MemorySessionCache (otter), which +// would apply generic TTL/eviction policies to sync-scoped validator state. +// The dedicated service keeps the path uncached and the message shape +// explicit. +// +// The parent has exactly one active Lookup registered at a time (set per +// sync via SetSourceCache on the server), so the wire format carries no +// sync_id; routing is implicit. +// +// Per the Lookup contract, an RPC failure degrades to a miss rather than +// failing the connector's list call: a miss is always safe — the +// connector fetches fresh — while the error would ride back through the +// connector's handler wrappings, where its status code rarely survives to +// the syncer's retryer, hard-failing the sync on a transient loopback +// blip. Failures are logged at most once per minute with a running count +// (this object lives for the whole subprocess run, so a once-ever log +// would make a persistently broken parent path invisible after the first +// scope), and not at all when the caller's context is already done — a +// teardown-time Canceled is noise, and logging it would eat the rate +// window a real failure needs. Validation failures stay loud: a malformed +// row kind or scope hash is a connector bug, and degrading it to a miss +// would let the connector silently cold-fetch forever with no signal. +type GRPCLookup struct { + client v1.BatonSourceCacheServiceClient + // failures counts degraded lookups for the log line; lastLogNano is + // the unix-nano timestamp of the last emitted warning. + failures atomic.Int64 + lastLogNano atomic.Int64 +} + +// lookupFailureLogInterval rate-limits the degraded-to-miss warning. +const lookupFailureLogInterval = time.Minute + +// NewGRPCLookup returns a Lookup backed by the given client. A nil client +// yields NoopLookup so callers can configure the client optionally without +// nil checks at every call site. +func NewGRPCLookup(client v1.BatonSourceCacheServiceClient) Lookup { + if client == nil { + return NoopLookup{} + } + return &GRPCLookup{client: client} +} + +func (g *GRPCLookup) LookupPreviousSourceCache(ctx context.Context, rowKind RowKind, scopeHash string) (Entry, bool, error) { + if err := ValidateRowKind(rowKind); err != nil { + return Entry{}, false, err + } + if err := ValidateScopeHash(scopeHash); err != nil { + return Entry{}, false, err + } + resp, err := g.client.Lookup(ctx, v1.LookupRequest_builder{ + RowKind: string(rowKind), + ScopeHash: scopeHash, + }.Build()) + if err != nil { + count := g.failures.Add(1) + if ctx.Err() == nil { + now := time.Now().UnixNano() + last := g.lastLogNano.Load() + if now-last >= int64(lookupFailureLogInterval) && g.lastLogNano.CompareAndSwap(last, now) { + ctxzap.Extract(ctx).Warn("source cache rpc lookup failed; treating as miss", + zap.Error(err), + zap.Int64("failures_since_start", count)) + } + } + // Intentional nil error: a failed lookup degrades to a miss + // (connector fetches fresh) rather than failing the connector call. + return Entry{}, false, nil + } + if !resp.GetFound() { + return Entry{}, false, nil + } + return Entry{ETag: resp.GetEtag()}, true, nil +} diff --git a/pkg/sourcecache/grpc_lookup_test.go b/pkg/sourcecache/grpc_lookup_test.go new file mode 100644 index 000000000..3d8bcfcd1 --- /dev/null +++ b/pkg/sourcecache/grpc_lookup_test.go @@ -0,0 +1,145 @@ +package sourcecache + +import ( + "context" + "errors" + "sync" + "testing" + + "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" + "github.com/stretchr/testify/require" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" + "google.golang.org/grpc" + + v1 "github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1" +) + +type fakeLookupClient struct { + resp *v1.LookupResponse + err error +} + +func (f *fakeLookupClient) Lookup(context.Context, *v1.LookupRequest, ...grpc.CallOption) (*v1.LookupResponse, error) { + return f.resp, f.err +} + +// TestGRPCLookupDegradesRPCErrorToMiss pins the Lookup contract for the +// subprocess topology: a transport failure to the parent degrades to a +// miss (the connector fetches fresh — always safe) instead of failing the +// connector's list call, whose error would rarely keep a retryable status +// code by the time it reaches the syncer's retryer. +func TestGRPCLookupDegradesRPCErrorToMiss(t *testing.T) { + ctx := context.Background() + lookup := NewGRPCLookup(&fakeLookupClient{err: errors.New("connection refused")}) + + scope := HashScope("https://example.test/teams/1/members?page=1") + _, found, err := lookup.LookupPreviousSourceCache(ctx, RowKindGrants, scope) + require.NoError(t, err, "an RPC failure must degrade to a miss, not fail the connector call") + require.False(t, found) +} + +// capturedLog is one recorded zap entry (message + flattened fields). +// zaptest/observer is not vendored, so recordingCore stands in for it +// (same pattern as pkg/c1zsanitize). +type capturedLog struct { + msg string + fields map[string]any +} + +type recordingCore struct { + mu *sync.Mutex + logs *[]capturedLog +} + +func (c recordingCore) Enabled(zapcore.Level) bool { return true } +func (c recordingCore) With([]zapcore.Field) zapcore.Core { return c } +func (c recordingCore) Check(e zapcore.Entry, ce *zapcore.CheckedEntry) *zapcore.CheckedEntry { + return ce.AddCore(e, c) +} +func (c recordingCore) Sync() error { return nil } +func (c recordingCore) Write(e zapcore.Entry, fs []zapcore.Field) error { + enc := zapcore.NewMapObjectEncoder() + for _, f := range fs { + f.AddTo(enc) + } + c.mu.Lock() + *c.logs = append(*c.logs, capturedLog{msg: e.Message, fields: enc.Fields}) + c.mu.Unlock() + return nil +} + +func newRecordingContext() (context.Context, *[]capturedLog) { + logs := &[]capturedLog{} + logger := zap.New(recordingCore{mu: &sync.Mutex{}, logs: logs}) + return ctxzap.ToContext(context.Background(), logger), logs +} + +// TestGRPCLookupFailureLogRateLimited pins the failure-log policy: the +// degraded-to-miss warning is rate-limited (not once-ever — this object +// lives for the whole subprocess run), carries a running failure count, +// and is skipped entirely when the caller's context is already done so +// teardown-time cancellation can't consume the rate window. +func TestGRPCLookupFailureLogRateLimited(t *testing.T) { + ctx, logs := newRecordingContext() + lookup := NewGRPCLookup(&fakeLookupClient{err: errors.New("connection refused")}) + scope := HashScope("https://example.test/teams/1/members?page=1") + + // A failure on an already-canceled context degrades silently: the + // connector call is dying anyway, and logging would burn the rate + // window a real failure needs. + canceledCtx, cancel := context.WithCancel(ctx) + cancel() + _, found, err := lookup.LookupPreviousSourceCache(canceledCtx, RowKindGrants, scope) + require.NoError(t, err) + require.False(t, found) + require.Empty(t, *logs, "a failure on a done context must not log") + + // The first live-context failure logs immediately, with the running + // count including the silent canceled-context failure above. + _, found, err = lookup.LookupPreviousSourceCache(ctx, RowKindGrants, scope) + require.NoError(t, err) + require.False(t, found) + require.Len(t, *logs, 1) + require.Equal(t, "source cache rpc lookup failed; treating as miss", (*logs)[0].msg) + require.EqualValues(t, 2, (*logs)[0].fields["failures_since_start"]) + + // Immediate subsequent failures still degrade to misses but stay + // within the rate window and do not log again. + for range 10 { + _, found, err = lookup.LookupPreviousSourceCache(ctx, RowKindGrants, scope) + require.NoError(t, err) + require.False(t, found) + } + require.Len(t, *logs, 1, "failures inside the rate window must not log") +} + +// TestGRPCLookupValidationStaysLoud pins the other half: malformed +// arguments are connector bugs and must fail, never degrade — a miss here +// would let a broken connector silently cold-fetch forever. +func TestGRPCLookupValidationStaysLoud(t *testing.T) { + ctx := context.Background() + lookup := NewGRPCLookup(&fakeLookupClient{resp: v1.LookupResponse_builder{Found: true, Etag: "e"}.Build()}) + + _, _, err := lookup.LookupPreviousSourceCache(ctx, RowKind("bogus"), HashScope("s")) + require.Error(t, err, "invalid row kind must fail loudly") + + _, _, err = lookup.LookupPreviousSourceCache(ctx, RowKindGrants, "") + require.Error(t, err, "empty scope hash must fail loudly") +} + +func TestGRPCLookupHitAndMiss(t *testing.T) { + ctx := context.Background() + scope := HashScope("https://example.test/teams/1/members?page=1") + + hit := NewGRPCLookup(&fakeLookupClient{resp: v1.LookupResponse_builder{Found: true, Etag: `W/"etag-1"`}.Build()}) + entry, found, err := hit.LookupPreviousSourceCache(ctx, RowKindGrants, scope) + require.NoError(t, err) + require.True(t, found) + require.Equal(t, `W/"etag-1"`, entry.ETag) + + miss := NewGRPCLookup(&fakeLookupClient{resp: v1.LookupResponse_builder{Found: false}.Build()}) + _, found, err = miss.LookupPreviousSourceCache(ctx, RowKindGrants, scope) + require.NoError(t, err) + require.False(t, found) +} diff --git a/pkg/sourcecache/grpc_server.go b/pkg/sourcecache/grpc_server.go new file mode 100644 index 000000000..088a5fbeb --- /dev/null +++ b/pkg/sourcecache/grpc_server.go @@ -0,0 +1,83 @@ +package sourcecache + +import ( + "context" + "fmt" + "sync/atomic" + + "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" + + v1 "github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1" +) + +// GRPCServer is the parent-side BatonSourceCacheService implementation. +// +// The parent SDK holds a single GRPCServer for the lifetime of the connector +// subprocess and swaps the active Lookup via SetSourceCache as syncs come +// and go. The syncer installs a real lookup once it has resolved a usable +// previous sync, and clears it when the sync ends so a late RPC can't serve +// from a store the syncer no longer owns. +// +// Until the first SetSourceCache call the server answers every lookup with +// found=false, which the connector treats as "no previous sync" and falls +// back to an unconditional fetch. +type GRPCServer struct { + v1.UnimplementedBatonSourceCacheServiceServer + lookup atomic.Pointer[Lookup] +} + +var _ v1.BatonSourceCacheServiceServer = (*GRPCServer)(nil) +var _ SetLookup = (*GRPCServer)(nil) + +// NewGRPCServer returns a GRPCServer with no active Lookup registered. +func NewGRPCServer() *GRPCServer { + return &GRPCServer{} +} + +// SetSourceCache replaces the active lookup. Safe to call concurrently with +// in-flight RPCs: existing RPCs continue against the value they read at +// entry; new RPCs see the swapped value. +// +// The slot is still SINGLE — the wire carries no sync id — so two live +// syncs installing lookups cross-wire each other (one sync's connector +// resolves scopes against the other's previous store) and the first to +// finish blinds the survivor when it clears. That topology isn't +// supported; an install landing on an occupied slot warns loudly so it is +// visible instead of silently wrong. +func (s *GRPCServer) SetSourceCache(ctx context.Context, lookup Lookup) { + if lookup == nil { + s.lookup.Store(nil) + return + } + if prev := s.lookup.Swap(&lookup); prev != nil { + ctxzap.Extract(ctx).Warn("source cache: installing a lookup over an existing one; " + + "concurrent syncs against one connector share a single lookup slot and will cross-wire or blind each other") + } +} + +func (s *GRPCServer) Lookup(ctx context.Context, req *v1.LookupRequest) (*v1.LookupResponse, error) { + rowKind := RowKind(req.GetRowKind()) + if err := ValidateRowKind(rowKind); err != nil { + return nil, err + } + scopeHash := req.GetScopeHash() + if err := ValidateScopeHash(scopeHash); err != nil { + return nil, err + } + + lookupPtr := s.lookup.Load() + if lookupPtr == nil { + return v1.LookupResponse_builder{Found: false}.Build(), nil + } + entry, found, err := (*lookupPtr).LookupPreviousSourceCache(ctx, rowKind, scopeHash) + if err != nil { + return nil, fmt.Errorf("source cache lookup: %w", err) + } + if !found { + return v1.LookupResponse_builder{Found: false}.Build(), nil + } + return v1.LookupResponse_builder{ + Found: true, + Etag: entry.ETag, + }.Build(), nil +} diff --git a/pkg/sourcecache/sourcecache.go b/pkg/sourcecache/sourcecache.go new file mode 100644 index 000000000..1892cda35 --- /dev/null +++ b/pkg/sourcecache/sourcecache.go @@ -0,0 +1,140 @@ +// Package sourcecache defines the connector-facing surface of source-cache +// replay (see proto/c1/connector/v2/annotation_source_cache.proto). +// +// A connector that can cheaply revalidate upstream data — HTTP conditional +// requests (GitHub), delta queries (Microsoft Graph) — opts in by attaching +// SourceCacheCapability MODE_READ_WRITE to its Validate response. During a +// sync it looks up the previous validator for a scope via the Lookup the SDK +// provides on SyncOpAttrs, revalidates upstream, and either emits fresh rows +// tagged with SourceCacheScope or asks the SDK to replay the previous rows +// with SourceCacheReplay. +// +// The connector owns scope computation; the SDK only keys storage by the +// connector-supplied scope hash. The validator (etag, delta token) is opaque +// to the SDK. +// +// Invariant that keeps replay safe: a connector must only emit +// SourceCacheReplay for a scope whose validator it received from THIS sync's +// Lookup. The lookup need not happen in the same call that emits the +// replay: a planning call may batch-resolve many scopes and pass the +// verdicts to sibling cursors through SpawnCursors page tokens — that +// satisfies the invariant, because the validator still originates from the +// consuming sync. What's forbidden is a validator that outlives a sync +// (connector-side caches, config, upstream echoes). When source cache is +// disabled or degraded (no capability, no usable previous sync, unsupported +// storage engine) the SDK installs NoopLookup, every lookup misses, and a +// well-behaved connector naturally falls back to full fetch. +// +// Replay equivalence: a cached sync must reproduce what a full resync +// would produce. Replayed rows are verbatim copies of the previous sync's +// rows with one deliberate exception — expander-written Sources on direct +// grants (classified by a self-source entry, mirroring RollbackExpansion) +// are stripped at copy time so the current sync's expansion recomputes +// them from true state; re-expansion only adds contributions, so carrying +// them verbatim would immortalize contributions removed upstream. +// Connector-set Sources (no self-source) are public connector data and +// survive replay byte-for-byte. +package sourcecache + +import ( + "context" + "crypto/sha256" + "encoding/hex" + "fmt" + "time" +) + +// RowKind partitions source-cache scopes by the row type they produce. +// It doubles as the row_kind value stored in SourceCacheEntryRecord. +type RowKind string + +const ( + RowKindResources RowKind = "resources" + RowKindEntitlements RowKind = "entitlements" + RowKindGrants RowKind = "grants" +) + +// Valid reports whether k is one of the defined row kinds. +func (k RowKind) Valid() bool { + switch k { + case RowKindResources, RowKindEntitlements, RowKindGrants: + return true + } + return false +} + +// ValidateRowKind returns an error if rowKind is not one of the known +// RowKind* constants. +func ValidateRowKind(rowKind RowKind) error { + if !rowKind.Valid() { + return fmt.Errorf("invalid source cache row kind: %q", rowKind) + } + return nil +} + +// maxScopeHashLen bounds scope identifiers on the wire and in storage +// keys. Deliberately generous: the shape is a connector convention +// (HashScope produces 64 hex chars) and is not enforced beyond +// non-emptiness and this cap while the model is being proven out against +// real providers. +const maxScopeHashLen = 256 + +// ValidateScopeHash returns an error when scopeHash is empty or +// unreasonably long. Connectors conventionally use HashScope, but any +// stable identifier is accepted. +func ValidateScopeHash(scopeHash string) error { + if scopeHash == "" { + return fmt.Errorf("source cache scope hash is required") + } + if len(scopeHash) > maxScopeHashLen { + return fmt.Errorf("source cache scope hash too long: %d bytes (max %d)", len(scopeHash), maxScopeHashLen) + } + return nil +} + +// Entry is a previous sync's persisted validator for one scope. +type Entry struct { + // ETag is the opaque upstream validator: a literal HTTP ETag, a delta + // token, etc. Never interpreted by the SDK. + ETag string + + // DiscoveredAt is when the entry was written. + DiscoveredAt time.Time +} + +// Lookup resolves a scope's previous-sync validator. The SDK provides an +// implementation on SyncOpAttrs; connectors call it before revalidating +// upstream. +type Lookup interface { + // LookupPreviousSourceCache returns the previous sync's entry for + // (rowKind, scopeHash). found=false means no entry: fetch fresh. + // Implementations must treat internal read errors that leave fresh + // fetch available as misses rather than failing the connector call. + LookupPreviousSourceCache(ctx context.Context, rowKind RowKind, scopeHash string) (entry Entry, found bool, err error) +} + +// NoopLookup is the Lookup installed when source cache is disabled or +// degraded. Every lookup misses. +type NoopLookup struct{} + +var _ Lookup = NoopLookup{} + +func (NoopLookup) LookupPreviousSourceCache(context.Context, RowKind, string) (Entry, bool, error) { + return Entry{}, false, nil +} + +// SetLookup is implemented by connector clients/servers that can receive a +// source-cache lookup implementation from the sync runner. The SDK calls +// SetSourceCache(lookup) at the start of each sync and SetSourceCache(nil) +// when the sync ends so a late RPC can't read stale state. +type SetLookup interface { + SetSourceCache(ctx context.Context, lookup Lookup) +} + +// HashScope returns the lowercase-hex sha256 of a canonical scope string. +// Convenience for connectors; any stable identifier is acceptable as a +// scope hash (only non-emptiness and a length cap are enforced). +func HashScope(canonicalScope string) string { + sum := sha256.Sum256([]byte(canonicalScope)) + return hex.EncodeToString(sum[:]) +} diff --git a/pkg/sync/parallel_syncer.go b/pkg/sync/parallel_syncer.go index 387204852..3b2f59640 100644 --- a/pkg/sync/parallel_syncer.go +++ b/pkg/sync/parallel_syncer.go @@ -304,6 +304,24 @@ func (s *syncer) parallelSync( } } + if !s.dontExpandGrants && !s.state.NeedsExpansion() { + // A replay-only grants phase can arm expansion without + // processing a connector GrantExpandable row. If the + // process dies after its last grant action finishes but + // before that in-memory flag is checkpointed, resume + // reaches this action with NeedsExpansion=false even + // though the current store contains pending expansion + // rows. Reconcile from the authoritative store before + // deciding to skip; PendingExpansionPage is read-only and + // repeats the expansion loader's first page safely. + pending, _, pendingErr := s.store.Grants().PendingExpansionPage(ctx, stateAction.PageToken) + if pendingErr != nil { + return warnings, pendingErr + } + if len(pending) > 0 { + s.state.SetNeedsExpansion() + } + } if s.dontExpandGrants || !s.state.NeedsExpansion() { l.Debug("skipping grant expansion, no grants to expand") s.state.FinishAction(ctx, stateAction) diff --git a/pkg/sync/progresslog/progresslog.go b/pkg/sync/progresslog/progresslog.go index 734ccd548..51be5a361 100644 --- a/pkg/sync/progresslog/progresslog.go +++ b/pkg/sync/progresslog/progresslog.go @@ -38,6 +38,7 @@ type ProgressLog struct { lastEntitlementLog map[string]time.Time grantsProgress map[string]int lastGrantLog map[string]time.Time + grantsCountOnly map[string]bool mu sync.RWMutex l *zap.Logger maxLogFrequency time.Duration @@ -150,6 +151,7 @@ func NewProgressCounts(ctx context.Context, opts ...Option) *ProgressLog { lastEntitlementLog: make(map[string]time.Time), grantsProgress: make(map[string]int), lastGrantLog: make(map[string]time.Time), + grantsCountOnly: make(map[string]bool), l: ctxzap.Extract(ctx), maxLogFrequency: defaultMaxLogFrequency, mu: sync.RWMutex{}, @@ -256,18 +258,46 @@ func (p *ProgressLog) LogEntitlementsProgress(ctx context.Context, resourceType } } +// SetGrantsCountOnly marks a resource type's grant progress as a plain +// count with no resources-covered denominator. Used for type-scoped grant +// enumeration (v2.TypeScopedGrants), where cursors don't map 1:1 to +// resources: the per-cursor accounting would exceed the type's resource +// total and trip the "more grant resources than resources" warning for a +// perfectly healthy sync. Count-only types log synced row counts +// periodically and never compute the ratio. +func (p *ProgressLog) SetGrantsCountOnly(resourceType string) { + p.mu.Lock() + defer p.mu.Unlock() + p.grantsCountOnly[resourceType] = true +} + +// GrantsProgress returns the current grant-coverage counter for a resource +// type. For per-resource types this is "resources covered" and must never +// exceed the type's resource total — spawned sibling cursors +// (v2.SpawnCursors) don't increment it, only the origin action's chain end +// does. Exposed for tests pinning that accounting. +func (p *ProgressLog) GrantsProgress(resourceType string) int { + p.mu.RLock() + defer p.mu.RUnlock() + return p.grantsProgress[resourceType] +} + func (p *ProgressLog) LogGrantsProgress(ctx context.Context, resourceType string) { var grantsProgress, resources int var lastLogTime time.Time + var countOnly bool p.mu.RLock() grantsProgress = p.grantsProgress[resourceType] resources = p.resources[resourceType] lastLogTime = p.lastGrantLog[resourceType] + countOnly = p.grantsCountOnly[resourceType] p.mu.RUnlock() - if resources == 0 { - // if resuming sync, resource counts will be zero, so don't calculate percentage. just log every 10 seconds. + if resources == 0 || countOnly { + // Count-only: either a resumed sync (resource counts are zero) or a + // type-scoped grants type (no meaningful denominator). Log the raw + // synced count every log window; never compute a percentage. if time.Since(lastLogTime) > p.maxLogFrequency { p.l.Info("Syncing grants", zap.String("resource_type_id", resourceType), diff --git a/pkg/sync/source_cache.go b/pkg/sync/source_cache.go new file mode 100644 index 000000000..adf2ee880 --- /dev/null +++ b/pkg/sync/source_cache.go @@ -0,0 +1,537 @@ +package sync //nolint:revive,nolintlint // we can't change the package name for backwards compatibility + +import ( + "context" + "fmt" + stdsync "sync" + "time" + + "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" + "go.uber.org/zap" + + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + "github.com/conductorone/baton-sdk/pkg/annotations" + "github.com/conductorone/baton-sdk/pkg/connectorstore" + "github.com/conductorone/baton-sdk/pkg/dotc1z" + "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" + "github.com/conductorone/baton-sdk/pkg/sourcecache" +) + +// Source-cache replay, syncer side. See +// proto/c1/connector/v2/annotation_source_cache.proto for the contract. +// +// Setup degrades, replay fails loudly. Any setup problem (capability +// absent, store engine unsupported, no usable previous sync) installs the +// no-op lookup: the connector never sees a previous validator, never gets +// a conditional-request hit, and therefore never emits SourceCacheReplay +// — which is what makes it safe to treat a replay annotation arriving +// while degraded as a hard error (the connector already skipped row +// generation; there is nothing to fall back to). + +// syncerSourceCache is the per-sync source-cache state resolved by +// configureSourceCache. +// +// Write side and read side enable independently: the FIRST sync of a chain +// has no previous sync but must still stamp rows and write manifest +// entries, or the second sync would have nothing to replay. enabled covers +// the write side (capability declared + current store supports it); prev +// is non-nil only when a usable previous sync exists (read side — lookup +// hits and replay). +type syncerSourceCache struct { + enabled bool + // current is the writable output store's source-cache capability. + current dotc1z.SourceCacheStore + // prev is the previous sync's lookup/replay source (read-only). Nil + // when no usable previous sync exists; lookups then miss and replay + // annotations are hard errors. + prev dotc1z.SourceCacheStore + // prevReader is the same store as prev, typed for ReplaySourceCache. + prevReader connectorstore.Reader + // lookup is the connector-facing lookup built from prev (NoopLookup + // when prev is nil). The syncer also uses it to answer lookup asks + // from connectors on single-shot transports (the ask/answer + // continuation); both paths see identical results by construction. + lookup sourcecache.Lookup + // replayedScopes tracks scopes replayed this sync whose validator may + // arrive on a LATER page (multi-page overlay rounds), so the final + // page's manifest write is not also counted as a fresh stamp — one + // warm delta round would otherwise book as both a hit and a miss in + // the replay ratio. Only rounds that can span pages are recorded + // (overlay, or replay without an inline etag), keeping the set tiny: + // single-page 304 rounds never enter it. Pointer so copying the + // per-sync struct shares the set; guarded internally for parallel + // workers. Best-effort across resume (in-memory), stats-only. + replayedScopes *replayedScopeSet +} + +type replayedScopeSet struct { + mu stdsync.Mutex + m map[string]struct{} +} + +func (c *syncerSourceCache) markScopeReplayed(kind sourcecache.RowKind, scopeHash string) { + if c.replayedScopes == nil { + return + } + c.replayedScopes.mu.Lock() + defer c.replayedScopes.mu.Unlock() + if c.replayedScopes.m == nil { + c.replayedScopes.m = make(map[string]struct{}) + } + c.replayedScopes.m[string(kind)+"\x00"+scopeHash] = struct{}{} +} + +func (c *syncerSourceCache) scopeReplayedThisSync(kind sourcecache.RowKind, scopeHash string) bool { + if c.replayedScopes == nil { + return false + } + c.replayedScopes.mu.Lock() + defer c.replayedScopes.mu.Unlock() + _, ok := c.replayedScopes.m[string(kind)+"\x00"+scopeHash] + return ok +} + +// prevStoreLookup adapts the previous store's manifest to the +// connector-facing Lookup. Mid-sync read errors are logged once and +// treated as misses: at lookup time the connector can still fetch fresh, +// so degrading beats failing the sync. +type prevStoreLookup struct { + prev dotc1z.SourceCacheStore + logOnce *stdsync.Once +} + +var _ sourcecache.Lookup = prevStoreLookup{} + +func (p prevStoreLookup) LookupPreviousSourceCache(ctx context.Context, kind sourcecache.RowKind, scopeHash string) (sourcecache.Entry, bool, error) { + entry, found, err := p.prev.LookupSourceCacheEntry(ctx, kind, scopeHash) + if err != nil { + p.logOnce.Do(func() { + ctxzap.Extract(ctx).Warn("source cache lookup failed; treating as miss", zap.Error(err)) + }) + return sourcecache.Entry{}, false, nil //nolint:nilerr // intentional: a failed lookup degrades to a miss (connector fetches fresh) rather than failing the connector call + } + return entry, found, nil +} + +// configureSourceCache resolves per-sync source-cache state from the +// connector's Validate response and installs the connector-facing lookup. +// Called once per Sync, after Validate. +func (s *syncer) configureSourceCache(ctx context.Context, resp *v2.ConnectorServiceValidateResponse) error { + l := ctxzap.Extract(ctx) + s.sourceCache = syncerSourceCache{} + + setLookup, canSetLookup := s.connector.(sourcecache.SetLookup) + degrade := func(reason string) error { + if canSetLookup { + setLookup.SetSourceCache(ctx, sourcecache.NoopLookup{}) + } + if reason != "" { + l.Info("source cache disabled", zap.String("reason", reason)) + } + return nil + } + + capability := &v2.SourceCacheCapability{} + annos := annotations.Annotations(resp.GetAnnotations()) + ok, err := annos.Pick(capability) + if err != nil { + return fmt.Errorf("error parsing source cache capability annotation: %w", err) + } + if !ok || capability.GetMode() != v2.SourceCacheCapability_MODE_READ_WRITE { + // The common case; stay quiet. + return degrade("") + } + if s.syncType != connectorstore.SyncTypeFull { + // Replay is FULL-sync-only. A partial/targeted sync writes a + // targeted subset; replaying a whole scope could resurrect rows + // outside the targets, and the partial-sync related-resource + // fetch (ShouldFetchRelatedResources) only runs over response + // rows, so replayed grants could reference resources the partial + // store never received. Degrading installs the no-op lookup: + // every scope misses and the connector fetches fresh — + // cold-correct, and partial syncs are small by construction. + return degrade(fmt.Sprintf("source cache replay is only supported on full syncs (sync type %q)", s.syncType)) + } + current, ok := s.store.(dotc1z.SourceCacheStore) + if !ok { + return degrade("storage engine does not support source cache") + } + + // Write side enabled: rows produced under a scope get stamped and + // manifest entries get written, so this sync is usable as the NEXT + // sync's replay source even when this one has nothing to replay from. + s.sourceCache = syncerSourceCache{enabled: true, current: current, replayedScopes: &replayedScopeSet{}} + + // Read side: a usable previous sync makes lookups hit and replay legal. + var readReason string + if s.previousSyncReader == nil { + readReason = "no previous-sync c1z configured" + } else if prev, ok := s.previousSyncReader.(dotc1z.SourceCacheStore); !ok { + readReason = "previous-sync store engine does not support source cache" + } else if compacted, err := previousSyncIsCompacted(ctx, s.previousSyncReader); err != nil { + return fmt.Errorf("error reading previous sync's run metadata: %w", err) + } else if compacted { + // The explicit metadata gate, independent of the manifest's + // contents: a compacted artifact is a keep-newer merge, so no + // input sync's upstream validators describe it — a 304 against a + // carried etag would replay a row set the provider never vouched + // for. Compaction also clears the manifest (belt), but the flag + // is the contract (suspenders): "did this sync go through + // compaction" answers "can it be used for replay" with NO. + readReason = "previous sync is a compaction artifact; upstream validators do not describe its merged contents" + } else { + s.sourceCache.prev = prev + s.sourceCache.prevReader = s.previousSyncReader + } + + lookup := sourcecache.Lookup(sourcecache.NoopLookup{}) + if s.sourceCache.prev != nil { + lookup = prevStoreLookup{prev: s.sourceCache.prev, logOnce: &stdsync.Once{}} + } + s.sourceCache.lookup = lookup + if canSetLookup { + setLookup.SetSourceCache(ctx, lookup) + } else { + // The connector declared the capability but the client offers no + // way to deliver lookups. Its own lookup stays no-op, so every + // scope misses and no replay annotations can legally arrive. + l.Warn("source cache capability declared but connector client cannot receive lookups") + } + l.Info("source cache enabled", + zap.Bool("replay_available", s.sourceCache.prev != nil), + zap.String("replay_unavailable_reason", readReason), + ) + return nil +} + +// previousSyncIsCompacted reports whether the previous-sync reader's +// newest finished sync carries the compaction provenance flag +// (SyncRunRecord.compacted). A missing SyncMeta surface or absent run +// record degrades to "not compacted" — the manifest-based guards still +// apply — but a readable record is authoritative. +func previousSyncIsCompacted(ctx context.Context, reader connectorstore.Reader) (bool, error) { + metaHolder, ok := reader.(interface{ SyncMeta() c1zstore.SyncMeta }) + if !ok { + return false, nil + } + run, err := metaHolder.SyncMeta().LatestFinishedSyncOfAnyType(ctx) + if err != nil { + return false, err + } + if run == nil { + return false, nil + } + return run.Compacted, nil +} + +// clearSourceCacheLookup detaches the per-sync lookup so a late RPC from +// the connector cannot read a store the syncer no longer owns. +func (s *syncer) clearSourceCacheLookup(ctx context.Context) { + if setLookup, ok := s.connector.(sourcecache.SetLookup); ok { + setLookup.SetSourceCache(ctx, nil) + } +} + +// sourceCachePage carries one list response's source-cache instructions +// from beginSourceCachePage (before rows are written) to +// finishSourceCachePage (after rows are written). +type sourceCachePage struct { + kind sourcecache.RowKind + scopeHash string + etag string + // replayed reports that beginSourceCachePage copied the previous + // sync's rows for this scope into the current sync BEFORE the page's + // own rows commit. Consumers that dedupe against "already synced this + // sync" state (the resources path) must not skip this page's rows: + // they are the overlay, and the already-present row is the stale + // replayed base they exist to overwrite. + replayed bool + // replayRows / overlayRows carry the replay's copy count and the + // page's own row count from begin to finish, where the stats are + // recorded — recording at begin would double-count a page that is + // re-run after failing between replay and finish. + replayRows int64 + overlayRows int + // deletedIDs are canonical-id tombstones (grant/entitlement ids, + // resource BIDs); deletedPrincipalIDs are bare-object-id tombstones + // applied scope-relatively. Both may arrive on any page of a scope + // (replay annotation on the first page, scope annotation on every + // page) and apply after the page's rows commit. + deletedIDs []string + deletedPrincipalIDs []string +} + +// beginSourceCachePage inspects a list response's annotations, performs +// any requested replay, and returns the context to write the page's rows +// under (stamped with the scope when one is present). A nil page means the +// response carried no source-cache instructions. +// +// rowCount is the number of rows in the response; a non-overlay replay +// that also returned rows gets a warning (the rows are upserted anyway). +func (s *syncer) beginSourceCachePage( + ctx context.Context, + kind sourcecache.RowKind, + respAnnos annotations.Annotations, + rowCount int, +) (context.Context, *sourceCachePage, error) { + replay := &v2.SourceCacheReplay{} + hasReplay, err := respAnnos.Pick(replay) + if err != nil { + return ctx, nil, fmt.Errorf("source cache: error parsing replay annotation: %w", err) + } + scope := &v2.SourceCacheScope{} + hasScope, err := respAnnos.Pick(scope) + if err != nil { + return ctx, nil, fmt.Errorf("source cache: error parsing scope annotation: %w", err) + } + if !hasReplay && !hasScope { + return ctx, nil, nil + } + + if !s.sourceCache.enabled { + if hasReplay { + // The connector skipped row generation expecting a replay; with + // source cache disabled there is nothing to replay from. This is + // a connector bug (replay for a scope it never got a lookup hit + // on), not a degradable condition. + return ctx, nil, fmt.Errorf("source cache: connector requested replay for scope %q but source cache is disabled", replay.GetScopeHash()) + } + // Scope annotations without the capability handshake are ignored. + return ctx, nil, nil + } + + page := &sourceCachePage{kind: kind} + switch { + case hasReplay && hasScope: + if replay.GetScopeHash() != scope.GetScopeHash() { + return ctx, nil, fmt.Errorf("source cache: replay scope %q and page scope %q disagree", replay.GetScopeHash(), scope.GetScopeHash()) + } + page.scopeHash = replay.GetScopeHash() + case hasReplay: + page.scopeHash = replay.GetScopeHash() + default: + page.scopeHash = scope.GetScopeHash() + } + if err := sourcecache.ValidateScopeHash(page.scopeHash); err != nil { + return ctx, nil, fmt.Errorf("source cache: %w", err) + } + // Prefer the scope annotation's etag (the freshest validator on + // overlay pages); fall back to the replay's. + page.etag = scope.GetEtag() + if page.etag == "" { + page.etag = replay.GetEtag() + } + // Tombstones may ride either annotation — the replay annotation on a + // round's first page, the scope annotation on every page (so a + // multi-page delta round never buffers deletions). Fresh slices: an + // append onto the proto's returned slice could write into its backing + // array when it has spare capacity. + page.deletedIDs = append(append([]string{}, replay.GetDeletedIds()...), scope.GetDeletedIds()...) + page.deletedPrincipalIDs = append(append([]string{}, replay.GetDeletedPrincipalIds()...), scope.GetDeletedPrincipalIds()...) + + if hasReplay { + if s.sourceCache.prev == nil { + // Same invariant violation as the disabled case: the connector + // can only have gotten a lookup hit if a previous source exists. + return ctx, nil, fmt.Errorf("source cache: connector requested replay for scope %q but no previous sync is available", replay.GetScopeHash()) + } + page.replayed = true + if !replay.GetOverlay() && rowCount > 0 { + // The contract says a 304-style replay page is empty, but rows + // arriving here are more data, not less — upsert them on top of + // the replayed base (overlay semantics) rather than failing the + // sync. Kept lenient while the model is proven against real + // providers. + ctxzap.Extract(ctx).Warn("source cache: non-overlay replay returned rows; treating them as an overlay", + zap.String("scope_hash", page.scopeHash), + zap.Int("rows", rowCount), + ) + } + // Advisory check: a well-behaved connector only replays a scope + // whose validator came from this sync's lookup, so a missing + // previous manifest entry is suspicious — but not by itself data + // loss (the stamped rows may still exist, e.g. a partially carried + // file). The hard error below is reserved for a replay that + // produces nothing. + _, entryFound, err := s.sourceCache.prev.LookupSourceCacheEntry(ctx, kind, page.scopeHash) + if err != nil { + return ctx, nil, fmt.Errorf("source cache: error reading previous manifest for scope %q: %w", page.scopeHash, err) + } + if !entryFound { + ctxzap.Extract(ctx).Warn("source cache: replay requested for scope with no previous manifest entry", + zap.String("scope_hash", page.scopeHash)) + } + replayStart := time.Now() + res, err := s.sourceCache.current.ReplaySourceCache(ctx, s.sourceCache.prevReader, kind, page.scopeHash) + s.state.AddStepDuration("source_cache_replay", time.Since(replayStart)) + if err != nil { + return ctx, nil, fmt.Errorf("source cache: replay for scope %q failed: %w", page.scopeHash, err) + } + // Stats are stashed on the page and recorded in + // finishSourceCachePage so a page re-run after failing between + // replay and finish doesn't double-count. + page.replayRows = res.Rows + page.overlayRows = rowCount + if replay.GetOverlay() || page.etag == "" { + // This replay round can span pages: the validator arrives on + // a LATER page's scope annotation (delta rounds), and that + // page must count toward ScopesReplayed's scope, not as a + // freshly stamped one — see finishSourceCachePage. + s.sourceCache.markScopeReplayed(kind, page.scopeHash) + } + if res.Rows == 0 && !entryFound { + // The connector skipped row generation expecting a base that + // does not exist anywhere in the previous file — this sync + // would silently drop the scope's rows. + return ctx, nil, fmt.Errorf("source cache: replay for scope %q found no previous rows and no manifest entry; the connector replayed a scope it never looked up", page.scopeHash) + } + // Empty-vs-dropped discrimination via the previous file's scope + // index: a legitimately empty scope has NO index entries + // (StaleSkipped == 0, fine to proceed); a scope whose index says + // rows existed but none carried a matching value stamp means the + // rows were rewritten without index cleanup — proceeding would + // silently drop the scope's contents from this sync. + if res.Rows == 0 && res.StaleSkipped > 0 { + return ctx, nil, fmt.Errorf( + "source cache: replay for scope %q copied no rows but the previous file's scope index holds %d entries "+ + "whose rows no longer carry this scope's stamp; refusing to silently drop the scope", + page.scopeHash, res.StaleSkipped) + } + if res.StaleSkipped > 0 { + // Partial staleness: some rows copied, some index entries were + // dead. Not data loss by itself (the live rows carried their + // true stamp), but a signal a writer skipped index cleanup. + ctxzap.Extract(ctx).Warn("source cache: replay skipped stale scope-index entries", + zap.String("scope_hash", page.scopeHash), + zap.Int64("rows", res.Rows), + zap.Int64("stale_skipped", res.StaleSkipped), + ) + } + // Replay bypasses the connector-response path that normally arms + // grant expansion (seeing GrantExpandable on returned rows), so a + // sync whose expandable pages all replay would silently skip the + // expansion phase without this. + if kind == sourcecache.RowKindGrants && res.NeedsExpansion && !s.dontExpandGrants { + s.state.SetNeedsExpansion() + } + // Same rationale for external-resource-match processing: the + // flag is otherwise set only from response rows, so a sync whose + // match-annotated pages all replay would skip the transformation + // step and silently drop every matched grant. + if kind == sourcecache.RowKindGrants && res.HasExternalResourceGrants { + s.state.SetHasExternalResourcesGrants() + } + // Child scheduling is response-row-driven (getSubResources runs + // over resp.GetList()), so replayed parents never reach it — + // schedule their child resource syncs here or a warm sync + // silently drops whole subtrees. + if kind == sourcecache.RowKindResources { + for _, parent := range res.ChildResources { + s.pushChildResourceActions(ctx, parent.ChildTypeIDs, parent.ResourceTypeID, parent.ResourceID) + } + } + // Exclusion-group validation runs on response rows, so replayed + // entitlements would silently stop participating in the + // one-default / single-resource-type / size-cap invariants — a + // violation split across a replayed and a fresh scope would pass + // a warm sync that a full resync hard-fails. + if kind == sourcecache.RowKindEntitlements { + for _, eg := range res.ExclusionGroups { + if err := s.recordEntitlementExclusionGroup(eg.Group, eg.EntitlementID, eg.ResourceTypeID); err != nil { + return ctx, nil, fmt.Errorf("source cache: replayed entitlement failed exclusion-group validation: %w", err) + } + } + } + ctxzap.Extract(ctx).Debug("source cache replayed scope", + zap.String("row_kind", string(kind)), + zap.String("scope_hash", page.scopeHash), + zap.Int64("rows", res.Rows), + zap.Bool("needs_expansion", res.NeedsExpansion), + zap.Int("deleted_ids", len(page.deletedIDs)), + zap.Int("deleted_principal_ids", len(page.deletedPrincipalIDs)), + ) + } + + return sourcecache.WithScope(ctx, page.scopeHash), page, nil +} + +// finishSourceCachePage runs after the page's rows committed: applies +// delta tombstones and, when the page carried a validator, writes the +// current sync's manifest entry. The entry write is last so a failed page +// can never leave a phantom hit for the next sync. +func (s *syncer) finishSourceCachePage(ctx context.Context, page *sourceCachePage) error { + if page == nil { + return nil + } + var rowsDeleted int64 + if len(page.deletedIDs)+len(page.deletedPrincipalIDs) > 0 { + tombstoneStart := time.Now() + defer func() { + s.state.AddStepDuration("source_cache_tombstones", time.Since(tombstoneStart)) + }() + } + if len(page.deletedIDs) > 0 { + if page.kind == sourcecache.RowKindGrants { + // Grant-id tombstones resolve within the scope's own rows so + // connector-custom grant-id shapes (unreachable by the global + // bounded delete) work, and the cost stays bounded by the + // scope's size. + deleted, err := s.sourceCache.current.DeleteSourceCacheGrantsByIDInScope(ctx, page.scopeHash, page.deletedIDs) + if err != nil { + return fmt.Errorf("source cache: error applying grant deletions for scope %q: %w", page.scopeHash, err) + } + rowsDeleted += deleted + ctxzap.Extract(ctx).Debug("source cache applied grant-id deletions", + zap.String("scope_hash", page.scopeHash), + zap.Int("tombstones", len(page.deletedIDs)), + zap.Int64("rows_deleted", deleted), + ) + } else if err := s.sourceCache.current.DeleteSourceCacheRows(ctx, page.kind, page.deletedIDs); err != nil { + return fmt.Errorf("source cache: error applying deletions for scope %q: %w", page.scopeHash, err) + } + } + if len(page.deletedPrincipalIDs) > 0 { + deleted, err := s.sourceCache.current.DeleteSourceCacheRowsInScope(ctx, page.kind, page.scopeHash, page.deletedPrincipalIDs) + if err != nil { + return fmt.Errorf("source cache: error applying scoped deletions for scope %q: %w", page.scopeHash, err) + } + rowsDeleted += deleted + ctxzap.Extract(ctx).Debug("source cache applied scoped deletions", + zap.String("row_kind", string(page.kind)), + zap.String("scope_hash", page.scopeHash), + zap.Int("tombstones", len(page.deletedPrincipalIDs)), + zap.Int64("rows_deleted", deleted), + ) + } + if page.etag != "" { + if err := s.sourceCache.current.PutSourceCacheEntry(ctx, page.kind, page.scopeHash, page.etag); err != nil { + return fmt.Errorf("source cache: error writing manifest entry for scope %q: %w", page.scopeHash, err) + } + } + + stats := SourceCacheStats{ + TombstoneIDs: int64(len(page.deletedIDs)), + TombstonePrincipals: int64(len(page.deletedPrincipalIDs)), + RowsDeleted: rowsDeleted, + } + if page.replayed { + // Recorded here rather than in beginSourceCachePage so a page + // re-run after failing between replay and finish doesn't + // double-count the scope. + stats.ScopesReplayed = 1 + stats.RowsReplayed = map[string]int64{string(page.kind): page.replayRows} + stats.OverlayRows = int64(page.overlayRows) + } + if page.etag != "" && !page.replayed && !s.sourceCache.scopeReplayedThisSync(page.kind, page.scopeHash) { + // A cold/fresh page that persisted a validator: the denominator + // of the replay hit ratio. The replayed-scope check keeps a + // multi-page overlay round — whose new validator legally arrives + // on the final page, without a replay annotation — from booking + // its own hit as a miss too. + stats.ScopesStamped = 1 + } + if stats.TombstoneIDs > 0 || stats.TombstonePrincipals > 0 || stats.RowsDeleted > 0 || stats.ScopesStamped > 0 || page.replayed { + s.state.AddSourceCacheStats(stats) + } + return nil +} diff --git a/pkg/sync/source_cache_continuation.go b/pkg/sync/source_cache_continuation.go new file mode 100644 index 000000000..35dd69cdd --- /dev/null +++ b/pkg/sync/source_cache_continuation.go @@ -0,0 +1,229 @@ +package sync //nolint:revive,nolintlint // we can't change the package name for backwards compatibility + +// Syncer side of the source-cache lookup continuation (ask/answer). On +// single-shot transports (gRPC-over-Lambda) the connector cannot call the +// lookup service mid-request, so it answers a list RPC with a +// SourceCacheLookupAsk instead of rows; the syncer resolves the queries +// against its LOCAL previous-sync store — the same store replay copies +// from, so lookup and replay can never disagree — and re-invokes the same +// request with SourceCacheLookupAnswers attached. See +// docs/tasks/source-cache-lambda-lookup.md and the annotation contract in +// proto/c1/connector/v2/annotation_source_cache.proto. + +import ( + "context" + "fmt" + + "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" + "go.uber.org/zap" + + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + "github.com/conductorone/baton-sdk/pkg/annotations" + "github.com/conductorone/baton-sdk/pkg/sourcecache" +) + +const ( + // sourceCacheBounceCap bounds consecutive asks for the SAME request + // (same page token; only the answers annotation differs between + // re-invokes). A connector that keeps asking without progressing is + // broken (most commonly: swallowing ErrLookupDeferred and re-asking + // for scopes it already "handled"), and silence would be the + // stale-data failure mode — so fail loudly. + // + // Deliberately NOT per action: a multi-page action that asks once per + // page (e.g. a delta planner pre-resolving each planning page's chunk + // scopes) bounces once per page with monotonic progress — every + // NextPageToken advance is a new request and resets the counter. + // + // Shared with pkg/sourcecache so the connector-side answer-set cap + // (bounces × ask size) stays coherent with this loop's enforcement. + sourceCacheBounceCap = sourcecache.MaxLookupBouncesPerRequest + + // maxSpawnCursorsPerResponse bounds SpawnCursors fan-out per response + // (each spawned action persists in the checkpointed state token, so + // unbounded fan-out bloats every checkpoint). Matches the proto + // max_items validate rule on SpawnCursors.page_tokens. + maxSpawnCursorsPerResponse = 1024 + // maxSpawnCursorTokenBytes matches the page-token request field and + // the SpawnCursors item validation. Enforced before creating Actions: + // action tokens are checkpointed before they are sent through the + // later request field's validator. + maxSpawnCursorTokenBytes = 1 << 20 + + // sourceCacheAnswerBudget caps the FOUND-etag payload attached to one + // re-invoke, keeping the request under single-shot transport payload + // limits (Lambda invokes cap at 6MB; the dual-encoded frame at 5MiB). + // A found answer whose etag does not fit the remaining budget is + // DEGRADED to a not-found answer: the connector fetches that scope + // fresh (cold-correct, just slower). Leaving it absent-and-re-askable + // would livelock — answers accumulate across bounces (the connector + // re-executes from scratch each phase, so every re-invoke carries the + // union), so the budget only shrinks and an etag that did not fit once + // can never fit later; re-asking would burn straight into the bounce + // cap and hard-fail a sync that cold-degrades correctly. + sourceCacheAnswerBudget = 2 << 20 +) + +// Continuation counters live in SourceCacheStats on the sync state (see +// source_cache_stats.go): token-persisted so a resumed sync keeps them, +// reported once in the source-cache summary line. + +// listAttempt is one list-RPC attempt as observed by the continuation +// loop: enough of the response to detect and validate an ask. +type listAttempt struct { + annos annotations.Annotations + rows int + nextToken string +} + +// continuationCallMethod labels connector-call latency stats: re-invoked +// attempts (answer turns) get a ":cont" suffix so millisecond protocol +// turns don't blend into the base method's upstream-fetch distribution. +func continuationCallMethod(method string, attempt int) string { + if attempt > 0 { + return method + ":cont" + } + return method +} + +// withSourceCacheContinuation drives the ask/answer loop around one list +// RPC. issue performs the RPC with extra request annotations (the lookup +// offer, plus accumulated answers on re-invokes) and reports the response +// surface; the loop returns once a response carries no ask — that final +// response is the one the caller processes. attempt is 0 for the first +// issue and increments per re-invoke, so callers can label protocol turns +// separately in connector-call latency stats (a re-invoke that replays is +// not comparable to a cold upstream fetch). +// +// The offer is attached only when the syncer can actually answer (warm +// previous-sync lookup). Old or cold syncers never send it, and a +// compliant connector never asks without it — that pairing is what makes +// version skew degrade to a cold sync instead of a misread response. +func (s *syncer) withSourceCacheContinuation(ctx context.Context, op string, issue func(extra annotations.Annotations, attempt int) (listAttempt, error)) error { + l := ctxzap.Extract(ctx) + + warm := s.sourceCache.prev != nil + extra := annotations.Annotations{} + if warm { + extra.Update(&v2.SourceCacheLookupOffer{}) + } + + // Answers accumulate across bounces: the connector re-executes from + // scratch each phase, so every re-invoke must carry the union of all + // resolved queries, in first-resolved order (deterministic requests). + var ordered []sourcecache.Answer + seen := map[sourcecache.Query]bool{} + + asked, found, notFound, truncated := 0, 0, 0, 0 + for bounce := 0; ; bounce++ { + attempt, err := issue(extra, bounce) + if err != nil { + return err + } + + ask := &v2.SourceCacheLookupAsk{} + hasAsk, err := attempt.annos.Pick(ask) + if err != nil { + return fmt.Errorf("%s: error parsing source-cache lookup ask: %w", op, err) + } + if !hasAsk { + if bounce > 0 { + s.state.AddSourceCacheStats(SourceCacheStats{ + LookupBounces: int64(bounce), + LookupRequestsBounced: 1, + LookupScopesAsked: int64(asked), + LookupAnsweredFound: int64(found), + LookupAnsweredNotFound: int64(notFound), + LookupAnswersTruncated: int64(truncated), + LookupBouncesByOp: map[string]int64{op: int64(bounce)}, + }) + } + return nil + } + + // Ask legality. Failing loudly here is deliberate: every branch + // is a connector bug that would otherwise surface as silently + // wrong data or an unexplained stall. + if !warm { + return fmt.Errorf("%s: connector sent a source-cache lookup ask on a request that carried no offer (connector must gate asks on SourceCacheLookupOffer)", op) + } + if attempt.rows > 0 || attempt.nextToken != "" || + attempt.annos.Contains(&v2.SourceCacheScope{}) || attempt.annos.Contains(&v2.SourceCacheReplay{}) || + attempt.annos.Contains(&v2.SpawnCursors{}) { + return fmt.Errorf("%s: source-cache lookup ask response must carry ONLY the ask: "+ + "no rows, no next page token, no scope/replay annotations, no spawned cursors "+ + "(spawn on the re-invoked request's real response instead)", op) + } + if bounce >= sourceCacheBounceCap { + s.state.AddSourceCacheStats(SourceCacheStats{LookupCapFailures: 1}) + return fmt.Errorf("%s: source-cache lookup bounce cap (%d) exceeded for one request; "+ + "connector kept asking without progressing (%d scopes still unresolved) — "+ + "check for swallowed ErrLookupDeferred or nondeterministic scope computation", + op, sourceCacheBounceCap, len(ask.GetQueries())) + } + + queries, err := sourcecache.QueriesFromProto(ask) + if err != nil { + return fmt.Errorf("%s: invalid source-cache lookup ask: %w", op, err) + } + + budget := sourceCacheAnswerBudget + for _, a := range ordered { + budget -= len(a.ETag) + } + newAsked, newFound, newNotFound, newTruncated := 0, 0, 0, 0 + for _, q := range queries { + if seen[q] { + continue + } + newAsked++ + entry, ok, err := s.sourceCache.lookup.LookupPreviousSourceCache(ctx, q.RowKind, q.ScopeHash) + if err != nil { + return fmt.Errorf("%s: resolving source-cache lookup ask: %w", op, err) + } + if !ok { + seen[q] = true + ordered = append(ordered, sourcecache.Answer{Query: q, Found: false}) + newNotFound++ + continue + } + if len(entry.ETag) > budget { + // Found, but the etag does not fit the remaining budget. + // Degrade to an explicit not-found so the connector + // fetches this scope fresh (cold-correct). See the + // sourceCacheAnswerBudget comment: an absent answer could + // never be resolved on a later bounce, only livelock into + // the bounce cap. + seen[q] = true + ordered = append(ordered, sourcecache.Answer{Query: q, Found: false}) + newTruncated++ + continue + } + budget -= len(entry.ETag) + seen[q] = true + ordered = append(ordered, sourcecache.Answer{Query: q, Found: true, ETag: entry.ETag}) + newFound++ + } + asked += newAsked + found += newFound + notFound += newNotFound + truncated += newTruncated + + if newAsked == 0 { + // Every query was already answered on the request the + // connector just saw; re-invoking cannot make progress. + return fmt.Errorf("%s: connector re-asked only already-answered scopes (%d queries); connector lookup handling is broken", op, len(queries)) + } + + extra.Update(sourcecache.AnswersProto(ordered)) + + l.Debug("source-cache lookup bounce", + zap.String("op", op), + zap.Int("bounce", bounce+1), + zap.Int("asked", newAsked), + zap.Int("found", newFound), + zap.Int("not_found", newNotFound), + zap.Int("degraded_to_cold_for_budget", newTruncated), + ) + } +} diff --git a/pkg/sync/source_cache_continuation_e2e_test.go b/pkg/sync/source_cache_continuation_e2e_test.go new file mode 100644 index 000000000..5cc066cc4 --- /dev/null +++ b/pkg/sync/source_cache_continuation_e2e_test.go @@ -0,0 +1,257 @@ +package sync //nolint:revive,nolintlint // we can't change the package name for backwards compatibility + +// Composed end-to-end test for the lookup continuation over the REAL +// Lambda stack: a connectorbuilder-built connector (unchanged connector +// code, lookup-before-fetch, errors propagated) registered on the actual +// gRPC-over-Lambda server, reached through the actual transport encodings +// (Request/Response MarshalJSON/UnmarshalJSON — dual-encoded requests, +// frame/legacy responses), driven by the real syncer. This is the +// production Lambda topology minus AWS: every layer between the syncer's +// bounce loop and the connector's SyncOpAttrs lookup is the shipping code +// path, including the wire round trip of the Offer/Ask/Answers +// annotations. + +import ( + "context" + "encoding/json" + "fmt" + "path/filepath" + "sync" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/conductorone/baton-sdk/internal/connector" + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + "github.com/conductorone/baton-sdk/pkg/annotations" + "github.com/conductorone/baton-sdk/pkg/connectorbuilder" + "github.com/conductorone/baton-sdk/pkg/connectorclient" + c1lambda "github.com/conductorone/baton-sdk/pkg/lambda/grpc" + "github.com/conductorone/baton-sdk/pkg/logging" + "github.com/conductorone/baton-sdk/pkg/sourcecache" + et "github.com/conductorone/baton-sdk/pkg/types/entitlement" + gt "github.com/conductorone/baton-sdk/pkg/types/grant" + rs "github.com/conductorone/baton-sdk/pkg/types/resource" +) + +// jsonWireTransport is the Lambda invoke minus AWS: the request is +// marshaled to the invoke payload and unmarshaled server-side, the +// response likewise back — both through the exact MarshalJSON / +// UnmarshalJSON paths the production transport uses. +type jsonWireTransport struct { + server *c1lambda.Server +} + +func (t jsonWireTransport) RoundTrip(ctx context.Context, req *c1lambda.Request) (*c1lambda.Response, error) { + payload, err := json.Marshal(req) + if err != nil { + return nil, fmt.Errorf("wire: marshal request: %w", err) + } + serverReq := &c1lambda.Request{} + if err := json.Unmarshal(payload, serverReq); err != nil { + return nil, fmt.Errorf("wire: unmarshal request: %w", err) + } + resp, err := t.server.Handler(ctx, serverReq) + if err != nil { + return nil, err + } + respPayload, err := json.Marshal(resp) + if err != nil { + return nil, fmt.Errorf("wire: marshal response: %w", err) + } + clientResp := &c1lambda.Response{} + if err := json.Unmarshal(respPayload, clientResp); err != nil { + return nil, fmt.Errorf("wire: unmarshal response: %w", err) + } + return clientResp, nil +} + +// lambdaE2EBuilder is a plain ConnectorBuilderV2 — the shape a real +// connector ships. No SetLookup, no test hooks in the sync path. +type lambdaE2EBuilder struct { + syncer *lambdaE2ESyncer +} + +func (b *lambdaE2EBuilder) Metadata(context.Context) (*v2.ConnectorMetadata, error) { + return v2.ConnectorMetadata_builder{DisplayName: "lambda-e2e"}.Build(), nil +} + +func (b *lambdaE2EBuilder) Validate(context.Context) (annotations.Annotations, error) { + return annotations.New(v2.SourceCacheCapability_builder{ + Mode: v2.SourceCacheCapability_MODE_READ_WRITE, + }.Build()), nil +} + +func (b *lambdaE2EBuilder) ResourceSyncers(context.Context) []connectorbuilder.ResourceSyncerV2 { + return []connectorbuilder.ResourceSyncerV2{b.syncer, &lambdaE2EUserSyncer{}} +} + +type lambdaE2EUserSyncer struct{} + +func (s *lambdaE2EUserSyncer) ResourceType(context.Context) *v2.ResourceType { + return userResourceType +} + +func (s *lambdaE2EUserSyncer) List(_ context.Context, _ *v2.ResourceId, _ rs.SyncOpAttrs) ([]*v2.Resource, *rs.SyncOpResults, error) { + var out []*v2.Resource + for i := 0; i < 4; i++ { + u, err := rs.NewUserResource(fmt.Sprintf("user-%02d", i), userResourceType, fmt.Sprintf("user-%02d", i), nil, + rs.WithAnnotation(&v2.SkipEntitlementsAndGrants{})) + if err != nil { + return nil, nil, err + } + out = append(out, u) + } + return out, &rs.SyncOpResults{}, nil +} + +func (s *lambdaE2EUserSyncer) Entitlements(context.Context, *v2.Resource, rs.SyncOpAttrs) ([]*v2.Entitlement, *rs.SyncOpResults, error) { + return nil, &rs.SyncOpResults{}, nil +} + +func (s *lambdaE2EUserSyncer) Grants(context.Context, *v2.Resource, rs.SyncOpAttrs) ([]*v2.Grant, *rs.SyncOpResults, error) { + return nil, &rs.SyncOpResults{}, nil +} + +// lambdaE2ESyncer serves group member grants with per-group scopes, +// written exactly the way the contract tells connector authors to: +// lookup FIRST (propagating errors with %w), replay on hit, cold + scope +// on miss. +type lambdaE2ESyncer struct { + mu sync.Mutex + members map[string][]string + etagByGroup map[string]string + lookups int + replays int + colds int +} + +func (s *lambdaE2ESyncer) ResourceType(context.Context) *v2.ResourceType { + return groupResourceType +} + +func (s *lambdaE2ESyncer) group(gid string) (*v2.Resource, error) { + return rs.NewGroupResource(gid, groupResourceType, gid, nil) +} + +func (s *lambdaE2ESyncer) List(_ context.Context, _ *v2.ResourceId, _ rs.SyncOpAttrs) ([]*v2.Resource, *rs.SyncOpResults, error) { + s.mu.Lock() + defer s.mu.Unlock() + var out []*v2.Resource + for gid := range s.members { + g, err := s.group(gid) + if err != nil { + return nil, nil, err + } + out = append(out, g) + } + return out, &rs.SyncOpResults{}, nil +} + +func (s *lambdaE2ESyncer) Entitlements(_ context.Context, r *v2.Resource, _ rs.SyncOpAttrs) ([]*v2.Entitlement, *rs.SyncOpResults, error) { + ent := et.NewAssignmentEntitlement(r, "member", et.WithGrantableTo(userResourceType)) + ent.SetSlug("member") + return []*v2.Entitlement{ent}, &rs.SyncOpResults{}, nil +} + +func (s *lambdaE2ESyncer) Grants(ctx context.Context, r *v2.Resource, opts rs.SyncOpAttrs) ([]*v2.Grant, *rs.SyncOpResults, error) { + gid := r.GetId().GetResource() + scope := "e2e/groups/" + gid + "/members" + + s.mu.Lock() + s.lookups++ + etag := s.etagByGroup[gid] + uids := append([]string(nil), s.members[gid]...) + s.mu.Unlock() + + entry, found, err := opts.SourceCache.LookupPreviousSourceCache(ctx, sourcecache.RowKindGrants, scope) + if err != nil { + return nil, nil, fmt.Errorf("revalidating %s: %w", gid, err) + } + ret := &rs.SyncOpResults{Annotations: annotations.Annotations{}} + if found && entry.ETag == etag { + s.mu.Lock() + s.replays++ + s.mu.Unlock() + ret.Annotations.Update(v2.SourceCacheReplay_builder{ScopeHash: scope, Etag: etag}.Build()) + return nil, ret, nil + } + + s.mu.Lock() + s.colds++ + s.mu.Unlock() + grants := make([]*v2.Grant, 0, len(uids)) + for _, uid := range uids { + u, err := rs.NewUserResource(uid, userResourceType, uid, nil) + if err != nil { + return nil, nil, err + } + grants = append(grants, gt.NewGrant(r, "member", u.GetId())) + } + ret.Annotations.Update(v2.SourceCacheScope_builder{ScopeHash: scope, Etag: etag}.Build()) + return grants, ret, nil +} + +// TestSourceCacheContinuation_LambdaStackE2E composes the shipping layers: +// connectorbuilder (per-request ContinuationLookup + deferral interception) +// behind internal/connector.Register on the gRPC-over-Lambda server, +// through the real JSON wire encodings, under the real syncer bounce loop. +func TestSourceCacheContinuation_LambdaStackE2E(t *testing.T) { + ctx, err := logging.Init(context.Background()) + require.NoError(t, err) + tmpDir := t.TempDir() + + es := &lambdaE2ESyncer{ + members: map[string][]string{ + "group-00": {"user-00", "user-01"}, + "group-01": {"user-02", "user-03"}, + }, + etagByGroup: map[string]string{"group-00": "e1", "group-01": "e1"}, + } + cb := &lambdaE2EBuilder{syncer: es} + + // The Lambda server path: connectorbuilder WITHOUT any runner-supplied + // lookup (RunTimeOpts.SourceCacheLookup is not wired on Lambda), so + // the continuation is the only lookup transport available. + cs, err := connectorbuilder.NewConnector(ctx, cb) + require.NoError(t, err) + server := c1lambda.NewServer(nil) + connector.Register(ctx, server, cs, nil) + client := connectorclient.NewConnectorClient(ctx, c1lambda.NewClientConn(jsonWireTransport{server: server})) + + // Sync 1: cold. No previous c1z → no offer → the connector's lookup is + // NoopLookup; everything fetches cold with zero bounces. + sync1 := filepath.Join(tmpDir, "sync1.c1z") + require.NoError(t, runContinuationSync(ctx, t, client, sync1, "", tmpDir, 0)) + require.Equal(t, 2, es.colds) + require.Zero(t, es.replays) + grants1 := listGrantsInFile(ctx, t, sync1) + require.Len(t, grants1, 4) + + // Sync 2: warm. The offer rides the wire, the connector's first + // execution defers (ErrLookupDeferred through the builder, ask through + // the transport), the re-invoke carries answers, and both groups + // replay. lookups counts handler EXECUTIONS: 2 phase-1 + 2 phase-2. + sync2 := filepath.Join(tmpDir, "sync2.c1z") + require.NoError(t, runContinuationSync(ctx, t, client, sync2, sync1, tmpDir, 0)) + require.Equal(t, 2, es.replays, "both groups replay via ask/answer over the wire") + require.Equal(t, 2, es.colds, "no cold fetches on the warm sync") + require.Equal(t, 6, es.lookups, "2 cold + 2 deferred phase-1 + 2 answered phase-2 handler executions") + requireSameGrantIDs(t, grants1, listGrantsInFile(ctx, t, sync2), "lambda-stack warm vs cold") + + // Churn one group; warm sync 3: stale answer → cold refetch for it, + // replay for the other; equivalence against a cold control. + es.mu.Lock() + es.members["group-01"] = []string{"user-02"} + es.etagByGroup["group-01"] = "e2" + es.mu.Unlock() + + sync3 := filepath.Join(tmpDir, "sync3.c1z") + require.NoError(t, runContinuationSync(ctx, t, client, sync3, sync2, tmpDir, 0)) + require.Equal(t, 3, es.replays) + require.Equal(t, 3, es.colds) + + control := filepath.Join(tmpDir, "control.c1z") + require.NoError(t, runContinuationSync(ctx, t, client, control, "", tmpDir, 0)) + requireSameGrantIDs(t, listGrantsInFile(ctx, t, control), listGrantsInFile(ctx, t, sync3), "churned lambda-stack warm vs cold control") +} diff --git a/pkg/sync/source_cache_continuation_test.go b/pkg/sync/source_cache_continuation_test.go new file mode 100644 index 000000000..25d7454ae --- /dev/null +++ b/pkg/sync/source_cache_continuation_test.go @@ -0,0 +1,780 @@ +package sync //nolint:revive,nolintlint // we can't change the package name for backwards compatibility + +// Syncer-side tests for the source-cache lookup continuation (ask/answer). +// The mocks here play the LAMBDA side of the protocol by hand: they do NOT +// implement sourcecache.SetLookup (no direct lookup install — exactly the +// single-shot-transport topology), and instead answer list RPCs with +// SourceCacheLookupAsk and consume SourceCacheLookupAnswers from the +// re-invoked request. + +import ( + "fmt" + "path/filepath" + "strings" + "sync" + "testing" + + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + "github.com/conductorone/baton-sdk/pkg/annotations" + "github.com/conductorone/baton-sdk/pkg/dotc1z" + "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" + "github.com/conductorone/baton-sdk/pkg/logging" + "github.com/conductorone/baton-sdk/pkg/sourcecache" + "github.com/conductorone/baton-sdk/pkg/types" + gt "github.com/conductorone/baton-sdk/pkg/types/grant" + rs "github.com/conductorone/baton-sdk/pkg/types/resource" + + "context" +) + +func runContinuationSync(ctx context.Context, t *testing.T, mc types.ConnectorClient, path, prevPath, tmpDir string, workers int) error { + t.Helper() + store, err := dotc1z.NewStore(ctx, path, + dotc1z.WithEngine(c1zstore.EnginePebble), + dotc1z.WithTmpDir(tmpDir), + ) + require.NoError(t, err) + opts := []SyncOpt{WithConnectorStore(store), WithTmpDir(tmpDir)} + if prevPath != "" { + opts = append(opts, WithPreviousSyncC1ZPath(prevPath)) + } + if workers > 0 { + opts = append(opts, WithWorkerCount(workers)) + } + syncer, err := NewSyncer(ctx, mc, opts...) + require.NoError(t, err) + syncErr := syncer.Sync(ctx) + if syncErr != nil { + cctx, cancel := context.WithCancel(ctx) + cancel() + _ = syncer.Close(cctx) + return syncErr + } + require.NoError(t, syncer.Close(ctx)) + return nil +} + +// continuationMockConnector: one grants page per group, one scope per +// group. Lambda-side protocol by hand; misuse modes for the loud-failure +// paths. +type continuationMockConnector struct { + *mockConnector + + mu sync.Mutex + etagByGroup map[string]string + grantsByGroup map[string][]*v2.Grant + + askAlways bool // never satisfied: bounce-cap test + askWithRows bool // protocol violation: ask + rows + askNoOffer bool // protocol violation: ask on offerless request + askWithSpawn bool // protocol violation: ask + SpawnCursors + + asks int + replayPages int + coldPages int + offerSeen bool +} + +func (mc *continuationMockConnector) Validate(context.Context, *v2.ConnectorServiceValidateRequest, ...grpc.CallOption) (*v2.ConnectorServiceValidateResponse, error) { + return v2.ConnectorServiceValidateResponse_builder{ + Annotations: annotations.New(v2.SourceCacheCapability_builder{ + Mode: v2.SourceCacheCapability_MODE_READ_WRITE, + }.Build()), + }.Build(), nil +} + +func groupScope(gid string) string { return "groups/" + gid + "/members" } + +func (mc *continuationMockConnector) askResponse(scope string, rows []*v2.Grant) *v2.GrantsServiceListGrantsResponse { + mc.mu.Lock() + mc.asks++ + mc.mu.Unlock() + return v2.GrantsServiceListGrantsResponse_builder{ + List: rows, + Annotations: annotations.New(sourcecache.AskProto([]sourcecache.Query{ + {RowKind: sourcecache.RowKindGrants, ScopeHash: scope}, + })), + }.Build() +} + +func (mc *continuationMockConnector) ListGrants(ctx context.Context, in *v2.GrantsServiceListGrantsRequest, _ ...grpc.CallOption) (*v2.GrantsServiceListGrantsResponse, error) { + gid := in.GetResource().GetId().GetResource() + if in.GetResource().GetId().GetResourceType() != groupResourceType.GetId() { + return v2.GrantsServiceListGrantsResponse_builder{}.Build(), nil + } + scope := groupScope(gid) + reqAnnos := annotations.Annotations(in.GetAnnotations()) + hasOffer := reqAnnos.Contains(&v2.SourceCacheLookupOffer{}) + answersMsg := &v2.SourceCacheLookupAnswers{} + hasAnswers, err := reqAnnos.Pick(answersMsg) + if err != nil { + return nil, err + } + mc.mu.Lock() + if hasOffer { + mc.offerSeen = true + } + etag := mc.etagByGroup[gid] + rows := mc.grantsByGroup[gid] + mc.mu.Unlock() + + if mc.askAlways { + return mc.askResponse(scope, nil), nil + } + if mc.askNoOffer && !hasAnswers { + return mc.askResponse(scope, nil), nil + } + if mc.askWithRows { + return mc.askResponse(scope, rows), nil + } + if mc.askWithSpawn && !hasAnswers && hasOffer { + resp := mc.askResponse(scope, nil) + annos := annotations.Annotations(resp.GetAnnotations()) + annos.Update(v2.SpawnCursors_builder{PageTokens: []string{"bogus"}}.Build()) + resp.SetAnnotations(annos) + return resp, nil + } + + if hasAnswers { + answers, answersErr := sourcecache.AnswersFromProto(answersMsg) + if answersErr != nil { + return nil, answersErr + } + for _, a := range answers { + if a.ScopeHash != scope || a.RowKind != sourcecache.RowKindGrants { + continue + } + if a.Found && a.ETag == etag { + mc.mu.Lock() + mc.replayPages++ + mc.mu.Unlock() + return v2.GrantsServiceListGrantsResponse_builder{ + Annotations: annotations.New(v2.SourceCacheReplay_builder{ + ScopeHash: scope, + Etag: etag, + }.Build()), + }.Build(), nil + } + break // answered but stale/missing: cold below + } + } else if hasOffer { + // Lookup-before-fetch, deferred: phase 1 does no upstream work. + return mc.askResponse(scope, nil), nil + } + + mc.mu.Lock() + mc.coldPages++ + mc.mu.Unlock() + return v2.GrantsServiceListGrantsResponse_builder{ + List: rows, + Annotations: annotations.New(v2.SourceCacheScope_builder{ + ScopeHash: scope, + Etag: etag, + }.Build()), + }.Build(), nil +} + +func newContinuationMockConnector(t *testing.T, groups int) *continuationMockConnector { + t.Helper() + ctx := context.Background() + mc := &continuationMockConnector{ + mockConnector: newMockConnector(), + etagByGroup: map[string]string{}, + grantsByGroup: map[string][]*v2.Grant{}, + } + mc.rtDB = append(mc.rtDB, groupResourceType, userResourceType) + for i := 0; i < groups*2; i++ { + uid := fmt.Sprintf("user-%02d", i) + u, err := rs.NewUserResource(uid, userResourceType, uid, nil, rs.WithAnnotation(&v2.SkipEntitlementsAndGrants{})) + require.NoError(t, err) + mc.AddResource(ctx, u) + } + for i := 0; i < groups; i++ { + gid := fmt.Sprintf("group-%02d", i) + g, err := rs.NewGroupResource(gid, groupResourceType, gid, nil) + require.NoError(t, err) + mc.AddResource(ctx, g) + mc.entDB[gid] = []*v2.Entitlement{mkMemberEnt(g)} + mc.etagByGroup[gid] = "etag-v1" + for m := 0; m < 2; m++ { + uid := fmt.Sprintf("user-%02d", (i*2+m)%(groups*2)) + ur, err := rs.NewUserResource(uid, userResourceType, uid, nil) + require.NoError(t, err) + mc.grantsByGroup[gid] = append(mc.grantsByGroup[gid], gt.NewGrant(g, "member", ur.GetId())) + } + } + return mc +} + +// TestSourceCacheContinuation_WarmReplay is the whole protocol, end to +// end, against the real syncer loop: cold sync (no offer → no asks), warm +// sync (offer → one ask per group → answers → replay), and a third sync +// chained off the replay-only file (manifest carried forward through +// ask-path pages). +func TestSourceCacheContinuation_WarmReplay(t *testing.T) { + for _, workers := range []int{0, 4} { + t.Run(fmt.Sprintf("workers=%d", workers), func(t *testing.T) { + ctx, err := logging.Init(context.Background()) + require.NoError(t, err) + tmpDir := t.TempDir() + mc := newContinuationMockConnector(t, 5) + + sync1 := filepath.Join(tmpDir, "sync1.c1z") + require.NoError(t, runContinuationSync(ctx, t, mc, sync1, "", tmpDir, workers)) + require.False(t, mc.offerSeen, "cold sync (no previous c1z) must not offer lookups") + require.Zero(t, mc.asks, "no offer → a compliant connector never asks") + require.Equal(t, 5, mc.coldPages) + grants1 := listGrantsInFile(ctx, t, sync1) + require.Len(t, grants1, 10) + + sync2 := filepath.Join(tmpDir, "sync2.c1z") + require.NoError(t, runContinuationSync(ctx, t, mc, sync2, sync1, tmpDir, workers)) + require.True(t, mc.offerSeen) + require.Equal(t, 5, mc.asks, "one ask (bounce) per group page") + require.Equal(t, 5, mc.replayPages, "every page replays after its answer") + require.Equal(t, 5, mc.coldPages, "no cold fetches on the warm sync") + requireSameGrantIDs(t, grants1, listGrantsInFile(ctx, t, sync2), "warm-via-continuation vs cold") + + // Replay-only file remains a valid replay source. + sync3 := filepath.Join(tmpDir, "sync3.c1z") + require.NoError(t, runContinuationSync(ctx, t, mc, sync3, sync2, tmpDir, workers)) + require.Equal(t, 10, mc.replayPages) + requireSameGrantIDs(t, grants1, listGrantsInFile(ctx, t, sync3), "chained continuation sync") + + // Churn one group: its answer goes stale → cold refetch. + mc.mu.Lock() + g0 := mc.grantsByGroup["group-00"] + mc.grantsByGroup["group-00"] = g0[:1] + mc.etagByGroup["group-00"] = "etag-v2" + mc.mu.Unlock() + sync4 := filepath.Join(tmpDir, "sync4.c1z") + require.NoError(t, runContinuationSync(ctx, t, mc, sync4, sync3, tmpDir, workers)) + require.Equal(t, 6, mc.coldPages, "stale answer → exactly one cold refetch") + grants4 := listGrantsInFile(ctx, t, sync4) + require.Len(t, grants4, 9) + require.NotContains(t, grants4, g0[1].GetId()) + }) + } +} + +// verdictSpawnMock is the planner shape both consumer POCs asked for: the +// ORIGIN call batch-asks for every stored page's validator in ONE bounce +// (LookupMany semantics), then its phase-2 response serves page 0 and +// spawns sibling cursors whose tokens CARRY the verdicts. Siblings serve +// replay/cold from their token alone — zero asks, zero bounces. This pins +// the two contract commitments from the review briefs: the ask/answer loop +// composes with a phase-2 SpawnCursors response, and a validator resolved +// in an EARLIER action of the same sync legally rides page tokens. +type verdictSpawnMock struct { + *mockConnector + + mu sync.Mutex + members map[string][]string // 2 rows per page + prevPages map[string]int + resByID map[string]*v2.Resource + + originAsks int + siblingAsks int + replays int + colds int +} + +func (mc *verdictSpawnMock) Validate(context.Context, *v2.ConnectorServiceValidateRequest, ...grpc.CallOption) (*v2.ConnectorServiceValidateResponse, error) { + return v2.ConnectorServiceValidateResponse_builder{ + Annotations: annotations.New(v2.SourceCacheCapability_builder{ + Mode: v2.SourceCacheCapability_MODE_READ_WRITE, + }.Build()), + }.Build(), nil +} + +func vsPageScope(gid string, page int) string { return fmt.Sprintf("g/%s/members?page=%d", gid, page) } + +func (mc *verdictSpawnMock) pageRows(gid string, page int) []string { + all := mc.members[gid] + lo, hi := page*2, page*2+2 + if lo >= len(all) { + return nil + } + if hi > len(all) { + hi = len(all) + } + return all[lo:hi] +} + +func vsPageEtag(gid string, page int, rows []string) string { + return fmt.Sprintf("v|%s|%d|%v", gid, page, rows) +} + +func (mc *verdictSpawnMock) servePage(gid string, page int, chain bool) *v2.GrantsServiceListGrantsResponse { + rows := mc.pageRows(gid, page) + grants := make([]*v2.Grant, 0, len(rows)) + g := mc.resByID[gid] + for _, uid := range rows { + grants = append(grants, gt.NewGrant(g, "member", mc.resByID[uid].GetId())) + } + next := "" + if chain && len(mc.pageRows(gid, page+1)) > 0 { + next = fmt.Sprintf("page=%d", page+1) + } + mc.colds++ + return v2.GrantsServiceListGrantsResponse_builder{ + List: grants, + Annotations: annotations.New(v2.SourceCacheScope_builder{ + ScopeHash: vsPageScope(gid, page), + Etag: vsPageEtag(gid, page, rows), + }.Build()), + NextPageToken: next, + }.Build() +} + +func (mc *verdictSpawnMock) ListGrants(_ context.Context, in *v2.GrantsServiceListGrantsRequest, _ ...grpc.CallOption) (*v2.GrantsServiceListGrantsResponse, error) { + gid := in.GetResource().GetId().GetResource() + if in.GetResource().GetId().GetResourceType() != groupResourceType.GetId() { + return v2.GrantsServiceListGrantsResponse_builder{}.Build(), nil + } + mc.mu.Lock() + defer mc.mu.Unlock() + + if tok := in.GetPageToken(); tok != "" { + // Sibling with a verdict-carrying token: "page=N|1|". + var page int + var found int + var etag string + if n, _ := fmt.Sscanf(tok, "page=%d|%d|", &page, &found); n == 2 { + if i := strings.Index(tok, "|"); i >= 0 { + if j := strings.Index(tok[i+1:], "|"); j >= 0 { + etag = tok[i+1+j+1:] + } + } + rows := mc.pageRows(gid, page) + if found == 1 && etag == vsPageEtag(gid, page, rows) { + mc.replays++ + return v2.GrantsServiceListGrantsResponse_builder{ + Annotations: annotations.New(v2.SourceCacheReplay_builder{ + ScopeHash: vsPageScope(gid, page), + Etag: etag, + }.Build()), + }.Build(), nil + } + // Stale/missing verdict: cold, still no ask. + return mc.servePage(gid, page, false), nil + } + // Plain cold chain token: "page=N". + if _, err := fmt.Sscanf(tok, "page=%d", &page); err != nil { + return nil, fmt.Errorf("bad token %q", tok) + } + return mc.servePage(gid, page, true), nil + } + + // Origin call. + reqAnnos := annotations.Annotations(in.GetAnnotations()) + answersMsg := &v2.SourceCacheLookupAnswers{} + hasAnswers, err := reqAnnos.Pick(answersMsg) + if err != nil { + return nil, err + } + prev := mc.prevPages[gid] + + if !hasAnswers { + if reqAnnos.Contains(&v2.SourceCacheLookupOffer{}) && prev > 0 { + // ONE batch ask for the whole stored chain. + mc.originAsks++ + queries := make([]sourcecache.Query, 0, prev) + for p := 0; p < prev; p++ { + queries = append(queries, sourcecache.Query{RowKind: sourcecache.RowKindGrants, ScopeHash: vsPageScope(gid, p)}) + } + return v2.GrantsServiceListGrantsResponse_builder{ + Annotations: annotations.New(sourcecache.AskProto(queries)), + }.Build(), nil + } + // Cold first sync: serial chain. + return mc.servePage(gid, 0, true), nil + } + + // Phase 2: serve page 0 per its answer, spawn verdict-token siblings. + byScope := map[string]sourcecache.Answer{} + answers, answersErr := sourcecache.AnswersFromProto(answersMsg) + if answersErr != nil { + return nil, answersErr + } + for _, a := range answers { + byScope[a.ScopeHash] = a + } + var resp *v2.GrantsServiceListGrantsResponse + if a, ok := byScope[vsPageScope(gid, 0)]; ok && a.Found && a.ETag == vsPageEtag(gid, 0, mc.pageRows(gid, 0)) { + mc.replays++ + resp = v2.GrantsServiceListGrantsResponse_builder{ + Annotations: annotations.New(v2.SourceCacheReplay_builder{ + ScopeHash: vsPageScope(gid, 0), + Etag: a.ETag, + }.Build()), + }.Build() + } else { + resp = mc.servePage(gid, 0, false) + } + var tokens []string + for p := 1; p < prev; p++ { + a := byScope[vsPageScope(gid, p)] + found := 0 + if a.Found { + found = 1 + } + tokens = append(tokens, fmt.Sprintf("page=%d|%d|%s", p, found, a.ETag)) + } + if len(tokens) > 0 { + annos := annotations.Annotations(resp.GetAnnotations()) + annos.Update(v2.SpawnCursors_builder{PageTokens: tokens}.Build()) + resp.SetAnnotations(annos) + resp.SetNextPageToken("") + } + return resp, nil +} + +func TestSourceCacheContinuation_Phase2SpawnCursors(t *testing.T) { + for _, workers := range []int{0, 4} { + t.Run(fmt.Sprintf("workers=%d", workers), func(t *testing.T) { + ctx, err := logging.Init(context.Background()) + require.NoError(t, err) + tmpDir := t.TempDir() + + mc := &verdictSpawnMock{ + mockConnector: newMockConnector(), + members: map[string][]string{}, + prevPages: map[string]int{}, + resByID: map[string]*v2.Resource{}, + } + mc.rtDB = append(mc.rtDB, groupResourceType, userResourceType) + for i := 0; i < 8; i++ { + uid := fmt.Sprintf("user-%02d", i) + u, err := rs.NewUserResource(uid, userResourceType, uid, nil, rs.WithAnnotation(&v2.SkipEntitlementsAndGrants{})) + require.NoError(t, err) + mc.AddResource(ctx, u) + mc.resByID[uid] = u + } + for i := 0; i < 2; i++ { + gid := fmt.Sprintf("group-%02d", i) + g, err := rs.NewGroupResource(gid, groupResourceType, gid, nil) + require.NoError(t, err) + mc.AddResource(ctx, g) + mc.resByID[gid] = g + mc.entDB[gid] = []*v2.Entitlement{mkMemberEnt(g)} + for m := 0; m < 4; m++ { + mc.members[gid] = append(mc.members[gid], fmt.Sprintf("user-%02d", (i*4+m)%8)) + } + } + + sync1 := filepath.Join(tmpDir, "sync1.c1z") + require.NoError(t, runContinuationSync(ctx, t, mc, sync1, "", tmpDir, workers)) + require.Zero(t, mc.originAsks) + require.Equal(t, 4, mc.colds, "cold: 2 groups × 2 chained pages") + grants1 := listGrantsInFile(ctx, t, sync1) + require.Len(t, grants1, 8) + mc.mu.Lock() + for gid := range mc.members { + mc.prevPages[gid] = 2 + } + mc.mu.Unlock() + + sync2 := filepath.Join(tmpDir, "sync2.c1z") + require.NoError(t, runContinuationSync(ctx, t, mc, sync2, sync1, tmpDir, workers)) + require.Equal(t, 2, mc.originAsks, "ONE batch ask per group's origin action") + require.Zero(t, mc.siblingAsks, "verdict-carrying siblings never ask") + require.Equal(t, 4, mc.replays, "page 0 (phase 2) + spawned page 1, per group") + require.Equal(t, 4, mc.colds, "no cold fetches on the warm sync") + requireSameGrantIDs(t, grants1, listGrantsInFile(ctx, t, sync2), "planner-batch warm vs cold") + + // Churn group-01's tail page: its verdict goes stale, the + // sibling refetches cold from the token alone — still no ask. + mc.mu.Lock() + mc.members["group-01"][3] = "user-00" + mc.mu.Unlock() + sync3 := filepath.Join(tmpDir, "sync3.c1z") + require.NoError(t, runContinuationSync(ctx, t, mc, sync3, sync2, tmpDir, workers)) + require.Zero(t, mc.siblingAsks) + + control := filepath.Join(tmpDir, "control.c1z") + require.NoError(t, runContinuationSync(ctx, t, mc, control, "", tmpDir, workers)) + requireSameGrantIDs(t, listGrantsInFile(ctx, t, control), listGrantsInFile(ctx, t, sync3), "churned warm vs cold control") + }) + } +} + +func TestSourceCacheContinuation_BounceCapFails(t *testing.T) { + ctx, err := logging.Init(context.Background()) + require.NoError(t, err) + tmpDir := t.TempDir() + mc := newContinuationMockConnector(t, 1) + + sync1 := filepath.Join(tmpDir, "sync1.c1z") + require.NoError(t, runContinuationSync(ctx, t, mc, sync1, "", tmpDir, 0)) + + mc.askAlways = true + err = runContinuationSync(ctx, t, mc, filepath.Join(tmpDir, "sync2.c1z"), sync1, tmpDir, 0) + require.Error(t, err) + require.True(t, + strings.Contains(err.Error(), "bounce cap") || strings.Contains(err.Error(), "re-asked only already-answered"), + "a connector that never stops asking on ONE request must fail loudly, got: %v", err) +} + +// multiPagePlannerMock is the Entra planner shape: a TYPE-SCOPED grants +// action whose planning walk spans SIX pages, each of which asks once +// (LookupMany over the chunk scopes it is about to spawn) before serving. +// Six asks exceed the per-request bounce cap of four — the sync must still +// complete, because the cap is per REQUEST (same page token) and every +// NextPageToken advance is a fresh request. The final planning page spawns +// the real cursor; it must spawn exactly once (the ask response never +// reaches SpawnCursors processing; the re-invoke serves the same page). +type multiPagePlannerMock struct { + *mockConnector + + mu sync.Mutex + groupIDs []string + resByID map[string]*v2.Resource + asks int + cursorServes int + spawns int + markerMissing bool // TypeScopedGrants marker absent on any request + answersScope string + answersPresent int // planning pages served with answers attached +} + +func (mc *multiPagePlannerMock) Validate(context.Context, *v2.ConnectorServiceValidateRequest, ...grpc.CallOption) (*v2.ConnectorServiceValidateResponse, error) { + return v2.ConnectorServiceValidateResponse_builder{ + Annotations: annotations.New(v2.SourceCacheCapability_builder{ + Mode: v2.SourceCacheCapability_MODE_READ_WRITE, + }.Build()), + }.Build(), nil +} + +const plannerPages = 6 + +func (mc *multiPagePlannerMock) ListGrants(_ context.Context, in *v2.GrantsServiceListGrantsRequest, _ ...grpc.CallOption) (*v2.GrantsServiceListGrantsResponse, error) { + if in.GetResource().GetId().GetResourceType() != typeScopedGroupRT.GetId() { + return v2.GrantsServiceListGrantsResponse_builder{}.Build(), nil + } + reqAnnos := annotations.Annotations(in.GetAnnotations()) + mc.mu.Lock() + defer mc.mu.Unlock() + if !reqAnnos.Contains(&v2.TypeScopedGrants{}) { + mc.markerMissing = true + } + + tok := in.GetPageToken() + if tok == "cursor" { + mc.cursorServes++ + grants := make([]*v2.Grant, 0, len(mc.groupIDs)) + for _, gid := range mc.groupIDs { + grants = append(grants, gt.NewGrant(mc.resByID[gid], "member", mc.resByID["user-00"].GetId())) + } + return v2.GrantsServiceListGrantsResponse_builder{List: grants}.Build(), nil + } + + page := 0 + if tok != "" { + if _, err := fmt.Sscanf(tok, "plan=%d", &page); err != nil { + return nil, fmt.Errorf("bad token %q", tok) + } + } + scope := fmt.Sprintf("chunks/page-%d", page) + + answersMsg := &v2.SourceCacheLookupAnswers{} + hasAnswers, err := reqAnnos.Pick(answersMsg) + if err != nil { + return nil, err + } + if !hasAnswers && reqAnnos.Contains(&v2.SourceCacheLookupOffer{}) { + mc.asks++ + return v2.GrantsServiceListGrantsResponse_builder{ + Annotations: annotations.New(sourcecache.AskProto([]sourcecache.Query{ + {RowKind: sourcecache.RowKindGrants, ScopeHash: scope}, + })), + }.Build(), nil + } + if hasAnswers { + mc.answersPresent++ + answers, answersErr := sourcecache.AnswersFromProto(answersMsg) + if answersErr != nil { + return nil, answersErr + } + for _, a := range answers { + if a.ScopeHash == scope { + mc.answersScope = scope + } + } + } + + // Serve the planning page: no rows; chain or spawn on the last page. + if page == plannerPages-1 { + mc.spawns++ + return v2.GrantsServiceListGrantsResponse_builder{ + Annotations: annotations.New(v2.SpawnCursors_builder{PageTokens: []string{"cursor"}}.Build()), + }.Build(), nil + } + return v2.GrantsServiceListGrantsResponse_builder{ + NextPageToken: fmt.Sprintf("plan=%d", page+1), + }.Build(), nil +} + +func TestSourceCacheContinuation_MultiPagePlannerBouncesPerRequest(t *testing.T) { + ctx, err := logging.Init(context.Background()) + require.NoError(t, err) + tmpDir := t.TempDir() + + mc := &multiPagePlannerMock{ + mockConnector: newMockConnector(), + resByID: map[string]*v2.Resource{}, + } + mc.rtDB = append(mc.rtDB, typeScopedGroupRT, userResourceType) + u, err := rs.NewUserResource("user-00", userResourceType, "user-00", nil, rs.WithAnnotation(&v2.SkipEntitlementsAndGrants{})) + require.NoError(t, err) + mc.AddResource(ctx, u) + mc.resByID["user-00"] = u + for i := 0; i < 2; i++ { + gid := fmt.Sprintf("group-%02d", i) + g, err := rs.NewGroupResource(gid, typeScopedGroupRT, gid, nil) + require.NoError(t, err) + mc.AddResource(ctx, g) + mc.resByID[gid] = g + mc.groupIDs = append(mc.groupIDs, gid) + mc.entDB[gid] = []*v2.Entitlement{mkMemberEnt(g)} + } + + // Cold: no offer, no asks; the planner walks its pages and spawns. + sync1 := filepath.Join(tmpDir, "sync1.c1z") + require.NoError(t, runContinuationSync(ctx, t, mc, sync1, "", tmpDir, 0)) + require.Zero(t, mc.asks) + require.Equal(t, 1, mc.spawns) + require.Equal(t, 1, mc.cursorServes) + grants1 := listGrantsInFile(ctx, t, sync1) + require.Len(t, grants1, 2) + + // Warm: every planning page asks once — 6 bounces in ONE action, + // exceeding the per-request cap of 4. Must complete: each page-token + // advance is a fresh request. + sync2 := filepath.Join(tmpDir, "sync2.c1z") + require.NoError(t, runContinuationSync(ctx, t, mc, sync2, sync1, tmpDir, 0), + "a multi-page planner asking once per page must not trip the per-request bounce cap") + require.Equal(t, plannerPages, mc.asks, "one ask per planning page") + require.Equal(t, plannerPages, mc.answersPresent, "every re-invoked planning page carries answers") + require.NotEmpty(t, mc.answersScope, "answers must cover the asked scope") + require.False(t, mc.markerMissing, "TypeScopedGrants routing marker must survive on every request, including re-invokes") + require.Equal(t, 2, mc.spawns, "the asking planner page spawns exactly once per sync") + require.Equal(t, 2, mc.cursorServes, "spawned cursor executes exactly once per sync") + requireSameGrantIDs(t, grants1, listGrantsInFile(ctx, t, sync2), "warm planner sync vs cold") +} + +// fatEtagLookup finds every scope with the same (large) etag. +type fatEtagLookup struct{ etag string } + +func (f fatEtagLookup) LookupPreviousSourceCache(context.Context, sourcecache.RowKind, string) (sourcecache.Entry, bool, error) { + return sourcecache.Entry{ETag: f.etag}, true, nil +} + +// TestSourceCacheContinuation_AnswerBudgetDegradesToCold pins the answer +// size budget's failure mode: one ask whose FOUND etags total more than +// sourceCacheAnswerBudget. Answers accumulate across bounces and the +// budget only shrinks, so an over-budget answer left absent could never +// be resolved by a re-ask — it must instead be degraded to an explicit +// not-found (the scope goes cold) and the request must complete in one +// answered turn, not livelock into the bounce cap. +func TestSourceCacheContinuation_AnswerBudgetDegradesToCold(t *testing.T) { + ctx, err := logging.Init(context.Background()) + require.NoError(t, err) + + const queryCount = 40 + etag := strings.Repeat("e", 65536) // the wire cap for one etag + fitting := sourceCacheAnswerBudget / len(etag) + require.Less(t, fitting, queryCount, "test must overflow the budget") + + s := &syncer{ + state: newState(), + sourceCache: syncerSourceCache{ + prev: struct{ dotc1z.SourceCacheStore }{}, // non-nil: warm + lookup: fatEtagLookup{etag: etag}, + }, + } + + queries := make([]sourcecache.Query, 0, queryCount) + for i := 0; i < queryCount; i++ { + queries = append(queries, sourcecache.Query{RowKind: sourcecache.RowKindGrants, ScopeHash: fmt.Sprintf("scope-%03d", i)}) + } + + turns := 0 + err = s.withSourceCacheContinuation(ctx, "grants", func(extra annotations.Annotations, attempt int) (listAttempt, error) { + turns++ + if attempt == 0 { + return listAttempt{annos: annotations.New(sourcecache.AskProto(queries))}, nil + } + answersMsg := &v2.SourceCacheLookupAnswers{} + hasAnswers, pickErr := extra.Pick(answersMsg) + require.NoError(t, pickErr) + require.True(t, hasAnswers) + answers, convErr := sourcecache.AnswersFromProto(answersMsg) + require.NoError(t, convErr) + require.Len(t, answers, queryCount, "every asked query must be answered; an absent answer can never resolve later") + found := 0 + for _, a := range answers { + if a.Found { + found++ + } + } + require.Equal(t, fitting, found, "found answers up to the budget") + return listAttempt{}, nil + }) + require.NoError(t, err, "an over-budget answer set must degrade to cold, not die at the bounce cap") + require.Equal(t, 2, turns, "one ask turn + one answered turn") + + stats := s.state.SourceCacheStatsSnapshot() + require.NotNil(t, stats) + require.Equal(t, int64(fitting), stats.LookupAnsweredFound) + require.Equal(t, int64(queryCount-fitting), stats.LookupAnswersTruncated) + require.Zero(t, stats.LookupAnsweredNotFound, "budget degradation counts as truncated, not as a genuine miss") +} + +func TestSourceCacheContinuation_AskWithSpawnCursorsFails(t *testing.T) { + ctx, err := logging.Init(context.Background()) + require.NoError(t, err) + tmpDir := t.TempDir() + mc := newContinuationMockConnector(t, 1) + + sync1 := filepath.Join(tmpDir, "sync1.c1z") + require.NoError(t, runContinuationSync(ctx, t, mc, sync1, "", tmpDir, 0)) + + mc.askWithSpawn = true + err = runContinuationSync(ctx, t, mc, filepath.Join(tmpDir, "sync2.c1z"), sync1, tmpDir, 0) + require.Error(t, err) + require.Contains(t, err.Error(), "no spawned cursors", + "SpawnCursors alongside an ask is a connector bug and must fail loudly") +} + +func TestSourceCacheContinuation_AskWithRowsFails(t *testing.T) { + ctx, err := logging.Init(context.Background()) + require.NoError(t, err) + tmpDir := t.TempDir() + mc := newContinuationMockConnector(t, 1) + + sync1 := filepath.Join(tmpDir, "sync1.c1z") + require.NoError(t, runContinuationSync(ctx, t, mc, sync1, "", tmpDir, 0)) + + mc.askWithRows = true + err = runContinuationSync(ctx, t, mc, filepath.Join(tmpDir, "sync2.c1z"), sync1, tmpDir, 0) + require.Error(t, err) + require.Contains(t, err.Error(), "must carry ONLY the ask") +} + +func TestSourceCacheContinuation_AskWithoutOfferFails(t *testing.T) { + ctx, err := logging.Init(context.Background()) + require.NoError(t, err) + tmpDir := t.TempDir() + mc := newContinuationMockConnector(t, 1) + mc.askNoOffer = true + + // Cold sync: no previous c1z → no offer → the ask is a protocol + // violation and must fail, not be misread as an empty page. + err = runContinuationSync(ctx, t, mc, filepath.Join(tmpDir, "sync1.c1z"), "", tmpDir, 0) + require.Error(t, err) + require.Contains(t, err.Error(), "no offer") +} diff --git a/pkg/sync/source_cache_equivalence_test.go b/pkg/sync/source_cache_equivalence_test.go new file mode 100644 index 000000000..41afd3dd5 --- /dev/null +++ b/pkg/sync/source_cache_equivalence_test.go @@ -0,0 +1,1827 @@ +package sync //nolint:revive,nolintlint // we can't change the package name for backwards compatibility + +// Replay-equivalence differential harness. +// +// THE PROPERTY: for any upstream state, a warm sync (replaying from the +// previous sync's c1z) must produce a c1z semantically identical to a +// cold sync of the same upstream state. This is the contract stated in +// pkg/sourcecache ("a cached sync must reproduce what a full resync +// would produce"), enforced here as a differential test instead of +// relying on humans to remember every response-row side effect. +// +// SHAPE: a fixed, feature-complete topology exercising every feature +// that interacts with replay — +// - parent/child resources (ChildResourceType scheduling), +// - grant expansion (GrantExpandable + needs_expansion re-arming), +// - external-resource matching (MatchID and MatchAll, with the +// external c1z churning UNDER a 304 of the internal scope — the one +// shape where a missed re-arm is observable), +// - InsertResourceGrants (grant-driven resource insertion), +// - exclusion-group annotations on replayed entitlements, +// - both delta tombstone channels (canonical grant ids on the replay +// annotation; bare principal ids on the scope annotation), +// - 304 rounds, delta-overlay rounds, cold refetch (validator +// rotation), zero-row scopes, and scopes appearing mid-chain — +// mutated across rounds by a forced schedule (guaranteeing each behavior +// occurs regardless of seed) plus seeded random churn (interleavings). +// The warm side CHAINS: round r replays from round r-1's warm output, so +// decay compounds instead of hiding. +// +// ANTI-VACUITY: the harness fails itself if it did not actually exercise +// what it claims — per-round and per-scenario counters assert 304s, +// overlays, cold refetches, both tombstone channels, child fetches, and +// the presence of expansion-derived, match-transformed, and +// grant-inserted artifacts in the outputs. A trivially-green run is +// treated as a harness bug. +// +// COMPARISON: full semantic diff of both files — resource types, +// resources, entitlements, grants, all fields, annotations normalized by +// sort — read through MULTIPLE paths (primary listings, by-parent +// listings, by-entitlement-resource grant listings) so replay-time index +// synthesis bugs surface, not just primary-row differences. +// +// Also compared, beyond the v2 read surface: the source-cache MANIFEST +// and per-scope INDEX COUNTS of both files. These are invisible to v2 +// reads but poison FUTURE syncs' replays — a warm sync could produce +// perfect v2 data while writing wrong stamps or phantom manifest +// entries, and a purely v2-level comparison would pass until the damage +// detonated rounds later. Row listings are also duplicate-checked (the +// deduping snapshot maps would otherwise hide pagination double-yields). +// +// Round 2's warm sync is additionally CRASHED mid-grants-phase (injected +// connector error after other scopes replayed) and resumed into the same +// file, asserting resume-not-restart — so page re-runs over partially +// replayed state (replay idempotency, checkpoint interplay) are part of +// the compared output, every scenario. +// +// EXPLICITLY NOT COVERED HERE (do not mistake green for proof of these): +// - error-outcome parity (covered by the dedicated exclusion-group +// parity test in source_cache_sideeffects_test.go); +// - the ask/answer lookup continuation topology (covered by +// source_cache_continuation_test.go / _e2e_test.go); +// - SpawnCursors / type-scoped grants (spawn_cursors_test.go, +// type_scoped_grants_test.go); +// - compaction in the warm chain (covered separately by +// pkg/synccompactor's TestCompactedFileIsColdReplaySource — an import +// cycle keeps the compactor out of this package; the contract is that +// compacted artifacts carry NO manifest entries and replay cold); +// - concurrent syncs; SQLite (source cache is Pebble-only); +// - delta tombstones combined with external-match annotations (a +// documented contract limitation — the model deliberately never +// generates that combination). +// +// The model DELIBERATELY overlaps MatchID and MatchAll on one +// entitlement (both resolving the same principals): the harness's first +// run caught that colliding transformed grants were resolved by store +// iteration order, which differs between cold syncs (iterating source +// grants) and warm syncs (iterating replayed transformed grants). +// processGrantsWithExternalPrincipals now resolves collisions by rule +// specificity; this topology pins that. + +import ( + "context" + "errors" + "fmt" + "math/rand" + "path/filepath" + "sort" + "strings" + stdsync "sync" + "testing" + + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/encoding/prototext" + "google.golang.org/protobuf/proto" + "google.golang.org/protobuf/types/known/anypb" + + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + "github.com/conductorone/baton-sdk/pkg/annotations" + "github.com/conductorone/baton-sdk/pkg/dotc1z" + "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" + pebbleengine "github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble" + "github.com/conductorone/baton-sdk/pkg/logging" + "github.com/conductorone/baton-sdk/pkg/sourcecache" + "github.com/conductorone/baton-sdk/pkg/types" + et "github.com/conductorone/baton-sdk/pkg/types/entitlement" + gt "github.com/conductorone/baton-sdk/pkg/types/grant" + rs "github.com/conductorone/baton-sdk/pkg/types/resource" +) + +// --------------------------------------------------------------------- +// Upstream model +// --------------------------------------------------------------------- + +var ( + equivOrgRT = v2.ResourceType_builder{Id: "eq_org", DisplayName: "Org"}.Build() + equivProjectRT = v2.ResourceType_builder{Id: "eq_project", DisplayName: "Project"}.Build() + equivRepoRT = v2.ResourceType_builder{Id: "eq_repo", DisplayName: "Repo"}.Build() + // equivTeamRT is TYPE-SCOPED: no per-resource grants pages; one + // planner call spawns chunk cursors (SpawnCursors), each chunk a + // scope covering two teams — the Entra/Okta planner shape. + equivTeamRT = v2.ResourceType_builder{ + Id: "eq_team", + DisplayName: "Team", + Annotations: annotations.New(&v2.TypeScopedGrants{}), + }.Build() +) + +type equivGroup struct { + id string + members map[string]bool + + // version is the grants scope's validator generation, bumped at most + // ONCE per mutation round (dirty guards it) so a round with several + // membership changes still reads as exactly one generation behind — + // the delta-overlay shape. delta* is the coalesced diff between + // version-1 and version (nil when the group was untouched last + // round). canonicalTombstones selects which delta channel removals + // ride: canonical grant ids on the replay annotation, or bare + // principal ids on the scope annotation. + version int + dirty bool + deltaAdds []string + deltaRemoves []string + canonicalTombstones bool + + // exclusionDefault attaches a (valid, single-member) exclusion-group + // default annotation to the member entitlement, so replayed + // entitlement rows carry exclusion state through validation. + exclusionDefault bool +} + +type equivOrg struct { + id string + projects map[string]string // project id -> display name + // projectsVersion is the per-org project-listing scope generation. + projectsVersion int + // repoGrants: repo id -> user id holding "admin". The org's grants + // page carries InsertResourceGrants; repoVersion is its generation + // (fresh-on-change, 304 when unchanged — no delta). + repoGrants map[string]string + repoVersion int +} + +type equivModel struct { + users map[string]bool + usersVersion int + + orgs map[string]*equivOrg + orgsVersion int + + groups map[string]*equivGroup + groupsVersion int + + // extGroup's grants page emits ExternalResourceMatchID grants (one + // per entry of extMatchIDs) plus one ExternalResourceMatchAll(USER) + // grant. extVersion is that scope's generation. extUsers is the + // EXTERNAL c1z's content — mutating it WITHOUT bumping extVersion is + // the divergence trap for the match re-arm path. + extGroupID string + extMatchIDs []string + extVersion int + extUsers map[string]bool + extDirty bool // external c1z must be rebuilt before the next sync + + // Expansion: expandableParent's grants scope permanently contains an + // expandable membership of expandableChild, so members of the child + // derive grants on the parent. Churning the child changes expansion + // output while the parent's scope 304s. + expandableParent string + expandableChild string + + // multiPageGroup's grants scope serves multi-page rounds: fresh + // fetches AND delta overlays span two pages, with the validator only + // on the final page and tombstones riding the final page's scope + // annotation (the page-spanning delta contract). + multiPageGroup string + + // teams (type-scoped grants): team id -> members. Teams are fixed; + // only membership churns. teamChunkVersions[i] is chunk i's + // validator generation (chunk = two teams by sorted order); chunks + // are fresh-or-304 scopes, no deltas. + teams map[string]map[string]bool + teamChunkVersions []int +} + +// teamChunkOf returns the chunk index a team belongs to (two teams per +// chunk, by sorted team id). Teams are never added or removed, so the +// mapping is stable across rounds. +func (m *equivModel) teamChunkOf(teamID string) int { + ids := sortedKeys(m.teams) + for i, id := range ids { + if id == teamID { + return i / 2 + } + } + panic("unknown team " + teamID) +} + +func (m *equivModel) teamsInChunk(chunk int) []string { + ids := sortedKeys(m.teams) + var out []string + for i, id := range ids { + if i/2 == chunk { + out = append(out, id) + } + } + return out +} + +func (m *equivModel) mutateTeam(teamID, uid string, add bool) { + members := m.teams[teamID] + if add == members[uid] { + return + } + if add { + members[uid] = true + } else { + delete(members, uid) + } + m.teamChunkVersions[m.teamChunkOf(teamID)]++ +} + +func newEquivModel() *equivModel { + m := &equivModel{ + users: map[string]bool{}, + orgs: map[string]*equivOrg{}, + groups: map[string]*equivGroup{}, + + extGroupID: "gext", + extMatchIDs: []string{"ext-0", "ext-1"}, + extUsers: map[string]bool{"ext-0": true, "ext-1": true, "ext-2": true}, + extDirty: true, + + expandableParent: "g0", + expandableChild: "g3", + multiPageGroup: "g1", + + teams: map[string]map[string]bool{ + "t0": {"u0": true, "u1": true}, + "t1": {"u2": true}, + "t2": {"u3": true, "u4": true}, + "t3": {"u9": true}, + }, + teamChunkVersions: []int{0, 0}, + } + for i := 0; i < 10; i++ { + m.users[fmt.Sprintf("u%d", i)] = true + } + m.orgs["org0"] = &equivOrg{ + id: "org0", + projects: map[string]string{"p0": "Project 0", "p1": "Project 1"}, + repoGrants: map[string]string{"repo0": "u0"}, + } + m.orgs["org1"] = &equivOrg{ + id: "org1", + projects: map[string]string{"p2": "Project 2", "p3": "Project 3"}, + repoGrants: map[string]string{}, // zero-row scope: entry must still persist + } + m.groups["g0"] = &equivGroup{id: "g0", members: map[string]bool{"u0": true, "u1": true}, canonicalTombstones: true, exclusionDefault: true} + m.groups["g1"] = &equivGroup{id: "g1", members: map[string]bool{"u2": true, "u3": true}, exclusionDefault: true} + m.groups["g2"] = &equivGroup{id: "g2", members: map[string]bool{"u4": true}} + m.groups["g3"] = &equivGroup{id: "g3", members: map[string]bool{"u5": true, "u6": true}} + m.groups[m.extGroupID] = &equivGroup{id: m.extGroupID, members: map[string]bool{}} + return m +} + +// clearDeltas resets last round's diffs; called at the start of each +// mutation round so overlays only ever describe one round's changes. +func (m *equivModel) clearDeltas() { + for _, g := range m.groups { + g.deltaAdds, g.deltaRemoves = nil, nil + g.dirty = false + } + m.extDirty = false +} + +func (g *equivGroup) touch() { + if !g.dirty { + g.dirty = true + g.version++ + } +} + +func (m *equivModel) addMember(gid, uid string) { + g := m.groups[gid] + if g.members[uid] { + return + } + g.members[uid] = true + g.touch() + g.deltaAdds = append(g.deltaAdds, uid) +} + +func (m *equivModel) removeMember(gid, uid string) { + g := m.groups[gid] + if !g.members[uid] { + return + } + delete(g.members, uid) + g.touch() + g.deltaRemoves = append(g.deltaRemoves, uid) +} + +func sortedKeys[V any](in map[string]V) []string { + out := make([]string, 0, len(in)) + for k := range in { + out = append(out, k) + } + sort.Strings(out) + return out +} + +// mutate applies round r's forced schedule plus seeded churn. The forced +// schedule guarantees every replay behavior occurs at least once per +// scenario regardless of seed; see the round comments. +func (m *equivModel) mutate(round int, rng *rand.Rand) { + m.clearDeltas() + switch round { + case 1: + // Canonical-id tombstone (g0), principal-id tombstone on the + // MULTI-PAGE overlay group (g1), overlay add (g2), project + // rename (cold refetch of a resources scope), repo grant add + // (fresh InsertResourceGrants page), new external user + match + // (external file rebuild + fresh page). + m.removeMember("g0", "u1") + m.removeMember("g1", "u3") + m.addMember("g1", "u9") + m.addMember("g2", "u7") + m.orgs["org0"].projects["p0"] = "Project 0 (renamed)" + m.orgs["org0"].projectsVersion++ + m.orgs["org0"].repoGrants["repo1"] = "u2" + m.orgs["org0"].repoVersion++ + m.extUsers["ext-3"] = true + m.extMatchIDs = append(m.extMatchIDs, "ext-3") + m.extVersion++ + m.extDirty = true + // Team churn: chunk 0 refetches, chunk 1 must 304. + m.mutateTeam("t0", "u5", true) + case 2: + // THE DIVERGENCE TRAP: the external c1z changes while the + // internal match scope's validator does NOT rotate (the internal + // source grants are unchanged, so the connector legitimately + // 304s). Cold output reflects the new external set; a warm sync + // that fails to re-run match processing keeps stale transformed + // grants. Also: user-set refetch (multi-page fresh), repo grant + // removal, and expansion input churn under the parent's 304. + // The RUNNER additionally crashes and resumes this round's warm + // sync mid-grants-phase. + delete(m.extUsers, "ext-1") + m.extDirty = true + m.users["u10"] = true + m.usersVersion++ + delete(m.orgs["org0"].repoGrants, "repo1") + m.orgs["org0"].repoVersion++ + m.addMember(m.expandableChild, "u8") + m.removeMember(m.expandableChild, "u5") + case 3: + // Deletions: a project, a WHOLE group, and a user vanish (their + // rows must vanish from the warm output exactly as from a cold + // one — rows only enter a sync via fresh pages or replay, so a + // leftover here means a replay copied outside its scope). A new + // group also appears mid-chain (cold scopes while everything + // else replays), plus seeded churn. + delete(m.orgs["org0"].projects, "p0") + m.orgs["org0"].projectsVersion++ + delete(m.groups, "g2") + m.groupsVersion++ + delete(m.users, "u7") // was only ever a member of g2 + m.usersVersion++ + ng := &equivGroup{id: "g4", members: map[string]bool{"u9": true}, canonicalTombstones: rng.Intn(2) == 0} + m.groups[ng.id] = ng + m.mutateTeam("t0", "u5", false) + m.randomChurn(rng) + default: + // Round 4: deliberate no-op — every scope must 304 and the + // output must STILL match a cold sync (chain-decay probe). + } +} + +func (m *equivModel) randomChurn(rng *rand.Rand) { + users := sortedKeys(m.users) + for _, gid := range sortedKeys(m.groups) { + if gid == m.extGroupID { + continue // never mix delta churn into the match scope + } + g := m.groups[gid] + switch rng.Intn(3) { + case 0: // add an absent user + for _, u := range users { + if !g.members[u] { + m.addMember(gid, u) + break + } + } + case 1: // remove a present member (keep expansion child non-empty) + members := sortedKeys(g.members) + if len(members) > 1 { + m.removeMember(gid, members[rng.Intn(len(members))]) + } + default: // untouched: must 304 + } + } +} + +// --------------------------------------------------------------------- +// Connector +// --------------------------------------------------------------------- + +type equivCounters struct { + pages304 int + pagesOverlay int // final (or only) page of an overlay round + pagesOverlayInterim int // non-final overlay page (no validator yet) + pagesColdStale int // lookup hit but validator stale -> full refetch + pagesFresh int // lookup miss -> full fetch (final page) + pagesFreshInterim int // non-final fresh page (no validator yet) + tombsCanonical int + tombsPrincipal int + childPageCalls int + injectedFailures int + plannerPages int // type-scoped planning calls (SpawnCursors emitters) + chunkPages304 int // spawned chunk cursors that replayed + chunkPagesFresh int // spawned chunk cursors fetched fresh + askPages int // continuation phase-1 responses (lookup asks) +} + +// equivConnector serves the model. forceCold ignores the lookup and +// always emits full fresh pages (still scope-annotated — stamps are +// invisible to the comparator), which is exactly what a from-scratch +// resync sees. +type equivConnector struct { + *mockConnector + + mu stdsync.Mutex + model *equivModel + lookup sourcecache.Lookup + forceCold bool + // continuation switches the lookup topology: instead of the direct + // SetSourceCache lookup, lookups are served from request-delivered + // SourceCacheLookupAnswers and deferred via SourceCacheLookupAsk + // responses — the single-shot (Lambda) transport, played by hand. + // The connector must be wrapped in equivContinuationClient so the + // syncer does not see a SetLookup implementation. + continuation bool + counters equivCounters + // failNext injects one transient failure per key ("grants:"), + // simulating a crash mid-sync; the runner resumes the same file and + // the resumed output is what gets compared. + failNext map[string]bool +} + +// equivContinuationClient hides the embedded SetSourceCache from the +// syncer's sourcecache.SetLookup type assertion, making the connector a +// single-shot transport: the syncer attaches SourceCacheLookupOffer to +// requests and the connector answers with asks. +type equivContinuationClient struct { + *equivConnector +} + +// SetSourceCache shadows the embedded method with a different signature +// so equivContinuationClient does NOT satisfy sourcecache.SetLookup. +func (equivContinuationClient) SetSourceCache() {} + +func (c *equivConnector) injectGrantsFailure(resourceID string) { + c.mu.Lock() + defer c.mu.Unlock() + if c.failNext == nil { + c.failNext = map[string]bool{} + } + c.failNext["grants:"+resourceID] = true +} + +func newEquivConnector(model *equivModel, forceCold bool) *equivConnector { + mc := &equivConnector{mockConnector: newMockConnector(), model: model, forceCold: forceCold} + mc.rtDB = []*v2.ResourceType{equivOrgRT, equivProjectRT, equivRepoRT, equivTeamRT, groupResourceType, userResourceType} + return mc +} + +func (c *equivConnector) SetSourceCache(_ context.Context, lookup sourcecache.Lookup) { + c.mu.Lock() + defer c.mu.Unlock() + c.lookup = lookup +} + +func (c *equivConnector) Validate(context.Context, *v2.ConnectorServiceValidateRequest, ...grpc.CallOption) (*v2.ConnectorServiceValidateResponse, error) { + return v2.ConnectorServiceValidateResponse_builder{ + Annotations: annotations.New(v2.SourceCacheCapability_builder{ + Mode: v2.SourceCacheCapability_MODE_READ_WRITE, + }.Build()), + }.Build(), nil +} + +func equivEtag(version int) string { return fmt.Sprintf("v%d", version) } + +// revalidate consults the lookup for (kind, scope) and classifies the +// round: fresh (miss), current (304), overlay-eligible (exactly one +// generation behind), or stale (older: full refetch). forceCold always +// reports fresh. +type equivVerdict int + +const ( + equivFresh equivVerdict = iota + equivCurrent + equivOneBehind + equivStale +) + +// requestLookup resolves the lookup for one request. Direct topology: +// the SetSourceCache-installed lookup. Continuation topology: a +// per-request ContinuationLookup built from the request's answers (nil +// when the request carries neither offer nor answers — cold requests +// must not ask). The returned *ContinuationLookup is non-nil only when +// deferred lookups must be converted into an ask response. +func (c *equivConnector) requestLookup(reqAnnos []*anypb.Any) (sourcecache.Lookup, *sourcecache.ContinuationLookup, error) { + if !c.continuation { + c.mu.Lock() + lookup := c.lookup + c.mu.Unlock() + if lookup == nil { + lookup = sourcecache.NoopLookup{} + } + return lookup, nil, nil + } + annos := annotations.Annotations(reqAnnos) + answersMsg := &v2.SourceCacheLookupAnswers{} + hasAnswers, err := annos.Pick(answersMsg) + if err != nil { + return nil, nil, err + } + if !hasAnswers && !annos.Contains(&v2.SourceCacheLookupOffer{}) { + return sourcecache.NoopLookup{}, nil, nil + } + var answers []sourcecache.Answer + if hasAnswers { + answers, err = sourcecache.AnswersFromProto(answersMsg) + if err != nil { + return nil, nil, err + } + } + cl := sourcecache.NewContinuationLookup(answers) + return cl, cl, nil +} + +// askAnnos converts a deferred lookup into the ask response's +// annotations, counting the phase-1 turn. +func (c *equivConnector) askAnnos(cl *sourcecache.ContinuationLookup) annotations.Annotations { + c.count(func(ct *equivCounters) { ct.askPages++ }) + return annotations.New(sourcecache.AskProto(cl.Asked())) +} + +// isDeferred reports whether err is a deferred continuation lookup that +// must become an ask response (only possible when cl is non-nil). +func isDeferred(cl *sourcecache.ContinuationLookup, err error) bool { + return cl != nil && err != nil && errors.Is(err, sourcecache.ErrLookupDeferred) +} + +func (c *equivConnector) revalidate(ctx context.Context, lookup sourcecache.Lookup, kind sourcecache.RowKind, scope string, version int) (equivVerdict, error) { + if c.forceCold { + return equivFresh, nil + } + entry, found, err := lookup.LookupPreviousSourceCache(ctx, kind, scope) + if err != nil { + return equivFresh, err + } + if !found { + return equivFresh, nil + } + switch entry.ETag { + case equivEtag(version): + return equivCurrent, nil + case equivEtag(version - 1): + return equivOneBehind, nil + default: + return equivStale, nil + } +} + +func (c *equivConnector) count(f func(*equivCounters)) { + c.mu.Lock() + defer c.mu.Unlock() + f(&c.counters) +} + +func (c *equivConnector) snapshotCounters() equivCounters { + c.mu.Lock() + defer c.mu.Unlock() + return c.counters +} + +func equivScopeAnnos(scope string, version int) annotations.Annotations { + return annotations.New(v2.SourceCacheScope_builder{ + ScopeHash: sourcecache.HashScope(scope), + Etag: equivEtag(version), + }.Build()) +} + +func equivReplayAnnos(scope string, version int) annotations.Annotations { + return annotations.New(v2.SourceCacheReplay_builder{ + ScopeHash: sourcecache.HashScope(scope), + Etag: equivEtag(version), + }.Build()) +} + +// --- resource builders (deterministic; shared by fresh pages) --------- + +func equivOrgResource(id string) *v2.Resource { + r, err := rs.NewResource("Org "+id, equivOrgRT, id, + rs.WithAnnotation(v2.ChildResourceType_builder{ResourceTypeId: equivProjectRT.GetId()}.Build()), + rs.WithAnnotation(&v2.SkipEntitlements{}), + ) + if err != nil { + panic(err) + } + return r +} + +func equivProjectResource(orgID, id, displayName string) *v2.Resource { + r, err := rs.NewResource(displayName, equivProjectRT, id, + rs.WithParentResourceID(v2.ResourceId_builder{ResourceType: equivOrgRT.GetId(), Resource: orgID}.Build()), + rs.WithAnnotation(&v2.SkipEntitlementsAndGrants{}), + ) + if err != nil { + panic(err) + } + return r +} + +func equivGroupResource(id string) *v2.Resource { + r, err := rs.NewGroupResource(id, groupResourceType, id, nil) + if err != nil { + panic(err) + } + return r +} + +func equivUserResource(id string) *v2.Resource { + r, err := rs.NewUserResource(id, userResourceType, id, nil, rs.WithAnnotation(&v2.SkipEntitlementsAndGrants{})) + if err != nil { + panic(err) + } + return r +} + +func equivRepoResource(id string) *v2.Resource { + r, err := rs.NewResource("Repo "+id, equivRepoRT, id, rs.WithAnnotation(&v2.SkipEntitlementsAndGrants{})) + if err != nil { + panic(err) + } + return r +} + +func equivTeamResource(id string) *v2.Resource { + r, err := rs.NewResource("Team "+id, equivTeamRT, id) + if err != nil { + panic(err) + } + return r +} + +func equivTeamGrant(tid, uid string) *v2.Grant { + return gt.NewGrant(equivTeamResource(tid), "member", + v2.ResourceId_builder{ResourceType: userResourceType.GetId(), Resource: uid}.Build()) +} + +func equivMemberEnt(g *v2.Resource, exclusionDefault bool) *v2.Entitlement { + ent := et.NewAssignmentEntitlement(g, "member", et.WithGrantableTo(groupResourceType, userResourceType)) + ent.SetSlug("member") + if exclusionDefault { + annos := annotations.Annotations(ent.GetAnnotations()) + annos.Update(v2.EntitlementExclusionGroup_builder{ + ExclusionGroupId: "eg-" + g.GetId().GetResource(), + IsDefault: true, + }.Build()) + ent.SetAnnotations(annos) + } + return ent +} + +func equivMemberGrant(gid, uid string) *v2.Grant { + return gt.NewGrant(equivGroupResource(gid), "member", + v2.ResourceId_builder{ResourceType: userResourceType.GetId(), Resource: uid}.Build()) +} + +// --- list RPCs --------------------------------------------------------- + +func (c *equivConnector) ListResources(ctx context.Context, in *v2.ResourcesServiceListResourcesRequest, opts ...grpc.CallOption) (*v2.ResourcesServiceListResourcesResponse, error) { + lookup, cl, err := c.requestLookup(in.GetAnnotations()) + if err != nil { + return nil, err + } + resp, err := c.listResourcesInner(ctx, lookup, in) + if isDeferred(cl, err) { + return v2.ResourcesServiceListResourcesResponse_builder{ + Annotations: c.askAnnos(cl), + }.Build(), nil + } + return resp, err +} + +func (c *equivConnector) listResourcesInner(ctx context.Context, lookup sourcecache.Lookup, in *v2.ResourcesServiceListResourcesRequest) (*v2.ResourcesServiceListResourcesResponse, error) { + c.mu.Lock() + m := c.model + switch in.GetResourceTypeId() { + case equivOrgRT.GetId(): + version := m.orgsVersion + orgIDs := sortedKeys(m.orgs) + c.mu.Unlock() + return c.resourcesPage(ctx, lookup, "res:org", version, func() []*v2.Resource { + out := make([]*v2.Resource, 0, len(orgIDs)) + for _, id := range orgIDs { + out = append(out, equivOrgResource(id)) + } + return out + }) + case equivProjectRT.GetId(): + if in.GetParentResourceId().GetResource() == "" { + c.mu.Unlock() + return v2.ResourcesServiceListResourcesResponse_builder{}.Build(), nil + } + orgID := in.GetParentResourceId().GetResource() + org := m.orgs[orgID] + if org == nil { + c.mu.Unlock() + return v2.ResourcesServiceListResourcesResponse_builder{}.Build(), nil + } + version := org.projectsVersion + projects := make(map[string]string, len(org.projects)) + for k, v := range org.projects { + projects[k] = v + } + c.mu.Unlock() + c.count(func(ct *equivCounters) { ct.childPageCalls++ }) + return c.resourcesPage(ctx, lookup, "res:project:"+orgID, version, func() []*v2.Resource { + out := make([]*v2.Resource, 0, len(projects)) + for _, id := range sortedKeys(projects) { + out = append(out, equivProjectResource(orgID, id, projects[id])) + } + return out + }) + case groupResourceType.GetId(): + version := m.groupsVersion + groupIDs := sortedKeys(m.groups) + c.mu.Unlock() + return c.resourcesPage(ctx, lookup, "res:group", version, func() []*v2.Resource { + out := make([]*v2.Resource, 0, len(groupIDs)) + for _, id := range groupIDs { + out = append(out, equivGroupResource(id)) + } + return out + }) + case equivTeamRT.GetId(): + // Teams are UNSCOPED: listed fresh every sync (mixes never-scoped + // pages into an otherwise warm sync). + teamIDs := sortedKeys(m.teams) + c.mu.Unlock() + out := make([]*v2.Resource, 0, len(teamIDs)) + for _, id := range teamIDs { + out = append(out, equivTeamResource(id)) + } + return v2.ResourcesServiceListResourcesResponse_builder{List: out}.Build(), nil + case userResourceType.GetId(): + version := m.usersVersion + userIDs := sortedKeys(m.users) + c.mu.Unlock() + return c.usersPageMulti(ctx, lookup, in.GetPageToken(), version, userIDs) + default: // repo: insert-only, never listed + c.mu.Unlock() + return v2.ResourcesServiceListResourcesResponse_builder{}.Build(), nil + } +} + +// usersPageMulti serves the users scope as a MULTI-PAGE scope: fresh +// fetches span two pages, with the scope annotation on both but the +// validator only on the final page (the interim-page contract). A 304 is +// a single page like any other scope. +func (c *equivConnector) usersPageMulti(ctx context.Context, lookup sourcecache.Lookup, pageToken string, version int, userIDs []string) (*v2.ResourcesServiceListResourcesResponse, error) { + const scope = "res:user" + verdict, err := c.revalidate(ctx, lookup, sourcecache.RowKindResources, sourcecache.HashScope(scope), version) + if err != nil { + return nil, err + } + if verdict == equivCurrent { + c.count(func(ct *equivCounters) { ct.pages304++ }) + return v2.ResourcesServiceListResourcesResponse_builder{ + Annotations: equivReplayAnnos(scope, version), + }.Build(), nil + } + half := len(userIDs) / 2 + buildRows := func(ids []string) []*v2.Resource { + out := make([]*v2.Resource, 0, len(ids)) + for _, id := range ids { + out = append(out, equivUserResource(id)) + } + return out + } + if pageToken == "" { + c.count(func(ct *equivCounters) { ct.pagesFreshInterim++ }) + return v2.ResourcesServiceListResourcesResponse_builder{ + List: buildRows(userIDs[:half]), + Annotations: annotations.New(v2.SourceCacheScope_builder{ + ScopeHash: sourcecache.HashScope(scope), + // No etag: the validator arrives on the final page. + }.Build()), + NextPageToken: "users:2", + }.Build(), nil + } + c.count(func(ct *equivCounters) { + if verdict == equivFresh { + ct.pagesFresh++ + } else { + ct.pagesColdStale++ + } + }) + return v2.ResourcesServiceListResourcesResponse_builder{ + List: buildRows(userIDs[half:]), + Annotations: equivScopeAnnos(scope, version), + }.Build(), nil +} + +// resourcesPage implements the fresh/304/stale protocol for a resources +// scope (no deltas: resources scopes refetch fully on change). +func (c *equivConnector) resourcesPage(ctx context.Context, lookup sourcecache.Lookup, scope string, version int, build func() []*v2.Resource) (*v2.ResourcesServiceListResourcesResponse, error) { + verdict, err := c.revalidate(ctx, lookup, sourcecache.RowKindResources, sourcecache.HashScope(scope), version) + if err != nil { + return nil, err + } + if verdict == equivCurrent { + c.count(func(ct *equivCounters) { ct.pages304++ }) + return v2.ResourcesServiceListResourcesResponse_builder{ + Annotations: equivReplayAnnos(scope, version), + }.Build(), nil + } + c.count(func(ct *equivCounters) { + if verdict == equivFresh { + ct.pagesFresh++ + } else { + ct.pagesColdStale++ + } + }) + return v2.ResourcesServiceListResourcesResponse_builder{ + List: build(), + Annotations: equivScopeAnnos(scope, version), + }.Build(), nil +} + +func (c *equivConnector) ListEntitlements(ctx context.Context, in *v2.EntitlementsServiceListEntitlementsRequest, opts ...grpc.CallOption) (*v2.EntitlementsServiceListEntitlementsResponse, error) { + lookup, cl, err := c.requestLookup(in.GetAnnotations()) + if err != nil { + return nil, err + } + resp, err := c.listEntitlementsInner(ctx, lookup, in) + if isDeferred(cl, err) { + return v2.EntitlementsServiceListEntitlementsResponse_builder{ + Annotations: c.askAnnos(cl), + }.Build(), nil + } + return resp, err +} + +func (c *equivConnector) listEntitlementsInner( + ctx context.Context, + lookup sourcecache.Lookup, + in *v2.EntitlementsServiceListEntitlementsRequest, +) (*v2.EntitlementsServiceListEntitlementsResponse, error) { + rid := in.GetResource().GetId() + if rid.GetResourceType() == equivTeamRT.GetId() { + // Team entitlements are unscoped: served fresh every sync. + c.mu.Lock() + _, ok := c.model.teams[rid.GetResource()] + c.mu.Unlock() + if !ok { + return v2.EntitlementsServiceListEntitlementsResponse_builder{}.Build(), nil + } + return v2.EntitlementsServiceListEntitlementsResponse_builder{ + List: []*v2.Entitlement{equivMemberEnt(equivTeamResource(rid.GetResource()), false)}, + }.Build(), nil + } + if rid.GetResourceType() != groupResourceType.GetId() { + return v2.EntitlementsServiceListEntitlementsResponse_builder{}.Build(), nil + } + c.mu.Lock() + g := c.model.groups[rid.GetResource()] + c.mu.Unlock() + if g == nil { + return v2.EntitlementsServiceListEntitlementsResponse_builder{}.Build(), nil + } + scope := "ents:" + g.id + // Entitlement scopes never rotate after creation: warm rounds are + // pure 304s, which is exactly what keeps exclusion-group state on + // the replayed path. + verdict, err := c.revalidate(ctx, lookup, sourcecache.RowKindEntitlements, sourcecache.HashScope(scope), 0) + if err != nil { + return nil, err + } + if verdict == equivCurrent { + c.count(func(ct *equivCounters) { ct.pages304++ }) + return v2.EntitlementsServiceListEntitlementsResponse_builder{ + Annotations: equivReplayAnnos(scope, 0), + }.Build(), nil + } + c.count(func(ct *equivCounters) { ct.pagesFresh++ }) + return v2.EntitlementsServiceListEntitlementsResponse_builder{ + List: []*v2.Entitlement{equivMemberEnt(equivGroupResource(g.id), g.exclusionDefault)}, + Annotations: equivScopeAnnos(scope, 0), + }.Build(), nil +} + +func (c *equivConnector) ListGrants(ctx context.Context, in *v2.GrantsServiceListGrantsRequest, opts ...grpc.CallOption) (*v2.GrantsServiceListGrantsResponse, error) { + rid := in.GetResource().GetId() + c.mu.Lock() + if c.failNext["grants:"+rid.GetResource()] { + delete(c.failNext, "grants:"+rid.GetResource()) + c.counters.injectedFailures++ + c.mu.Unlock() + // codes.Internal: NOT in the syncer's retryable set (only + // Unavailable/DeadlineExceeded are), so the sync dies here — + // the process-crash analog the resume path exists for. + return nil, status.Error(codes.Internal, "injected crash: simulated mid-sync failure") + } + c.mu.Unlock() + lookup, cl, err := c.requestLookup(in.GetAnnotations()) + if err != nil { + return nil, err + } + var resp *v2.GrantsServiceListGrantsResponse + switch rid.GetResourceType() { + case groupResourceType.GetId(): + resp, err = c.groupGrantsPage(ctx, lookup, rid.GetResource(), in.GetPageToken()) + case equivOrgRT.GetId(): + resp, err = c.orgGrantsPage(ctx, lookup, rid.GetResource()) + case equivTeamRT.GetId(): + resp, err = c.teamGrantsPage(ctx, lookup, in.GetPageToken()) + default: + return v2.GrantsServiceListGrantsResponse_builder{}.Build(), nil + } + if isDeferred(cl, err) { + return v2.GrantsServiceListGrantsResponse_builder{ + Annotations: c.askAnnos(cl), + }.Build(), nil + } + return resp, err +} + +// teamGrantsPage is the type-scoped planner shape: the planning call +// (empty page token) emits SpawnCursors — one token per chunk of two +// teams — and each spawned cursor serves its chunk as a fresh-or-304 +// scope. +func (c *equivConnector) teamGrantsPage(ctx context.Context, lookup sourcecache.Lookup, pageToken string) (*v2.GrantsServiceListGrantsResponse, error) { + if pageToken == "" { + c.mu.Lock() + chunks := len(c.model.teamChunkVersions) + c.mu.Unlock() + c.count(func(ct *equivCounters) { ct.plannerPages++ }) + tokens := make([]string, 0, chunks) + for i := 0; i < chunks; i++ { + tokens = append(tokens, fmt.Sprintf("chunk:%d", i)) + } + return v2.GrantsServiceListGrantsResponse_builder{ + Annotations: annotations.New(v2.SpawnCursors_builder{PageTokens: tokens}.Build()), + }.Build(), nil + } + var chunk int + if _, err := fmt.Sscanf(pageToken, "chunk:%d", &chunk); err != nil { + return nil, fmt.Errorf("teamGrantsPage: bad page token %q: %w", pageToken, err) + } + c.mu.Lock() + m := c.model + version := m.teamChunkVersions[chunk] + teamIDs := m.teamsInChunk(chunk) + membersByTeam := make(map[string][]string, len(teamIDs)) + for _, tid := range teamIDs { + membersByTeam[tid] = sortedKeys(m.teams[tid]) + } + c.mu.Unlock() + + scope := fmt.Sprintf("grants:team-chunk:%d", chunk) + verdict, err := c.revalidate(ctx, lookup, sourcecache.RowKindGrants, sourcecache.HashScope(scope), version) + if err != nil { + return nil, err + } + if verdict == equivCurrent { + c.count(func(ct *equivCounters) { ct.chunkPages304++ }) + return v2.GrantsServiceListGrantsResponse_builder{ + Annotations: equivReplayAnnos(scope, version), + }.Build(), nil + } + c.count(func(ct *equivCounters) { ct.chunkPagesFresh++ }) + var rows []*v2.Grant + for _, tid := range teamIDs { + for _, uid := range membersByTeam[tid] { + rows = append(rows, equivTeamGrant(tid, uid)) + } + } + return v2.GrantsServiceListGrantsResponse_builder{ + List: rows, + Annotations: equivScopeAnnos(scope, version), + }.Build(), nil +} + +// groupGrantsPage: delta-scope protocol for group membership (and the +// external-match group, which is fresh-or-304 only — the model never +// mixes tombstones into it). The multiPageGroup serves its overlay +// rounds across TWO pages: page 1 carries the replay annotation +// (overlay, no validator) plus the first add; page 2 carries the scope +// annotation with the remaining adds, the tombstones, and the NEW +// validator — the page-spanning delta contract. +func (c *equivConnector) groupGrantsPage(ctx context.Context, lookup sourcecache.Lookup, gid string, pageToken string) (*v2.GrantsServiceListGrantsResponse, error) { + c.mu.Lock() + m := c.model + g := m.groups[gid] + if g == nil { + c.mu.Unlock() + return v2.GrantsServiceListGrantsResponse_builder{}.Build(), nil + } + isExt := gid == m.extGroupID + isExpandableParent := gid == m.expandableParent + isMultiPage := gid == m.multiPageGroup + expandableChild := m.expandableChild + version := g.version + members := sortedKeys(g.members) + deltaAdds := append([]string{}, g.deltaAdds...) + deltaRemoves := append([]string{}, g.deltaRemoves...) + canonical := g.canonicalTombstones + extMatchIDs := append([]string{}, m.extMatchIDs...) + if isExt { + version = m.extVersion + } + c.mu.Unlock() + + scope := "grants:" + gid + fullRows := func() []*v2.Grant { + var out []*v2.Grant + if isExt { + groupRes := equivGroupResource(gid) + for _, extID := range extMatchIDs { + out = append(out, gt.NewGrant(groupRes, "member", + v2.ResourceId_builder{ResourceType: userResourceType.GetId(), Resource: "placeholder-" + extID}.Build(), + gt.WithAnnotation(v2.ExternalResourceMatchID_builder{Id: extID}.Build()), + )) + } + out = append(out, gt.NewGrant(groupRes, "member", + v2.ResourceId_builder{ResourceType: userResourceType.GetId(), Resource: "placeholder-all"}.Build(), + gt.WithAnnotation(v2.ExternalResourceMatchAll_builder{ResourceType: v2.ResourceType_TRAIT_USER}.Build()), + )) + return out + } + for _, uid := range members { + out = append(out, equivMemberGrant(gid, uid)) + } + if isExpandableParent { + childRes := equivGroupResource(expandableChild) + childEnt := equivMemberEnt(childRes, false) + out = append(out, gt.NewGrant(equivGroupResource(gid), "member", childRes, + gt.WithAnnotation(v2.GrantExpandable_builder{EntitlementIds: []string{childEnt.GetId()}}.Build()), + )) + } + return out + } + + verdict, err := c.revalidate(ctx, lookup, sourcecache.RowKindGrants, sourcecache.HashScope(scope), version) + if err != nil { + return nil, err + } + switch verdict { + case equivCurrent: + c.count(func(ct *equivCounters) { ct.pages304++ }) + return v2.GrantsServiceListGrantsResponse_builder{ + Annotations: equivReplayAnnos(scope, version), + }.Build(), nil + case equivOneBehind: + if isExt || (len(deltaAdds) == 0 && len(deltaRemoves) == 0) { + // No delta journal for this transition: full refetch. + break + } + adds := make([]*v2.Grant, 0, len(deltaAdds)) + for _, uid := range deltaAdds { + adds = append(adds, equivMemberGrant(gid, uid)) + } + if isMultiPage { + if pageToken == "" { + // Overlay page 1: replay annotation (overlay, NO + // validator yet) plus the first add. + c.count(func(ct *equivCounters) { ct.pagesOverlayInterim++ }) + firstAdds := adds + if len(firstAdds) > 1 { + firstAdds = adds[:1] + } + return v2.GrantsServiceListGrantsResponse_builder{ + List: firstAdds, + Annotations: annotations.New(v2.SourceCacheReplay_builder{ + ScopeHash: sourcecache.HashScope(scope), + Overlay: true, + }.Build()), + NextPageToken: "ov:2", + }.Build(), nil + } + // Overlay final page: remaining adds + tombstones + the NEW + // validator, all on the scope annotation. + c.count(func(ct *equivCounters) { ct.pagesOverlay++ }) + var restAdds []*v2.Grant + if len(adds) > 1 { + restAdds = adds[1:] + } + finalAnno := v2.SourceCacheScope_builder{ + ScopeHash: sourcecache.HashScope(scope), + Etag: equivEtag(version), + }.Build() + if canonical { + ids := make([]string, 0, len(deltaRemoves)) + for _, uid := range deltaRemoves { + ids = append(ids, equivMemberGrant(gid, uid).GetId()) + } + finalAnno.SetDeletedIds(ids) + c.count(func(ct *equivCounters) { ct.tombsCanonical += len(ids) }) + } else { + finalAnno.SetDeletedPrincipalIds(deltaRemoves) + c.count(func(ct *equivCounters) { ct.tombsPrincipal += len(deltaRemoves) }) + } + return v2.GrantsServiceListGrantsResponse_builder{ + List: restAdds, + Annotations: annotations.New(finalAnno), + }.Build(), nil + } + c.count(func(ct *equivCounters) { ct.pagesOverlay++ }) + replay := v2.SourceCacheReplay_builder{ + ScopeHash: sourcecache.HashScope(scope), + Etag: equivEtag(version), + Overlay: true, + }.Build() + scopeAnno := v2.SourceCacheScope_builder{ + ScopeHash: sourcecache.HashScope(scope), + }.Build() + if canonical { + ids := make([]string, 0, len(deltaRemoves)) + for _, uid := range deltaRemoves { + ids = append(ids, equivMemberGrant(gid, uid).GetId()) + } + replay.SetDeletedIds(ids) + c.count(func(ct *equivCounters) { ct.tombsCanonical += len(ids) }) + } else { + scopeAnno.SetDeletedPrincipalIds(deltaRemoves) + c.count(func(ct *equivCounters) { ct.tombsPrincipal += len(deltaRemoves) }) + } + return v2.GrantsServiceListGrantsResponse_builder{ + List: adds, + Annotations: annotations.New(replay, scopeAnno), + }.Build(), nil + default: + } + c.count(func(ct *equivCounters) { + if verdict == equivFresh { + ct.pagesFresh++ + } else { + ct.pagesColdStale++ + } + }) + return v2.GrantsServiceListGrantsResponse_builder{ + List: fullRows(), + Annotations: equivScopeAnnos(scope, version), + }.Build(), nil +} + +// orgGrantsPage: the InsertResourceGrants scope (fresh-or-304). +func (c *equivConnector) orgGrantsPage(ctx context.Context, lookup sourcecache.Lookup, orgID string) (*v2.GrantsServiceListGrantsResponse, error) { + c.mu.Lock() + org := c.model.orgs[orgID] + if org == nil { + c.mu.Unlock() + return v2.GrantsServiceListGrantsResponse_builder{}.Build(), nil + } + version := org.repoVersion + repoGrants := make(map[string]string, len(org.repoGrants)) + for k, v := range org.repoGrants { + repoGrants[k] = v + } + c.mu.Unlock() + + scope := "grants:org:" + orgID + verdict, err := c.revalidate(ctx, lookup, sourcecache.RowKindGrants, sourcecache.HashScope(scope), version) + if err != nil { + return nil, err + } + if verdict == equivCurrent { + c.count(func(ct *equivCounters) { ct.pages304++ }) + return v2.GrantsServiceListGrantsResponse_builder{ + Annotations: equivReplayAnnos(scope, version), + }.Build(), nil + } + c.count(func(ct *equivCounters) { + if verdict == equivFresh { + ct.pagesFresh++ + } else { + ct.pagesColdStale++ + } + }) + rows := make([]*v2.Grant, 0, len(repoGrants)) + for _, repoID := range sortedKeys(repoGrants) { + rows = append(rows, gt.NewGrant(equivRepoResource(repoID), "admin", + v2.ResourceId_builder{ResourceType: userResourceType.GetId(), Resource: repoGrants[repoID]}.Build())) + } + annos := equivScopeAnnos(scope, version) + annos.Update(&v2.InsertResourceGrants{}) + return v2.GrantsServiceListGrantsResponse_builder{ + List: rows, + Annotations: annos, + }.Build(), nil +} + +// --------------------------------------------------------------------- +// External c1z +// --------------------------------------------------------------------- + +// buildExternalC1z materializes the model's external users into a fresh +// c1z (rebuilt whenever extUsers changes; both warm and cold syncs of a +// round read the same file). +func buildExternalC1z(ctx context.Context, t *testing.T, m *equivModel, path, tmpDir string) { + t.Helper() + mc := newMockConnector() + mc.rtDB = append(mc.rtDB, userResourceType, groupResourceType) + for _, extID := range sortedKeys(m.extUsers) { + _, err := mc.AddUserProfile(ctx, extID, map[string]any{}) + require.NoError(t, err) + } + syncer, err := NewSyncer(ctx, mc, WithC1ZPath(path), WithTmpDir(tmpDir)) + require.NoError(t, err) + require.NoError(t, syncer.Sync(ctx)) + require.NoError(t, syncer.Close(ctx)) +} + +// --------------------------------------------------------------------- +// Snapshot + comparison +// --------------------------------------------------------------------- + +type c1zSnapshot struct { + resourceTypes map[string]*v2.ResourceType + resources map[string]*v2.Resource // "rt/id" + entitlements map[string]*v2.Entitlement + grants map[string]*v2.Grant + // byParent: parent "rt/id" -> child resource keys via the by_parent + // index read path (cross-index consistency). + byParent map[string][]string + // grantsByEntResource: entitlement-resource "rt/id" -> grant ids via + // the by_entitlement_resource index read path. + grantsByEntResource map[string][]string + // manifest ("row_kind\x00scope_hash" -> etag) and scopeIndexCounts + // (row kind -> scope hash -> index entries): the source-cache state + // invisible to v2 reads but load-bearing for the NEXT sync's replay. + manifest map[string]string + scopeIndexCounts map[string]map[string]int + // syncRuns is the number of sync runs in the file — the + // crash/resume round asserts exactly one (resume, not restart). + syncRuns int +} + +func resourceKey(id *v2.ResourceId) string { return id.GetResourceType() + "/" + id.GetResource() } + +func snapshotC1z(ctx context.Context, t *testing.T, path string) *c1zSnapshot { + t.Helper() + reopen, err := dotc1z.NewStore(ctx, path, + dotc1z.WithEngine(c1zstore.EnginePebble), + dotc1z.WithReadOnly(true), + ) + require.NoError(t, err) + defer func() { _ = reopen.Close(ctx) }() + + syncRun, err := reopen.SyncMeta().LatestFullSync(ctx) + require.NoError(t, err) + require.NotNil(t, syncRun) + require.NoError(t, reopen.SetCurrentSync(ctx, syncRun.ID)) + + snap := &c1zSnapshot{ + resourceTypes: map[string]*v2.ResourceType{}, + resources: map[string]*v2.Resource{}, + entitlements: map[string]*v2.Entitlement{}, + grants: map[string]*v2.Grant{}, + byParent: map[string][]string{}, + grantsByEntResource: map[string][]string{}, + } + + // Raw row counts per listing: the deduping maps below would silently + // hide duplicate rows (a pagination double-yield or a replay writing + // two rows under one identity), so every listing asserts raw == map. + pageToken := "" + rawRows := 0 + for { + resp, err := reopen.ListResourceTypes(ctx, v2.ResourceTypesServiceListResourceTypesRequest_builder{PageToken: pageToken}.Build()) + require.NoError(t, err) + for _, rt := range resp.GetList() { + snap.resourceTypes[rt.GetId()] = rt + rawRows++ + } + if pageToken = resp.GetNextPageToken(); pageToken == "" { + break + } + } + require.Equal(t, len(snap.resourceTypes), rawRows, "duplicate resource-type rows in %s", path) + rawRows = 0 + for { + resp, err := reopen.ListResources(ctx, v2.ResourcesServiceListResourcesRequest_builder{PageToken: pageToken}.Build()) + require.NoError(t, err) + for _, r := range resp.GetList() { + snap.resources[resourceKey(r.GetId())] = r + rawRows++ + } + if pageToken = resp.GetNextPageToken(); pageToken == "" { + break + } + } + require.Equal(t, len(snap.resources), rawRows, "duplicate resource rows in %s", path) + rawRows = 0 + for { + resp, err := reopen.ListEntitlements(ctx, v2.EntitlementsServiceListEntitlementsRequest_builder{PageToken: pageToken}.Build()) + require.NoError(t, err) + for _, e := range resp.GetList() { + snap.entitlements[e.GetId()] = e + rawRows++ + } + if pageToken = resp.GetNextPageToken(); pageToken == "" { + break + } + } + require.Equal(t, len(snap.entitlements), rawRows, "duplicate entitlement rows in %s", path) + rawRows = 0 + for { + resp, err := reopen.ListGrants(ctx, v2.GrantsServiceListGrantsRequest_builder{PageToken: pageToken}.Build()) + require.NoError(t, err) + for _, g := range resp.GetList() { + snap.grants[g.GetId()] = g + rawRows++ + } + if pageToken = resp.GetNextPageToken(); pageToken == "" { + break + } + } + require.Equal(t, len(snap.grants), rawRows, "duplicate grant rows in %s", path) + + // Source-cache state below the v2 read surface, plus the sync-run + // count (crash/resume must resume, never restart). + engineHolder, ok := any(reopen).(interface{ PebbleEngine() *pebbleengine.Engine }) + require.True(t, ok, "store must expose its pebble engine for source-cache inspection") + snap.manifest, err = engineHolder.PebbleEngine().SourceCacheManifestSnapshot(ctx) + require.NoError(t, err) + snap.scopeIndexCounts, err = engineHolder.PebbleEngine().SourceCacheScopeIndexSnapshot(ctx) + require.NoError(t, err) + runLister, ok := any(reopen).(interface { + ListSyncRuns(ctx context.Context, pageToken string, pageSize uint32) ([]*c1zstore.SyncRun, string, error) + }) + require.True(t, ok, "store must list sync runs") + runs, _, err := runLister.ListSyncRuns(ctx, "", 100) + require.NoError(t, err) + snap.syncRuns = len(runs) + + // Secondary read paths: by-parent resource listings for every parent + // that has children per the primary rows, and by-entitlement-resource + // grant listings for every entitlement resource seen in grants. + parents := map[string]*v2.ResourceId{} + for _, r := range snap.resources { + if p := r.GetParentResourceId(); p.GetResource() != "" { + parents[resourceKey(p)] = p + } + } + for pk, pid := range parents { + for { + resp, err := reopen.ListResources(ctx, v2.ResourcesServiceListResourcesRequest_builder{ + ResourceTypeId: "", + ParentResourceId: pid, + PageToken: pageToken, + }.Build()) + require.NoError(t, err) + for _, r := range resp.GetList() { + snap.byParent[pk] = append(snap.byParent[pk], resourceKey(r.GetId())) + } + if pageToken = resp.GetNextPageToken(); pageToken == "" { + break + } + } + sort.Strings(snap.byParent[pk]) + } + entResources := map[string]*v2.Resource{} + for _, g := range snap.grants { + res := g.GetEntitlement().GetResource() + if res.GetId() != nil { + entResources[resourceKey(res.GetId())] = res + } + } + for rk, res := range entResources { + for { + resp, err := reopen.ListGrants(ctx, v2.GrantsServiceListGrantsRequest_builder{ + Resource: res, + PageToken: pageToken, + }.Build()) + require.NoError(t, err) + for _, g := range resp.GetList() { + snap.grantsByEntResource[rk] = append(snap.grantsByEntResource[rk], g.GetId()) + } + if pageToken = resp.GetNextPageToken(); pageToken == "" { + break + } + } + sort.Strings(snap.grantsByEntResource[rk]) + } + return snap +} + +// normalizeMessage returns a clone with annotations sorted so proto +// equality ignores annotation ordering (the only legitimate ordering +// difference between cold and warm outputs). +func normalizeMessage[T proto.Message](msg T, get func(T) []*anypb.Any, set func(T, []*anypb.Any)) T { + clone, _ := proto.Clone(msg).(T) + annos := get(clone) + if len(annos) > 1 { + sorted := append([]*anypb.Any{}, annos...) + sort.Slice(sorted, func(i, j int) bool { + if sorted[i].GetTypeUrl() != sorted[j].GetTypeUrl() { + return sorted[i].GetTypeUrl() < sorted[j].GetTypeUrl() + } + return string(sorted[i].GetValue()) < string(sorted[j].GetValue()) + }) + set(clone, sorted) + } + return clone +} + +func diffMaps[T proto.Message](t *testing.T, label string, cold, warm map[string]T, normalize func(T) T) { + t.Helper() + for _, k := range sortedKeys(cold) { + w, ok := warm[k] + if !ok { + t.Errorf("replay equivalence: %s %q present in COLD output but MISSING from warm output", label, k) + continue + } + nc, nw := normalize(cold[k]), normalize(w) + if !proto.Equal(nc, nw) { + t.Errorf("replay equivalence: %s %q differs\n--- cold ---\n%s\n--- warm ---\n%s", + label, k, prototext.Format(nc), prototext.Format(nw)) + } + } + for _, k := range sortedKeys(warm) { + if _, ok := cold[k]; !ok { + t.Errorf("replay equivalence: %s %q present in WARM output but missing from cold output (stale row survived?)", label, k) + } + } +} + +func requireSnapshotsEqual(t *testing.T, round int, cold, warm *c1zSnapshot) { + t.Helper() + t.Logf("round %d: comparing cold (%d res, %d ents, %d grants) vs warm (%d res, %d ents, %d grants)", + round, len(cold.resources), len(cold.entitlements), len(cold.grants), + len(warm.resources), len(warm.entitlements), len(warm.grants)) + + diffMaps(t, "resource type", cold.resourceTypes, warm.resourceTypes, + func(m *v2.ResourceType) *v2.ResourceType { + return normalizeMessage(m, (*v2.ResourceType).GetAnnotations, (*v2.ResourceType).SetAnnotations) + }) + diffMaps(t, "resource", cold.resources, warm.resources, + func(m *v2.Resource) *v2.Resource { + return normalizeMessage(m, (*v2.Resource).GetAnnotations, (*v2.Resource).SetAnnotations) + }) + diffMaps(t, "entitlement", cold.entitlements, warm.entitlements, + func(m *v2.Entitlement) *v2.Entitlement { + return normalizeMessage(m, (*v2.Entitlement).GetAnnotations, (*v2.Entitlement).SetAnnotations) + }) + diffMaps(t, "grant", cold.grants, warm.grants, + func(m *v2.Grant) *v2.Grant { + return normalizeMessage(m, (*v2.Grant).GetAnnotations, (*v2.Grant).SetAnnotations) + }) + + require.Equal(t, cold.byParent, warm.byParent, + "round %d: by-parent index read path diverged (replay index synthesis?)", round) + require.Equal(t, cold.grantsByEntResource, warm.grantsByEntResource, + "round %d: by-entitlement-resource index read path diverged (replay index synthesis?)", round) + + // Below the v2 read surface: both connectors emitted identical scope + // annotations and validators, so the manifests and scope-index shapes + // must match. Divergence here is invisible to every check above but + // poisons the NEXT sync's replay from the warm file. + require.Equal(t, cold.manifest, warm.manifest, + "round %d: source-cache manifest diverged (phantom or missing entries poison future replays)", round) + require.Equal(t, cold.scopeIndexCounts, warm.scopeIndexCounts, + "round %d: source-cache scope-index shape diverged (stamp divergence poisons future replays)", round) + + if t.Failed() { + t.Fatalf("round %d: warm sync output is not equivalent to a cold resync of the same upstream state", round) + } +} + +// assertNoStaleResources is the reverse-containment check: every resource +// row of a modeled type must correspond to CURRENT model state (or an +// external principal). The forward diff proves warm == cold; this proves +// neither side is carrying rows the upstream no longer has — the +// deletion-round oracle. +func assertNoStaleResources(t *testing.T, round int, m *equivModel, label string, snap *c1zSnapshot) { + t.Helper() + for key := range snap.resources { + rt, id, _ := strings.Cut(key, "/") + switch rt { + case equivOrgRT.GetId(): + require.Contains(t, m.orgs, id, "round %d: %s output holds deleted org %s", round, label, id) + case equivProjectRT.GetId(): + found := false + for _, org := range m.orgs { + if _, ok := org.projects[id]; ok { + found = true + } + } + require.True(t, found, "round %d: %s output holds deleted project %s", round, label, id) + case equivRepoRT.GetId(): + found := false + for _, org := range m.orgs { + if _, ok := org.repoGrants[id]; ok { + found = true + } + } + require.True(t, found, "round %d: %s output holds deleted repo %s", round, label, id) + case groupResourceType.GetId(): + require.Contains(t, m.groups, id, "round %d: %s output holds deleted group %s", round, label, id) + case equivTeamRT.GetId(): + require.Contains(t, m.teams, id, "round %d: %s output holds unknown team %s", round, label, id) + case userResourceType.GetId(): + require.True(t, m.users[id] || m.extUsers[id], + "round %d: %s output holds deleted user %s", round, label, id) + } + } +} + +// expectedGrantIDs derives the COMPLETE grant set the model implies: +// group memberships, the expandable membership plus its expansion-derived +// grants, match-transformed external grants, repo (InsertResourceGrants) +// grants, and type-scoped team memberships. Comparing the cold output +// against this — exact set equality, not spot checks — removes the +// differential test's structural blind spot: cold and warm share the +// syncer, so a bug corrupting both identically would otherwise pass. +func expectedGrantIDs(m *equivModel) []string { + set := map[string]struct{}{} + for gid, g := range m.groups { + if gid == m.extGroupID { + continue + } + for uid := range g.members { + set[equivMemberGrant(gid, uid).GetId()] = struct{}{} + } + } + // The expandable group-to-group membership and its derived grants. + parentRes := equivGroupResource(m.expandableParent) + parentEnt := equivMemberEnt(parentRes, false) + set[gt.NewGrantID(equivGroupResource(m.expandableChild), parentEnt)] = struct{}{} + for uid := range m.groups[m.expandableChild].members { + set[gt.NewGrantID( + v2.ResourceId_builder{ResourceType: userResourceType.GetId(), Resource: uid}.Build(), + parentEnt)] = struct{}{} + } + // Match-transformed grants: MatchAll covers every external user (the + // MatchID targets are a subset with identical resulting ids). + extEnt := equivMemberEnt(equivGroupResource(m.extGroupID), false) + for extID := range m.extUsers { + set[gt.NewGrantID( + v2.ResourceId_builder{ResourceType: userResourceType.GetId(), Resource: extID}.Build(), + extEnt)] = struct{}{} + } + for _, org := range m.orgs { + for repoID, uid := range org.repoGrants { + set[gt.NewGrant(equivRepoResource(repoID), "admin", + v2.ResourceId_builder{ResourceType: userResourceType.GetId(), Resource: uid}.Build()).GetId()] = struct{}{} + } + } + for tid, members := range m.teams { + for uid := range members { + set[equivTeamGrant(tid, uid).GetId()] = struct{}{} + } + } + return sortedKeys(set) +} + +// --------------------------------------------------------------------- +// Coverage guards (anti-vacuity) +// --------------------------------------------------------------------- + +// assertColdArtifacts fails unless the cold output actually contains the +// artifacts of every feature the harness claims to cover — a comparison +// against an output that never exercised a feature proves nothing about +// that feature. +func assertColdArtifacts(t *testing.T, round int, m *equivModel, snap *c1zSnapshot) { + t.Helper() + for _, orgID := range sortedKeys(m.orgs) { + for projID := range m.orgs[orgID].projects { + require.Contains(t, snap.resources, equivProjectRT.GetId()+"/"+projID, + "round %d: coverage: child project %s must be present", round, projID) + } + for repoID := range m.orgs[orgID].repoGrants { + require.Contains(t, snap.resources, equivRepoRT.GetId()+"/"+repoID, + "round %d: coverage: InsertResourceGrants repo %s must be present", round, repoID) + } + } + // Expansion-derived grants: every member of the expandable child must + // hold a derived grant on the parent. + parentRes := equivGroupResource(m.expandableParent) + for uid := range m.groups[m.expandableChild].members { + derived := gt.NewGrantID( + v2.ResourceId_builder{ResourceType: userResourceType.GetId(), Resource: uid}.Build(), + equivMemberEnt(parentRes, false)) + require.Contains(t, snap.grants, derived, + "round %d: coverage: expansion-derived grant for %s must be present", round, uid) + } + // Match-transformed grants: MatchAll must cover every external user; + // MatchID entries must cover exactly the still-existing targets. + extGroupRes := equivGroupResource(m.extGroupID) + extEnt := equivMemberEnt(extGroupRes, false) + for extID := range m.extUsers { + transformed := gt.NewGrantID( + v2.ResourceId_builder{ResourceType: userResourceType.GetId(), Resource: extID}.Build(), extEnt) + require.Contains(t, snap.grants, transformed, + "round %d: coverage: MatchAll-transformed grant for %s must be present", round, extID) + } + for _, extID := range m.extMatchIDs { + transformed := gt.NewGrantID( + v2.ResourceId_builder{ResourceType: userResourceType.GetId(), Resource: extID}.Build(), extEnt) + if m.extUsers[extID] { + require.Contains(t, snap.grants, transformed, + "round %d: coverage: MatchID-transformed grant for %s must be present", round, extID) + } else { + require.NotContains(t, snap.grants, transformed, + "round %d: coverage: MatchID grant for REMOVED external user %s must be absent", round, extID) + } + } + // No placeholder principals may survive the transformation. + for id := range snap.grants { + require.NotContains(t, id, "placeholder-", + "round %d: coverage: untransformed placeholder grant leaked into output", round) + } + // The independent oracle: the cold grant set must EXACTLY equal the + // model-derived expectation — the check that doesn't trust the + // syncer code both outputs share. + require.Equal(t, expectedGrantIDs(m), sortedKeys(snap.grants), + "round %d: cold output's grant set must equal the model-derived expectation", round) +} + +// --------------------------------------------------------------------- +// The harness +// --------------------------------------------------------------------- + +const equivRounds = 4 + +// equivCrashRound is the round whose WARM sync is crashed mid-grants-phase +// (injected connector failure) and resumed into the same file. +const equivCrashRound = 2 + +// runEquivSync runs one sync into path (creating or RESUMING the file's +// unfinished sync), returning Sync's error. Path-based store creation — +// not WithConnectorStore — so a crashed sync's file can be reopened and +// resumed exactly like production resume. +func runEquivSync(ctx context.Context, t *testing.T, cc types.ConnectorClient, path, prevPath, tmpDir string, extraOpts ...SyncOpt) error { + t.Helper() + opts := []SyncOpt{ + WithC1ZPath(path), + WithStorageEngine(c1zstore.EnginePebble), + WithTmpDir(tmpDir), + } + if prevPath != "" { + opts = append(opts, WithPreviousSyncC1ZPath(prevPath)) + } + opts = append(opts, extraOpts...) + syncer, err := NewSyncer(ctx, cc, opts...) + require.NoError(t, err) + syncErr := syncer.Sync(ctx) + if syncErr != nil { + // Close under a canceled context so the unfinished sync's state + // survives for resumption (same pattern as the resume tests). + cctx, cancel := context.WithCancel(ctx) + cancel() + _ = syncer.Close(cctx) + return syncErr + } + require.NoError(t, syncer.Close(ctx)) + return nil +} + +func TestSourceCache_ReplayEquivalence(t *testing.T) { + scenarios := []struct { + workers int + seed int64 + continuation bool + }{ + {workers: 0, seed: 1}, + {workers: 0, seed: 2}, + {workers: 4, seed: 3}, + // The single-shot (Lambda) topology: lookups ride the ask/answer + // continuation instead of a direct SetSourceCache install. + {workers: 0, seed: 4, continuation: true}, + {workers: 4, seed: 5, continuation: true}, + } + for _, sc := range scenarios { + name := fmt.Sprintf("workers=%d/seed=%d", sc.workers, sc.seed) + if sc.continuation { + name += "/continuation" + } + t.Run(name, func(t *testing.T) { + runReplayEquivalenceScenario(t, sc.workers, sc.seed, sc.continuation) + }) + } +} + +func runReplayEquivalenceScenario(t *testing.T, workers int, seed int64, continuation bool) { + ctx, err := logging.Init(t.Context()) + require.NoError(t, err) + tmpDir := t.TempDir() + rng := rand.New(rand.NewSource(seed)) //nolint:gosec // deterministic test randomness + t.Logf("replay-equivalence scenario: workers=%d seed=%d rounds=%d continuation=%v", workers, seed, equivRounds, continuation) + + model := newEquivModel() + warmConn := newEquivConnector(model, false) + coldConn := newEquivConnector(model, true) + warmConn.continuation = continuation + coldConn.continuation = continuation + var warmClient types.ConnectorClient = warmConn + var coldClient types.ConnectorClient = coldConn + if continuation { + warmClient = equivContinuationClient{warmConn} + coldClient = equivContinuationClient{coldConn} + } + + extPath := filepath.Join(tmpDir, "external-r0.c1z") + buildExternalC1z(ctx, t, model, extPath, tmpDir) + model.extDirty = false + + syncOpts := func() []SyncOpt { + opts := []SyncOpt{WithExternalResourceC1ZPath(extPath)} + if workers > 0 { + opts = append(opts, WithWorkerCount(workers)) + } + return opts + } + + prevWarm := "" + for round := 0; round <= equivRounds; round++ { + if round > 0 { + model.mutate(round, rng) + if model.extDirty { + extPath = filepath.Join(tmpDir, fmt.Sprintf("external-r%d.c1z", round)) + buildExternalC1z(ctx, t, model, extPath, tmpDir) + } + } + warmPath := filepath.Join(tmpDir, fmt.Sprintf("warm-r%d.c1z", round)) + coldPath := filepath.Join(tmpDir, fmt.Sprintf("cold-r%d.c1z", round)) + + before := warmConn.snapshotCounters() + if round == equivCrashRound { + // Crash the warm sync mid-grants-phase (after other scopes + // replayed), then resume the SAME file. The resumed output — + // pages re-run over partially replayed state — is what gets + // compared. + warmConn.injectGrantsFailure(model.expandableChild) + crashErr := runEquivSync(ctx, t, warmClient, warmPath, prevWarm, tmpDir, syncOpts()...) + require.Error(t, crashErr, "round %d: the injected failure must crash the warm sync", round) + require.Contains(t, crashErr.Error(), "injected crash") + require.Equal(t, 1, warmConn.snapshotCounters().injectedFailures-before.injectedFailures, + "round %d: exactly one injected failure expected", round) + } + require.NoError(t, runEquivSync(ctx, t, warmClient, warmPath, prevWarm, tmpDir, syncOpts()...)) + after := warmConn.snapshotCounters() + if round > 0 { + require.Positive(t, (after.pages304+after.pagesOverlay)-(before.pages304+before.pagesOverlay), + "round %d: VACUOUS RUN — the warm sync replayed nothing, so equivalence proves nothing", round) + } + require.NoError(t, runEquivSync(ctx, t, coldClient, coldPath, "", tmpDir, syncOpts()...)) + + coldSnap := snapshotC1z(ctx, t, coldPath) + warmSnap := snapshotC1z(ctx, t, warmPath) + if round == equivCrashRound { + require.Equal(t, 1, warmSnap.syncRuns, + "round %d: the crashed warm sync must RESUME (one sync run), not restart", round) + } + assertColdArtifacts(t, round, model, coldSnap) + assertNoStaleResources(t, round, model, "cold", coldSnap) + assertNoStaleResources(t, round, model, "warm", warmSnap) + requireSnapshotsEqual(t, round, coldSnap, warmSnap) + + prevWarm = warmPath + } + + // Scenario-level coverage: every replay behavior must have actually + // occurred, or this scenario silently tested less than it claims. + final := warmConn.snapshotCounters() + require.Positive(t, final.pages304, "coverage: no 304 replay round ever happened") + require.Positive(t, final.pagesOverlay, "coverage: no delta-overlay round ever happened") + require.Positive(t, final.pagesOverlayInterim, "coverage: no multi-page overlay round ever happened") + require.Positive(t, final.pagesColdStale, "coverage: no validator rotation (cold refetch) ever happened") + require.Positive(t, final.pagesFresh, "coverage: no cold-miss page ever happened") + require.Positive(t, final.pagesFreshInterim, "coverage: no multi-page fresh fetch ever happened") + require.Positive(t, final.tombsCanonical, "coverage: canonical-id tombstones never exercised") + require.Positive(t, final.tombsPrincipal, "coverage: principal-id tombstones never exercised") + require.Positive(t, final.childPageCalls, "coverage: child resource pages never fetched") + require.Positive(t, final.injectedFailures, "coverage: the crash/resume round never crashed") + require.Positive(t, final.plannerPages, "coverage: the type-scoped planner never ran") + require.Positive(t, final.chunkPages304, "coverage: no spawned chunk cursor ever replayed") + require.Positive(t, final.chunkPagesFresh, "coverage: no spawned chunk cursor ever fetched fresh") + if continuation { + require.Positive(t, final.askPages, "coverage: continuation topology never asked") + } else { + require.Zero(t, final.askPages, "harness bug: direct topology must never ask") + } + coldFinal := coldConn.snapshotCounters() + require.Zero(t, coldFinal.pages304+coldFinal.pagesOverlay, + "harness bug: the cold connector must never replay") + require.Zero(t, coldFinal.askPages, "harness bug: the cold connector must never ask (no offer without a warm lookup)") +} diff --git a/pkg/sync/source_cache_sideeffects_test.go b/pkg/sync/source_cache_sideeffects_test.go new file mode 100644 index 000000000..6ab04bf9b --- /dev/null +++ b/pkg/sync/source_cache_sideeffects_test.go @@ -0,0 +1,776 @@ +package sync //nolint:revive,nolintlint // we can't change the package name for backwards compatibility + +// Replay side-effect regression tests: response-row-driven side effects +// (child scheduling, external-resource-match processing, related-resource +// insertion) must survive a warm replay round exactly as a cold sync +// would produce them. Each test runs sync 1 cold, sync 2 as a 304-style +// replay, and asserts the side effect's output is present in sync 2. + +import ( + "context" + "fmt" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + "github.com/conductorone/baton-sdk/pkg/annotations" + "github.com/conductorone/baton-sdk/pkg/dotc1z" + "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" + pebbleengine "github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble" + "github.com/conductorone/baton-sdk/pkg/logging" + "github.com/conductorone/baton-sdk/pkg/sourcecache" + "github.com/conductorone/baton-sdk/pkg/types" + gt "github.com/conductorone/baton-sdk/pkg/types/grant" + rs "github.com/conductorone/baton-sdk/pkg/types/resource" +) + +// runSourceCacheSyncClient is runSourceCacheSync for wrapper connectors +// that embed sourceCacheMockConnector but override list methods: the +// OUTER value must be handed to the syncer or the overrides never run. +func runSourceCacheSyncClient(ctx context.Context, t *testing.T, cc types.ConnectorClient, path, prevPath, tmpDir string, extraOpts ...SyncOpt) { + t.Helper() + store, err := dotc1z.NewStore(ctx, path, + dotc1z.WithEngine(c1zstore.EnginePebble), + dotc1z.WithTmpDir(tmpDir), + ) + require.NoError(t, err) + opts := []SyncOpt{WithConnectorStore(store), WithTmpDir(tmpDir)} + if prevPath != "" { + opts = append(opts, WithPreviousSyncC1ZPath(prevPath)) + } + opts = append(opts, extraOpts...) + syncer, err := NewSyncer(ctx, cc, opts...) + require.NoError(t, err) + require.NoError(t, syncer.Sync(ctx)) + require.NoError(t, syncer.Close(ctx)) +} + +// listResourcesInFile reopens a finished Pebble c1z and returns its +// resources of one type, keyed by resource ID. +func listResourcesInFile(ctx context.Context, t *testing.T, path, resourceTypeID string) map[string]*v2.Resource { + t.Helper() + reopen, err := dotc1z.NewStore(ctx, path, + dotc1z.WithEngine(c1zstore.EnginePebble), + dotc1z.WithReadOnly(true), + ) + require.NoError(t, err) + defer func() { _ = reopen.Close(ctx) }() + + sync, err := reopen.SyncMeta().LatestFullSync(ctx) + require.NoError(t, err) + require.NotNil(t, sync) + require.NoError(t, reopen.SetCurrentSync(ctx, sync.ID)) + + out := map[string]*v2.Resource{} + pageToken := "" + for { + resp, err := reopen.ListResources(ctx, v2.ResourcesServiceListResourcesRequest_builder{ + ResourceTypeId: resourceTypeID, + PageToken: pageToken, + }.Build()) + require.NoError(t, err) + for _, r := range resp.GetList() { + out[r.GetId().GetResource()] = r + } + pageToken = resp.GetNextPageToken() + if pageToken == "" { + break + } + } + return out +} + +// --------------------------------------------------------------------- +// 1. Replayed parent resources must still schedule child resource syncs. +// --------------------------------------------------------------------- + +// childReplayConnector serves a parent resource type as a source-cache +// scope (fresh on miss, 304 replay on hit) whose resources carry +// ChildResourceType annotations. Child resources are only returned when +// the request names a parent — the standard parent/child connector shape. +type childReplayConnector struct { + *sourceCacheMockConnector + + parentType *v2.ResourceType + childType *v2.ResourceType + parents []*v2.Resource + children map[string][]*v2.Resource // parent resource id -> children + + resourceEtag string + resourceLookupHits int +} + +func (c *childReplayConnector) ListResources(ctx context.Context, in *v2.ResourcesServiceListResourcesRequest, opts ...grpc.CallOption) (*v2.ResourcesServiceListResourcesResponse, error) { + switch in.GetResourceTypeId() { + case c.parentType.GetId(): + scope := sourcecache.HashScope("mock://resources/" + c.parentType.GetId()) + c.mu.Lock() + lookup := c.lookup + etag := c.resourceEtag + c.mu.Unlock() + if lookup == nil { + lookup = sourcecache.NoopLookup{} + } + entry, found, err := lookup.LookupPreviousSourceCache(ctx, sourcecache.RowKindResources, scope) + if err != nil { + return nil, err + } + if found && entry.ETag == etag { + c.mu.Lock() + c.resourceLookupHits++ + c.mu.Unlock() + return v2.ResourcesServiceListResourcesResponse_builder{ + Annotations: annotations.New(v2.SourceCacheReplay_builder{ + ScopeHash: scope, + Etag: etag, + }.Build()), + }.Build(), nil + } + return v2.ResourcesServiceListResourcesResponse_builder{ + List: c.parents, + Annotations: annotations.New(v2.SourceCacheScope_builder{ + ScopeHash: scope, + Etag: etag, + }.Build()), + }.Build(), nil + case c.childType.GetId(): + if in.GetParentResourceId() == nil { + // Children are only reachable through a parent. + return v2.ResourcesServiceListResourcesResponse_builder{}.Build(), nil + } + return v2.ResourcesServiceListResourcesResponse_builder{ + List: c.children[in.GetParentResourceId().GetResource()], + }.Build(), nil + } + return c.sourceCacheMockConnector.ListResources(ctx, in, opts...) +} + +// TestSourceCache_ReplaySchedulesChildResources pins the child-scheduling +// contract under replay: a 304 replay of a parent resources page copies +// the parents engine-side and returns no rows, so the response-row loop +// that normally pushes child SyncResources actions never sees them. The +// replay path must schedule those children itself, or a warm sync +// silently drops whole subtrees. +func TestSourceCache_ReplaySchedulesChildResources(t *testing.T) { + for _, workers := range workerCounts { + t.Run(fmt.Sprintf("workers=%d", workers), func(t *testing.T) { + runReplaySchedulesChildResourcesScenario(t, workerSyncOpts(workers)) + }) + } +} + +func runReplaySchedulesChildResourcesScenario(t *testing.T, workerOpts []SyncOpt) { + ctx, err := logging.Init(t.Context()) + require.NoError(t, err) + tmpDir := t.TempDir() + + parentRT := v2.ResourceType_builder{Id: "sc_parent", DisplayName: "Parent"}.Build() + childRT := v2.ResourceType_builder{Id: "sc_child", DisplayName: "Child"}.Build() + + parent1, err := rs.NewResource("Parent 1", parentRT, "parent_1", + rs.WithAnnotation(v2.ChildResourceType_builder{ResourceTypeId: childRT.GetId()}.Build()), + rs.WithAnnotation(&v2.SkipEntitlementsAndGrants{}), + ) + require.NoError(t, err) + child1, err := rs.NewResource("Child 1", childRT, "child_1", + rs.WithParentResourceID(parent1.GetId()), + rs.WithAnnotation(&v2.SkipEntitlementsAndGrants{}), + ) + require.NoError(t, err) + child2, err := rs.NewResource("Child 2", childRT, "child_2", + rs.WithParentResourceID(parent1.GetId()), + rs.WithAnnotation(&v2.SkipEntitlementsAndGrants{}), + ) + require.NoError(t, err) + + inner := newSourceCacheMockConnector() + inner.rtDB = []*v2.ResourceType{parentRT, childRT} + mc := &childReplayConnector{ + sourceCacheMockConnector: inner, + parentType: parentRT, + childType: childRT, + parents: []*v2.Resource{parent1}, + children: map[string][]*v2.Resource{ + parent1.GetId().GetResource(): {child1, child2}, + }, + resourceEtag: `W/"resources-v1"`, + } + + sync1 := filepath.Join(tmpDir, "sync1.c1z") + sync2 := filepath.Join(tmpDir, "sync2.c1z") + + runSourceCacheSyncClient(ctx, t, mc, sync1, "", tmpDir, workerOpts...) + require.Zero(t, mc.resourceLookupHits, "sync 1 has no previous sync to hit") + require.Len(t, listResourcesInFile(ctx, t, sync1, childRT.GetId()), 2, + "sanity: cold sync must fetch children through the parent") + + // Upstream unchanged: the parent page replays. The children must + // still be scheduled and fetched. + runSourceCacheSyncClient(ctx, t, mc, sync2, sync1, tmpDir, workerOpts...) + require.NotZero(t, mc.resourceLookupHits, "sanity: sync 2 must have replayed the parent page") + + parents2 := listResourcesInFile(ctx, t, sync2, parentRT.GetId()) + require.Contains(t, parents2, parent1.GetId().GetResource(), "replayed parent must be present") + + children2 := listResourcesInFile(ctx, t, sync2, childRT.GetId()) + require.Contains(t, children2, child1.GetId().GetResource(), + "child resources of a replayed parent must be scheduled and synced") + require.Contains(t, children2, child2.GetId().GetResource(), + "child resources of a replayed parent must be scheduled and synced") +} + +// --------------------------------------------------------------------- +// 2. External-resource-match grants must survive a warm replay. +// --------------------------------------------------------------------- + +// TestSourceCache_ReplayPreservesExternalMatchGrants pins the +// external-match contract under replay. Cold sync 1 emits a +// scope-stamped grant carrying ExternalResourceMatchID; the +// external-resources step transforms it into a grant on the matched +// external principal and deletes the source grant. Sync 2's 304 replay +// must carry the transformed grant forward — and the replay path must +// re-arm HasExternalResourcesGrants so match processing still runs on +// replayed rows. +func TestSourceCache_ReplayPreservesExternalMatchGrants(t *testing.T) { + for _, workers := range workerCounts { + t.Run(fmt.Sprintf("workers=%d", workers), func(t *testing.T) { + runReplayPreservesExternalMatchGrantsScenario(t, workerSyncOpts(workers)) + }) + } +} + +func runReplayPreservesExternalMatchGrantsScenario(t *testing.T, workerOpts []SyncOpt) { + ctx, err := logging.Init(t.Context()) + require.NoError(t, err) + tmpDir := t.TempDir() + + // External c1z: one user the match annotation will resolve to. + externalMc := newMockConnector() + externalMc.rtDB = append(externalMc.rtDB, userResourceType, groupResourceType) + externalUser, err := externalMc.AddUserProfile(ctx, "ext_user_1", map[string]any{}) + require.NoError(t, err) + + externalC1z := filepath.Join(tmpDir, "external.c1z") + externalSyncer, err := NewSyncer(ctx, externalMc, WithC1ZPath(externalC1z), WithTmpDir(tmpDir)) + require.NoError(t, err) + require.NoError(t, externalSyncer.Sync(ctx)) + require.NoError(t, externalSyncer.Close(ctx)) + + // Internal connector: one group whose grants page is a source-cache + // scope. The single grant references a placeholder principal and + // carries ExternalResourceMatchID pointing at the external user. + mc := newSourceCacheMockConnector() + internalGroup, ent, err := mc.AddGroup(ctx, "internal_group") + require.NoError(t, err) + + placeholder := v2.ResourceId_builder{ResourceType: "user", Resource: "placeholder_1"}.Build() + sourceGrant := gt.NewGrant(internalGroup, "member", placeholder, + gt.WithAnnotation(v2.ExternalResourceMatchID_builder{ + Id: externalUser.GetId().GetResource(), + }.Build()), + ) + mc.grantDB[internalGroup.GetId().GetResource()] = []*v2.Grant{sourceGrant} + mc.etagByResource[internalGroup.GetId().GetResource()] = `W/"grants-v1"` + + transformedGrantID := gt.NewGrantID(externalUser, sourceGrant.GetEntitlement()) + _ = ent + + extOpts := append([]SyncOpt{WithExternalResourceC1ZPath(externalC1z)}, workerOpts...) + + sync1 := filepath.Join(tmpDir, "sync1.c1z") + sync2 := filepath.Join(tmpDir, "sync2.c1z") + sync3 := filepath.Join(tmpDir, "sync3.c1z") + + runSourceCacheSync(ctx, t, mc, sync1, "", tmpDir, extOpts...) + grants1 := listGrantsInFile(ctx, t, sync1) + require.Contains(t, grants1, transformedGrantID, "sanity: cold sync must produce the transformed grant") + require.NotContains(t, grants1, sourceGrant.GetId(), "sanity: the placeholder source grant must be deleted") + + // Upstream unchanged: the grants scope replays. The transformed + // grant must survive. + runSourceCacheSync(ctx, t, mc, sync2, sync1, tmpDir, extOpts...) + require.NotZero(t, mc.lookupHits, "sanity: sync 2 must have replayed") + + grants2 := listGrantsInFile(ctx, t, sync2) + require.Contains(t, grants2, transformedGrantID, + "transformed external-match grant must survive a 304 replay of its source scope") + require.NotContains(t, grants2, sourceGrant.GetId()) + + // And the chain must not decay: sync 2's file must still work as the + // next replay source. + runSourceCacheSync(ctx, t, mc, sync3, sync2, tmpDir, extOpts...) + grants3 := listGrantsInFile(ctx, t, sync3) + require.Contains(t, grants3, transformedGrantID, + "transformed grant must survive a second consecutive replay round") +} + +// --------------------------------------------------------------------- +// 3. InsertResourceGrants resources must survive a warm replay. +// --------------------------------------------------------------------- + +// insertResourceGrantsConnector serves one grants page (for target) as a +// source-cache scope whose response carries InsertResourceGrants: the +// grants reference a resource no ListResources call ever returns, so the +// only way the resource reaches the store is grant-driven insertion. +type insertResourceGrantsConnector struct { + *sourceCacheMockConnector + + target string + insertGrants []*v2.Grant +} + +func (c *insertResourceGrantsConnector) ListGrants(ctx context.Context, in *v2.GrantsServiceListGrantsRequest, _ ...grpc.CallOption) (*v2.GrantsServiceListGrantsResponse, error) { + resourceID := in.GetResource().GetId().GetResource() + if resourceID != c.target { + return v2.GrantsServiceListGrantsResponse_builder{}.Build(), nil + } + scope := sourcecache.HashScope("mock://grants/insert/" + resourceID) + + c.mu.Lock() + lookup := c.lookup + currentEtag := c.etagByResource[resourceID] + c.mu.Unlock() + if lookup == nil { + lookup = sourcecache.NoopLookup{} + } + entry, found, err := lookup.LookupPreviousSourceCache(ctx, sourcecache.RowKindGrants, scope) + if err != nil { + return nil, err + } + if found && entry.ETag == currentEtag { + c.mu.Lock() + c.lookupHits++ + c.mu.Unlock() + return v2.GrantsServiceListGrantsResponse_builder{ + Annotations: annotations.New(v2.SourceCacheReplay_builder{ + ScopeHash: scope, + Etag: currentEtag, + }.Build()), + }.Build(), nil + } + c.mu.Lock() + c.lookupMisses++ + c.mu.Unlock() + return v2.GrantsServiceListGrantsResponse_builder{ + List: c.insertGrants, + Annotations: annotations.New( + v2.SourceCacheScope_builder{ScopeHash: scope, Etag: currentEtag}.Build(), + &v2.InsertResourceGrants{}, + ), + }.Build(), nil +} + +// TestSourceCache_ReplayPreservesInsertResourceGrants pins related- +// resource insertion under replay: a grants page annotated with +// InsertResourceGrants writes the grants' entitlement resources into the +// resources table. On a 304 replay the grant rows are copied engine-side +// but the response has no rows, so the insertion loop never runs — the +// replay must recreate those resource rows or sync 2 holds grants whose +// resources do not exist. +func TestSourceCache_ReplayPreservesInsertResourceGrants(t *testing.T) { + for _, workers := range workerCounts { + t.Run(fmt.Sprintf("workers=%d", workers), func(t *testing.T) { + runReplayPreservesInsertResourceGrantsScenario(t, workerSyncOpts(workers)) + }) + } +} + +func runReplayPreservesInsertResourceGrantsScenario(t *testing.T, workerOpts []SyncOpt) { + ctx, err := logging.Init(t.Context()) + require.NoError(t, err) + tmpDir := t.TempDir() + + projectRT := v2.ResourceType_builder{Id: "project", DisplayName: "Project"}.Build() + + inner := newSourceCacheMockConnector() + inner.rtDB = append(inner.rtDB, projectRT) + internalGroup, _, err := inner.AddGroup(ctx, "g1") + require.NoError(t, err) + alice, err := inner.AddUser(ctx, "alice") + require.NoError(t, err) + + // The project resource is NEVER returned by ListResources — it only + // exists as the embedded resource of the grant below. + projectRes, err := rs.NewResource("Project 1", projectRT, "proj_1", + rs.WithAnnotation(&v2.SkipEntitlementsAndGrants{}), + ) + require.NoError(t, err) + projectGrant := gt.NewGrant(projectRes, "admin", alice) + + inner.etagByResource[internalGroup.GetId().GetResource()] = `W/"insert-v1"` + mc := &insertResourceGrantsConnector{ + sourceCacheMockConnector: inner, + target: internalGroup.GetId().GetResource(), + insertGrants: []*v2.Grant{projectGrant}, + } + + sync1 := filepath.Join(tmpDir, "sync1.c1z") + sync2 := filepath.Join(tmpDir, "sync2.c1z") + + runSourceCacheSyncClient(ctx, t, mc, sync1, "", tmpDir, workerOpts...) + projects1 := listResourcesInFile(ctx, t, sync1, projectRT.GetId()) + require.Contains(t, projects1, "proj_1", "sanity: cold sync must insert the grant's resource") + require.Equal(t, "Project 1", projects1["proj_1"].GetDisplayName()) + require.Contains(t, listGrantsInFile(ctx, t, sync1), projectGrant.GetId()) + + // Upstream unchanged: the grants scope replays. The inserted + // resource must survive alongside its grants. + runSourceCacheSyncClient(ctx, t, mc, sync2, sync1, tmpDir, workerOpts...) + require.NotZero(t, mc.lookupHits, "sanity: sync 2 must have replayed") + + grants2 := listGrantsInFile(ctx, t, sync2) + require.Contains(t, grants2, projectGrant.GetId(), "replayed grant must be present") + + projects2 := listResourcesInFile(ctx, t, sync2, projectRT.GetId()) + require.Contains(t, projects2, "proj_1", + "InsertResourceGrants resource must be recreated when its grants page replays") + require.Equal(t, "Project 1", projects2["proj_1"].GetDisplayName(), + "recreated resource must be a faithful copy of the previous sync's row") +} + +// --------------------------------------------------------------------- +// 4. Replayed entitlements must participate in exclusion-group checks. +// --------------------------------------------------------------------- + +// exclusionGroupReplayConnector serves entitlements pages as source-cache +// scopes for resources with a registered validator (fresh on miss, 304 +// replay on hit); resources without one list entitlements plain. +type exclusionGroupReplayConnector struct { + *sourceCacheMockConnector + + entEtags map[string]string // resource id -> upstream validator + entLookupHits int + entLookupMiss int +} + +func (c *exclusionGroupReplayConnector) ListEntitlements( + ctx context.Context, + in *v2.EntitlementsServiceListEntitlementsRequest, + opts ...grpc.CallOption, +) (*v2.EntitlementsServiceListEntitlementsResponse, error) { + resourceID := in.GetResource().GetId().GetResource() + c.mu.Lock() + lookup := c.lookup + etag := c.entEtags[resourceID] + c.mu.Unlock() + if etag == "" { + return c.sourceCacheMockConnector.ListEntitlements(ctx, in, opts...) + } + scope := sourcecache.HashScope("mock://entitlements/" + resourceID) + if lookup == nil { + lookup = sourcecache.NoopLookup{} + } + entry, found, err := lookup.LookupPreviousSourceCache(ctx, sourcecache.RowKindEntitlements, scope) + if err != nil { + return nil, err + } + if found && entry.ETag == etag { + c.mu.Lock() + c.entLookupHits++ + c.mu.Unlock() + return v2.EntitlementsServiceListEntitlementsResponse_builder{ + Annotations: annotations.New(v2.SourceCacheReplay_builder{ + ScopeHash: scope, + Etag: etag, + }.Build()), + }.Build(), nil + } + c.mu.Lock() + c.entLookupMiss++ + c.mu.Unlock() + return v2.EntitlementsServiceListEntitlementsResponse_builder{ + List: c.entDB[resourceID], + Annotations: annotations.New(v2.SourceCacheScope_builder{ + ScopeHash: scope, + Etag: etag, + }.Build()), + }.Build(), nil +} + +// TestSourceCache_ReplayedEntitlementsKeepExclusionGroupValidation pins +// the exclusion-group invariants under replay: validation normally runs +// on response rows only, so a replayed entitlement's exclusion-group +// membership would silently stop participating — a duplicate default +// split across a replayed scope and a fresh scope would pass a warm sync +// that a full resync hard-fails. +func TestSourceCache_ReplayedEntitlementsKeepExclusionGroupValidation(t *testing.T) { + for _, workers := range workerCounts { + t.Run(fmt.Sprintf("workers=%d", workers), func(t *testing.T) { + runReplayedEntitlementExclusionGroupScenario(t, workerSyncOpts(workers)) + }) + } +} + +func runReplayedEntitlementExclusionGroupScenario(t *testing.T, workerOpts []SyncOpt) { + ctx, err := logging.Init(t.Context()) + require.NoError(t, err) + tmpDir := t.TempDir() + + inner := newSourceCacheMockConnector() + mc := &exclusionGroupReplayConnector{ + sourceCacheMockConnector: inner, + entEtags: map[string]string{}, + } + + defaultExclusionGroup := v2.EntitlementExclusionGroup_builder{ + ExclusionGroupId: "eg1", + IsDefault: true, + }.Build() + + // Group A: its entitlements page is a scope, carrying the exclusion + // group's default entitlement. + groupA, entA, err := inner.AddGroup(ctx, "group_a") + require.NoError(t, err) + entAAnnos := annotations.Annotations(entA.GetAnnotations()) + entAAnnos.Update(defaultExclusionGroup) + entA.SetAnnotations(entAAnnos) + mc.entEtags[groupA.GetId().GetResource()] = `W/"ents-a-v1"` + + sync1 := filepath.Join(tmpDir, "sync1.c1z") + runSourceCacheSyncClient(ctx, t, mc, sync1, "", tmpDir, workerOpts...) + require.NotZero(t, mc.entLookupMiss, "sanity: sync 1 must fetch group A's entitlements fresh") + + // Group B appears with a FRESH entitlement claiming the same + // exclusion group's default. A full resync would fail; the warm sync + // (group A's page replays) must fail identically. + groupB, entB, err := inner.AddGroup(ctx, "group_b") + require.NoError(t, err) + entBAnnos := annotations.Annotations(entB.GetAnnotations()) + entBAnnos.Update(defaultExclusionGroup) + entB.SetAnnotations(entBAnnos) + _ = groupB + + sync2 := filepath.Join(tmpDir, "sync2.c1z") + store, err := dotc1z.NewStore(ctx, sync2, + dotc1z.WithEngine(c1zstore.EnginePebble), + dotc1z.WithTmpDir(tmpDir), + ) + require.NoError(t, err) + opts := append([]SyncOpt{WithConnectorStore(store), WithTmpDir(tmpDir), WithPreviousSyncC1ZPath(sync1)}, workerOpts...) + syncer, err := NewSyncer(ctx, mc, opts...) + require.NoError(t, err) + err = syncer.Sync(ctx) + require.Error(t, err, + "a duplicate exclusion-group default split across a replayed and a fresh scope must fail the warm sync like a cold sync would") + require.Contains(t, err.Error(), "multiple default entitlements") + require.NoError(t, syncer.Close(ctx)) + require.NotZero(t, mc.entLookupHits, "sanity: sync 2 must have replayed group A's entitlements page") +} + +// --------------------------------------------------------------------- +// Partial syncs must not replay. +// --------------------------------------------------------------------- + +// TestSourceCache_PartialSyncDegradesToCold pins the full-sync-only gate: +// a partial (targeted) sync with a warm previous file must install the +// no-op lookup — every scope misses, the connector fetches fresh — rather +// than replaying whole scopes into a targeted subset (which could +// resurrect out-of-target rows, and whose related-resource fetch only +// runs over response rows). +func TestSourceCache_PartialSyncDegradesToCold(t *testing.T) { + ctx, err := logging.Init(t.Context()) + require.NoError(t, err) + tmpDir := t.TempDir() + + group, ent, alice, _ := sourceCacheTestFixtures(t) + g1 := gt.NewGrant(group, "member", alice) + + mc := newSourceCacheMockConnector() + mc.AddResource(ctx, group) + mc.AddResource(ctx, alice) + mc.entDB[group.GetId().GetResource()] = []*v2.Entitlement{ent} + mc.grantDB[group.GetId().GetResource()] = []*v2.Grant{g1} + mc.etagByResource[group.GetId().GetResource()] = `W/"etag-v1"` + + sync1 := filepath.Join(tmpDir, "sync1.c1z") + runSourceCacheSync(ctx, t, mc, sync1, "", tmpDir) + require.Zero(t, mc.lookupHits) + + // Partial sync targeting the group, with the warm file supplied: the + // connector's lookups must all MISS despite the warm previous sync. + sync2 := filepath.Join(tmpDir, "sync2.c1z") + store, err := dotc1z.NewStore(ctx, sync2, + dotc1z.WithEngine(c1zstore.EnginePebble), + dotc1z.WithTmpDir(tmpDir), + ) + require.NoError(t, err) + missesBefore := mc.lookupMisses + syncer, err := NewSyncer(ctx, mc, + WithConnectorStore(store), + WithTmpDir(tmpDir), + WithPreviousSyncC1ZPath(sync1), + WithTargetedSyncResources([]*v2.Resource{group}), + ) + require.NoError(t, err) + require.NoError(t, syncer.Sync(ctx)) + require.NoError(t, syncer.Close(ctx)) + + require.Zero(t, mc.lookupHits, "a partial sync must never get a source-cache lookup hit") + require.Greater(t, mc.lookupMisses, missesBefore, + "sanity: the connector must have looked up and missed (no-op lookup installed)") +} + +// --------------------------------------------------------------------- +// Compacted previous syncs must not replay. +// --------------------------------------------------------------------- + +// TestSourceCache_CompactedPreviousSyncDegradesToCold pins the METADATA +// gate: a previous sync whose run record carries the compacted flag must +// never serve replay — independent of its manifest contents. The test +// leaves the manifest fully intact and only flips the flag, so a pass +// proves the gate itself (not the compactor's manifest clearing) causes +// the degradation. Compaction invalidates validators categorically: a +// keep-newer merge is not a row set any input's etag describes. +func TestSourceCache_CompactedPreviousSyncDegradesToCold(t *testing.T) { + ctx, err := logging.Init(t.Context()) + require.NoError(t, err) + tmpDir := t.TempDir() + + group, ent, alice, _ := sourceCacheTestFixtures(t) + g1 := gt.NewGrant(group, "member", alice) + + mc := newSourceCacheMockConnector() + mc.AddResource(ctx, group) + mc.AddResource(ctx, alice) + mc.entDB[group.GetId().GetResource()] = []*v2.Entitlement{ent} + mc.grantDB[group.GetId().GetResource()] = []*v2.Grant{g1} + mc.etagByResource[group.GetId().GetResource()] = `W/"etag-v1"` + + sync1 := filepath.Join(tmpDir, "sync1.c1z") + runSourceCacheSync(ctx, t, mc, sync1, "", tmpDir) + + // Flip ONLY the compacted flag on sync 1's run record; its + // source-cache manifest stays intact and would otherwise replay. + marker, err := dotc1z.NewStore(ctx, sync1, + dotc1z.WithEngine(c1zstore.EnginePebble), + dotc1z.WithTmpDir(tmpDir), + ) + require.NoError(t, err) + run, err := marker.SyncMeta().LatestFullSync(ctx) + require.NoError(t, err) + require.NotNil(t, run) + engineHolder, ok := any(marker).(interface{ PebbleEngine() *pebbleengine.Engine }) + require.True(t, ok) + rec, err := engineHolder.PebbleEngine().GetSyncRunRecord(ctx, run.ID) + require.NoError(t, err) + rec.SetCompacted(true) + require.NoError(t, engineHolder.PebbleEngine().PutSyncRunRecord(ctx, rec)) + require.True(t, pebbleengine.MarkStoreDirty(marker), "the flag write must persist through Close") + require.NoError(t, marker.Close(ctx)) + + // Warm sync against the compacted-marked file: every lookup must + // MISS despite the intact manifest. + sync2 := filepath.Join(tmpDir, "sync2.c1z") + missesBefore := mc.lookupMisses + runSourceCacheSync(ctx, t, mc, sync2, sync1, tmpDir) + require.Zero(t, mc.lookupHits, + "a compacted previous sync must never produce a source-cache lookup hit") + require.Greater(t, mc.lookupMisses, missesBefore, + "sanity: the connector must have looked up and missed (no-op lookup installed)") +} + +// --------------------------------------------------------------------- +// 5. Continuation answers accumulated across bounces must stay +// acceptable to the connector-side answer parser. +// --------------------------------------------------------------------- + +// TestSourceCacheContinuation_AnswerAccumulationBeyondOneAsk pins the +// cap coherence between the two sides of the ask/answer continuation: +// answers accumulate across bounces (every re-invoke carries the union), +// so a maximal first ask (4096 queries, the ask cap) followed by ONE +// late scope — explicitly legal per the annotation contract — produces +// 4097 answers on the next re-invoke. The connector-side parser must +// accept that union; capping it at one ask's size hard-fails a legal +// exchange. +func TestSourceCacheContinuation_AnswerAccumulationBeyondOneAsk(t *testing.T) { + ctx, err := logging.Init(t.Context()) + require.NoError(t, err) + + store := &stubSourceCacheStore{} + s := &syncer{ + state: newState(), + sourceCache: syncerSourceCache{ + enabled: true, + current: store, + prev: store, + lookup: staticLookup{etag: "e"}, + replayedScopes: &replayedScopeSet{}, + }, + } + + bigAsk := make([]sourcecache.Query, 0, 4096) + for i := 0; i < 4096; i++ { + bigAsk = append(bigAsk, sourcecache.Query{ + RowKind: sourcecache.RowKindGrants, + ScopeHash: fmt.Sprintf("scope-%04d", i), + }) + } + lateAsk := []sourcecache.Query{{RowKind: sourcecache.RowKindGrants, ScopeHash: "late-scope"}} + + err = s.withSourceCacheContinuation(ctx, "test-op", func(extra annotations.Annotations, attempt int) (listAttempt, error) { + answersMsg := &v2.SourceCacheLookupAnswers{} + hasAnswers, err := extra.Pick(answersMsg) + if err != nil { + return listAttempt{}, err + } + if hasAnswers { + // The connector-side boundary every re-invoke passes through + // (connectorbuilder's continuationOpAttrs): the syncer's + // accumulated union must never exceed what it accepts. + if _, err := sourcecache.AnswersFromProto(answersMsg); err != nil { + return listAttempt{}, err + } + } + switch attempt { + case 0: + require.False(t, hasAnswers) + return listAttempt{annos: annotations.New(sourcecache.AskProto(bigAsk))}, nil + case 1: + require.Len(t, answersMsg.GetAnswers(), 4096) + // Phase 2 legally queries one scope it did not ask before. + return listAttempt{annos: annotations.New(sourcecache.AskProto(lateAsk))}, nil + default: + require.Len(t, answersMsg.GetAnswers(), 4097) + return listAttempt{}, nil + } + }) + require.NoError(t, err, + "a maximal first ask plus one late scope is a legal exchange; the accumulated answer union must be accepted end to end") +} + +// TestSourceCacheContinuation_OversizedAskFailsLoudly pins the intended +// hard boundary on ONE ask: more than 4096 queries in a single ask is a +// connector bug (shard via SpawnCursors instead) and must fail the sync +// with a descriptive error, not silently truncate. +func TestSourceCacheContinuation_OversizedAskFailsLoudly(t *testing.T) { + ctx, err := logging.Init(t.Context()) + require.NoError(t, err) + + store := &stubSourceCacheStore{} + s := &syncer{ + state: newState(), + sourceCache: syncerSourceCache{ + enabled: true, + current: store, + prev: store, + lookup: staticLookup{etag: "e"}, + replayedScopes: &replayedScopeSet{}, + }, + } + + oversized := make([]sourcecache.Query, 0, 4097) + for i := 0; i < 4097; i++ { + oversized = append(oversized, sourcecache.Query{ + RowKind: sourcecache.RowKindGrants, + ScopeHash: fmt.Sprintf("scope-%04d", i), + }) + } + err = s.withSourceCacheContinuation(ctx, "test-op", func(extra annotations.Annotations, attempt int) (listAttempt, error) { + return listAttempt{annos: annotations.New(sourcecache.AskProto(oversized))}, nil + }) + require.Error(t, err) + require.Contains(t, err.Error(), "max 4096") +} diff --git a/pkg/sync/source_cache_stats.go b/pkg/sync/source_cache_stats.go new file mode 100644 index 000000000..0e43daee2 --- /dev/null +++ b/pkg/sync/source_cache_stats.go @@ -0,0 +1,143 @@ +package sync //nolint:revive,nolintlint // we can't change the package name for backwards compatibility + +// Source-cache counters, following the sync-stats model (see the stats +// fields on state and syncSummaryFields): recorded through the state so +// they persist in the checkpointed sync token — a warm sync that suspends +// and resumes keeps its counts — and reported once in the sync summary. + +import ( + "context" + + "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" + "go.uber.org/zap" +) + +// SourceCacheStats accumulates per-sync source-cache counters: replay +// effectiveness, delta tombstones, spawn fan-out, and the lookup +// continuation (ask/answer) protocol. All fields are cumulative for the +// sync; the zero value means "feature unused". +type SourceCacheStats struct { + // ScopesReplayed counts pages answered with SourceCacheReplay. With + // ScopesStamped it gives the sync's replay hit ratio — the headline + // warm-sync health metric. + ScopesReplayed int64 `json:"scopes_replayed,omitempty"` + // ScopesStamped counts manifest entries written for NON-replayed + // (cold/fresh) pages. Interim pages with no validator count in + // neither bucket. + ScopesStamped int64 `json:"scopes_stamped,omitempty"` + // RowsReplayed counts rows bulk-copied from the previous sync, keyed + // by row kind (resources / entitlements / grants). + RowsReplayed map[string]int64 `json:"rows_replayed,omitempty"` + // OverlayRows counts connector-emitted rows upserted on top of a + // replayed base (delta adds/changes). + OverlayRows int64 `json:"overlay_rows,omitempty"` + + // TombstoneIDs / TombstonePrincipals count tombstones received on + // replay/scope annotations; RowsDeleted counts rows they removed + // (where the delete path reports it). A delta connector applying zero + // tombstones for weeks is a signal worth charting. + TombstoneIDs int64 `json:"tombstone_ids,omitempty"` + TombstonePrincipals int64 `json:"tombstone_principals,omitempty"` + RowsDeleted int64 `json:"rows_deleted,omitempty"` + + // SpawnedCursors counts sibling actions enqueued via SpawnCursors + // (pairs with the per-response cap). + SpawnedCursors int64 `json:"spawned_cursors,omitempty"` + + // Lookup continuation (ask/answer) counters. BouncesByOp splits + // bounces by list-op kind, distinguishing planner asks (expected: one + // per planning page) from per-row asks (a batching opportunity). + // AnswersTruncated counts found answers degraded to not-found (cold + // fetch) because their etags did not fit the per-request answer size + // budget; nonzero means a connector is batching more fat-token scopes + // per ask than the transport can carry and should shard via + // SpawnCursors. + LookupBounces int64 `json:"lookup_bounces,omitempty"` + LookupRequestsBounced int64 `json:"lookup_requests_bounced,omitempty"` + LookupScopesAsked int64 `json:"lookup_scopes_asked,omitempty"` + LookupAnsweredFound int64 `json:"lookup_answered_found,omitempty"` + LookupAnsweredNotFound int64 `json:"lookup_answered_not_found,omitempty"` + LookupAnswersTruncated int64 `json:"lookup_answers_truncated,omitempty"` + LookupCapFailures int64 `json:"lookup_cap_failures,omitempty"` + LookupBouncesByOp map[string]int64 `json:"lookup_bounces_by_op,omitempty"` +} + +func (s *SourceCacheStats) merge(delta SourceCacheStats) { + s.ScopesReplayed += delta.ScopesReplayed + s.ScopesStamped += delta.ScopesStamped + for kind, n := range delta.RowsReplayed { + if s.RowsReplayed == nil { + s.RowsReplayed = make(map[string]int64, len(delta.RowsReplayed)) + } + s.RowsReplayed[kind] += n + } + s.OverlayRows += delta.OverlayRows + s.TombstoneIDs += delta.TombstoneIDs + s.TombstonePrincipals += delta.TombstonePrincipals + s.RowsDeleted += delta.RowsDeleted + s.SpawnedCursors += delta.SpawnedCursors + s.LookupBounces += delta.LookupBounces + s.LookupRequestsBounced += delta.LookupRequestsBounced + s.LookupScopesAsked += delta.LookupScopesAsked + s.LookupAnsweredFound += delta.LookupAnsweredFound + s.LookupAnsweredNotFound += delta.LookupAnsweredNotFound + s.LookupAnswersTruncated += delta.LookupAnswersTruncated + s.LookupCapFailures += delta.LookupCapFailures + for op, n := range delta.LookupBouncesByOp { + if s.LookupBouncesByOp == nil { + s.LookupBouncesByOp = make(map[string]int64, len(delta.LookupBouncesByOp)) + } + s.LookupBouncesByOp[op] += n + } +} + +// copy returns a deep copy (maps included). +func (s *SourceCacheStats) copy() *SourceCacheStats { + out := *s + if s.RowsReplayed != nil { + out.RowsReplayed = make(map[string]int64, len(s.RowsReplayed)) + for k, v := range s.RowsReplayed { + out.RowsReplayed[k] = v + } + } + if s.LookupBouncesByOp != nil { + out.LookupBouncesByOp = make(map[string]int64, len(s.LookupBouncesByOp)) + for k, v := range s.LookupBouncesByOp { + out.LookupBouncesByOp[k] = v + } + } + return &out +} + +func (st *state) AddSourceCacheStats(delta SourceCacheStats) { + st.mtx.Lock() + defer st.mtx.Unlock() + if st.sourceCacheStats == nil { + st.sourceCacheStats = &SourceCacheStats{} + } + st.sourceCacheStats.merge(delta) +} + +func (st *state) SourceCacheStatsSnapshot() *SourceCacheStats { + st.mtx.RLock() + defer st.mtx.RUnlock() + if st.sourceCacheStats == nil { + return nil + } + return st.sourceCacheStats.copy() +} + +// logSourceCacheStats emits the source-cache summary line when the +// feature saw any use. Deliberately NOT gated on recordStats: replay is +// opt-in per connector and these counters are its rollout telemetry (the +// replay hit ratio, and the bounce counts the consumer briefs chart). +func (s *syncer) logSourceCacheStats(ctx context.Context) { + if s.state == nil { + return + } + stats := s.state.SourceCacheStatsSnapshot() + if stats == nil { + return + } + ctxzap.Extract(ctx).Info("source cache stats", zap.Any("source_cache_stats", stats)) +} diff --git a/pkg/sync/source_cache_stats_test.go b/pkg/sync/source_cache_stats_test.go new file mode 100644 index 000000000..0b93a9ee4 --- /dev/null +++ b/pkg/sync/source_cache_stats_test.go @@ -0,0 +1,115 @@ +package sync //nolint:revive,nolintlint // we can't change the package name for backwards compatibility + +import ( + "context" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/conductorone/baton-sdk/pkg/dotc1z" + "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" + "github.com/conductorone/baton-sdk/pkg/logging" +) + +// TestSourceCacheStats_TokenRoundTrip pins the resume property: the +// counters live in the checkpointed sync token, so a suspended warm sync +// keeps its replay/bounce counts across Marshal → Unmarshal. +func TestSourceCacheStats_TokenRoundTrip(t *testing.T) { + st := newState() + require.NoError(t, st.Unmarshal("")) + + st.AddSourceCacheStats(SourceCacheStats{ + ScopesReplayed: 3, + ScopesStamped: 2, + RowsReplayed: map[string]int64{"grants": 40, "resources": 5}, + OverlayRows: 4, + TombstoneIDs: 2, + LookupBounces: 6, + LookupBouncesByOp: map[string]int64{ + "sync-grants-for-resource": 6, + }, + }) + // Second merge accumulates. + st.AddSourceCacheStats(SourceCacheStats{ + ScopesReplayed: 1, + RowsReplayed: map[string]int64{"grants": 10}, + LookupBouncesByOp: map[string]int64{"sync-resources": 1}, + }) + + token, err := st.Marshal() + require.NoError(t, err) + + resumed := newState() + require.NoError(t, resumed.Unmarshal(token)) + got := resumed.SourceCacheStatsSnapshot() + require.NotNil(t, got, "stats must survive the token round trip") + require.Equal(t, int64(4), got.ScopesReplayed) + require.Equal(t, int64(2), got.ScopesStamped) + require.Equal(t, int64(50), got.RowsReplayed["grants"]) + require.Equal(t, int64(5), got.RowsReplayed["resources"]) + require.Equal(t, int64(4), got.OverlayRows) + require.Equal(t, int64(2), got.TombstoneIDs) + require.Equal(t, int64(6), got.LookupBounces) + require.Equal(t, int64(6), got.LookupBouncesByOp["sync-grants-for-resource"]) + require.Equal(t, int64(1), got.LookupBouncesByOp["sync-resources"]) + + // Resumed state keeps accumulating on top of the restored counters. + resumed.AddSourceCacheStats(SourceCacheStats{ScopesReplayed: 1}) + require.Equal(t, int64(5), resumed.SourceCacheStatsSnapshot().ScopesReplayed) + + // A fresh (empty-input) state reports nothing. + fresh := newState() + require.NoError(t, fresh.Unmarshal("")) + require.Nil(t, fresh.SourceCacheStatsSnapshot()) +} + +// TestSourceCacheStats_EndToEnd runs a cold sync then a warm +// continuation sync and asserts the recorded counters match the known +// shape: 5 groups → 5 stamped scopes cold; 5 replayed scopes, 5 bounces +// (one ask per group page), 10 rows replayed, zero cold pages warm. +func TestSourceCacheStats_EndToEnd(t *testing.T) { + ctx, err := logging.Init(context.Background()) + require.NoError(t, err) + tmpDir := t.TempDir() + mc := newContinuationMockConnector(t, 5) + + runWithStats := func(path, prevPath string) *SourceCacheStats { + store, err := dotc1z.NewStore(ctx, path, + dotc1z.WithEngine(c1zstore.EnginePebble), + dotc1z.WithTmpDir(tmpDir), + ) + require.NoError(t, err) + opts := []SyncOpt{WithConnectorStore(store), WithTmpDir(tmpDir)} + if prevPath != "" { + opts = append(opts, WithPreviousSyncC1ZPath(prevPath)) + } + sc, err := NewSyncer(ctx, mc, opts...) + require.NoError(t, err) + require.NoError(t, sc.Sync(ctx)) + impl, ok := sc.(*syncer) + require.True(t, ok) + stats := impl.state.SourceCacheStatsSnapshot() + require.NoError(t, sc.Close(ctx)) + return stats + } + + sync1 := filepath.Join(tmpDir, "sync1.c1z") + cold := runWithStats(sync1, "") + require.NotNil(t, cold) + require.Equal(t, int64(5), cold.ScopesStamped, "cold: every group page stamps a scope") + require.Zero(t, cold.ScopesReplayed) + require.Zero(t, cold.LookupBounces, "no offer on a cold sync, so no bounces") + + warm := runWithStats(filepath.Join(tmpDir, "sync2.c1z"), sync1) + require.NotNil(t, warm) + require.Equal(t, int64(5), warm.ScopesReplayed, "warm: every group page replays") + require.Zero(t, warm.ScopesStamped, "no cold pages on the warm sync") + require.Equal(t, int64(10), warm.RowsReplayed["grants"], "5 groups × 2 member grants replayed") + require.Equal(t, int64(5), warm.LookupBounces, "one ask bounce per group page") + require.Equal(t, int64(5), warm.LookupRequestsBounced) + require.Equal(t, int64(5), warm.LookupScopesAsked) + require.Equal(t, int64(5), warm.LookupAnsweredFound) + require.Equal(t, int64(5), warm.LookupBouncesByOp["sync-grants-for-resource"]) + require.Zero(t, warm.LookupCapFailures) +} diff --git a/pkg/sync/source_cache_test.go b/pkg/sync/source_cache_test.go new file mode 100644 index 000000000..1f2714452 --- /dev/null +++ b/pkg/sync/source_cache_test.go @@ -0,0 +1,568 @@ +package sync //nolint:revive,nolintlint // we can't change the package name for backwards compatibility + +import ( + "context" + "fmt" + "path/filepath" + "sync" + "testing" + + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + "github.com/conductorone/baton-sdk/pkg/annotations" + "github.com/conductorone/baton-sdk/pkg/connectorstore" + "github.com/conductorone/baton-sdk/pkg/dotc1z" + "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" + "github.com/conductorone/baton-sdk/pkg/logging" + "github.com/conductorone/baton-sdk/pkg/sourcecache" + et "github.com/conductorone/baton-sdk/pkg/types/entitlement" + gt "github.com/conductorone/baton-sdk/pkg/types/grant" + rs "github.com/conductorone/baton-sdk/pkg/types/resource" +) + +// sourceCacheMockConnector emulates a connector against an upstream with +// cheap change detection. Every grants page for a resource has a stable +// scope; the connector looks up the previous validator via the lookup the +// syncer installs (SetSourceCache) and either: +// +// - miss, or validator != current upstream etag: return fresh rows + +// SourceCacheScope (a 200), +// - hit with matching etag: return no rows + SourceCacheReplay (a 304), +// - hit in delta mode: return only changed rows + overlay replay with +// tombstones and a rotated token (a Graph-style delta round). +type sourceCacheMockConnector struct { + *mockConnector + + mu sync.Mutex + lookup sourcecache.Lookup + + // etagByResource is the upstream's CURRENT validator per resource. + etagByResource map[string]string + + // delta mode: overlay rows + deletions returned on a lookup hit. + deltaMode bool + deltaAdds map[string][]*v2.Grant + deltaDeletedIDs map[string][]string + // deltaDeletedPrincipals are bare principal-id tombstones, emitted on + // the SourceCacheScope annotation (the per-page channel) rather than + // the replay annotation — exercising both proposal A and B paths. + deltaDeletedPrincipals map[string][]string + + lookupHits int + lookupMisses int +} + +func newSourceCacheMockConnector() *sourceCacheMockConnector { + mc := &sourceCacheMockConnector{ + mockConnector: newMockConnector(), + etagByResource: map[string]string{}, + deltaAdds: map[string][]*v2.Grant{}, + deltaDeletedIDs: map[string][]string{}, + deltaDeletedPrincipals: map[string][]string{}, + } + mc.rtDB = append(mc.rtDB, groupResourceType, userResourceType) + return mc +} + +func (mc *sourceCacheMockConnector) SetSourceCache(_ context.Context, lookup sourcecache.Lookup) { + mc.mu.Lock() + defer mc.mu.Unlock() + if lookup == nil { + lookup = sourcecache.NoopLookup{} + } + mc.lookup = lookup +} + +func (mc *sourceCacheMockConnector) Validate(context.Context, *v2.ConnectorServiceValidateRequest, ...grpc.CallOption) (*v2.ConnectorServiceValidateResponse, error) { + return v2.ConnectorServiceValidateResponse_builder{ + Annotations: annotations.New(v2.SourceCacheCapability_builder{ + Mode: v2.SourceCacheCapability_MODE_READ_WRITE, + }.Build()), + }.Build(), nil +} + +func grantScopeForResource(resourceID string) string { + return sourcecache.HashScope("mock://grants/" + resourceID) +} + +func (mc *sourceCacheMockConnector) ListGrants(ctx context.Context, in *v2.GrantsServiceListGrantsRequest, _ ...grpc.CallOption) (*v2.GrantsServiceListGrantsResponse, error) { + resourceID := in.GetResource().GetId().GetResource() + scope := grantScopeForResource(resourceID) + + mc.mu.Lock() + lookup := mc.lookup + currentEtag := mc.etagByResource[resourceID] + deltaMode := mc.deltaMode + adds := mc.deltaAdds[resourceID] + deleted := mc.deltaDeletedIDs[resourceID] + deletedPrincipals := mc.deltaDeletedPrincipals[resourceID] + mc.mu.Unlock() + + if lookup == nil { + lookup = sourcecache.NoopLookup{} + } + entry, found, err := lookup.LookupPreviousSourceCache(ctx, sourcecache.RowKindGrants, scope) + if err != nil { + return nil, err + } + + if found && deltaMode { + // Delta round: replay the base, overlay changes, tombstone + // removals, rotate the token. Canonical-id tombstones ride the + // replay annotation; bare principal-id tombstones ride the scope + // annotation (the per-page channel every page of a round can use). + mc.mu.Lock() + mc.lookupHits++ + mc.mu.Unlock() + return v2.GrantsServiceListGrantsResponse_builder{ + List: adds, + Annotations: annotations.New( + v2.SourceCacheReplay_builder{ + ScopeHash: scope, + Etag: currentEtag, // new token for this round + Overlay: true, + DeletedIds: deleted, + }.Build(), + v2.SourceCacheScope_builder{ + ScopeHash: scope, + DeletedPrincipalIds: deletedPrincipals, + }.Build(), + ), + }.Build(), nil + } + + if found && entry.ETag == currentEtag { + // Conditional-request 304: nothing changed upstream. + mc.mu.Lock() + mc.lookupHits++ + mc.mu.Unlock() + return v2.GrantsServiceListGrantsResponse_builder{ + List: []*v2.Grant{}, + Annotations: annotations.New(v2.SourceCacheReplay_builder{ + ScopeHash: scope, + Etag: currentEtag, + }.Build()), + }.Build(), nil + } + + // Fresh fetch (miss or changed). + mc.mu.Lock() + mc.lookupMisses++ + mc.mu.Unlock() + return v2.GrantsServiceListGrantsResponse_builder{ + List: mc.grantDB[resourceID], + Annotations: annotations.New(v2.SourceCacheScope_builder{ + ScopeHash: scope, + Etag: currentEtag, + }.Build()), + }.Build(), nil +} + +// runSourceCacheSync runs one full sync into a fresh Pebble c1z at path, +// optionally replaying from prevPath. +func runSourceCacheSync(ctx context.Context, t *testing.T, mc *sourceCacheMockConnector, path, prevPath, tmpDir string, extraOpts ...SyncOpt) { + t.Helper() + store, err := dotc1z.NewStore(ctx, path, + dotc1z.WithEngine(c1zstore.EnginePebble), + dotc1z.WithTmpDir(tmpDir), + ) + require.NoError(t, err) + opts := []SyncOpt{WithConnectorStore(store), WithTmpDir(tmpDir)} + if prevPath != "" { + opts = append(opts, WithPreviousSyncC1ZPath(prevPath)) + } + opts = append(opts, extraOpts...) + syncer, err := NewSyncer(ctx, mc, opts...) + require.NoError(t, err) + require.NoError(t, syncer.Sync(ctx)) + require.NoError(t, syncer.Close(ctx)) +} + +// workerCounts are the worker configurations every source-cache scenario +// runs under: 0 (sequential) and 4 (parallel). Parallel sync + replay +// shares one engine across workers, so the scenarios double as races on +// concurrent Replay*/PutSourceCacheEntry batches and expansion arming. +var workerCounts = []int{0, 4} + +func workerSyncOpts(workers int) []SyncOpt { + if workers > 0 { + return []SyncOpt{WithWorkerCount(workers)} + } + return nil +} + +// listGrantsInFile reopens a finished Pebble c1z and returns its grants +// keyed by grant ID. +func listGrantsInFile(ctx context.Context, t *testing.T, path string) map[string]*v2.Grant { + t.Helper() + reopen, err := dotc1z.NewStore(ctx, path, + dotc1z.WithEngine(c1zstore.EnginePebble), + dotc1z.WithReadOnly(true), + ) + require.NoError(t, err) + defer func() { _ = reopen.Close(ctx) }() + + sync, err := reopen.SyncMeta().LatestFullSync(ctx) + require.NoError(t, err) + require.NotNil(t, sync) + require.NoError(t, reopen.SetCurrentSync(ctx, sync.ID)) + + out := map[string]*v2.Grant{} + pageToken := "" + for { + resp, err := reopen.ListGrants(ctx, v2.GrantsServiceListGrantsRequest_builder{PageToken: pageToken}.Build()) + require.NoError(t, err) + for _, g := range resp.GetList() { + out[g.GetId()] = g + } + pageToken = resp.GetNextPageToken() + if pageToken == "" { + break + } + } + return out +} + +func sourceCacheTestFixtures(t *testing.T) (*v2.Resource, *v2.Entitlement, *v2.Resource, *v2.Resource) { + t.Helper() + group, err := rs.NewGroupResource("g1", groupResourceType, "g1", nil) + require.NoError(t, err) + ent := et.NewAssignmentEntitlement(group, "member", et.WithGrantableTo(groupResourceType, userResourceType)) + ent.SetSlug("member") + alice, err := rs.NewUserResource("alice", userResourceType, "alice", nil, rs.WithAnnotation(&v2.SkipEntitlementsAndGrants{})) + require.NoError(t, err) + bob, err := rs.NewUserResource("bob", userResourceType, "bob", nil, rs.WithAnnotation(&v2.SkipEntitlementsAndGrants{})) + require.NoError(t, err) + return group, ent, alice, bob +} + +// TestSourceCache_ConditionalRequestReplay is the GitHub-shape end-to-end +// test: sync 1 stamps rows and records the scope's etag; sync 2's lookup +// hits, the connector answers with a 304-style replay, and the grants are +// carried into the new c1z without the connector emitting any rows. +func TestSourceCache_ConditionalRequestReplay(t *testing.T) { + for _, workers := range workerCounts { + t.Run(fmt.Sprintf("workers=%d", workers), func(t *testing.T) { + runConditionalRequestReplayScenario(t, workerSyncOpts(workers)) + }) + } +} + +func runConditionalRequestReplayScenario(t *testing.T, workerOpts []SyncOpt) { + ctx, err := logging.Init(t.Context()) + require.NoError(t, err) + tmpDir := t.TempDir() + + group, ent, alice, bob := sourceCacheTestFixtures(t) + g1 := gt.NewGrant(group, "member", alice) + g2 := gt.NewGrant(group, "member", bob) + + mc := newSourceCacheMockConnector() + mc.AddResource(ctx, group) + mc.AddResource(ctx, alice) + mc.AddResource(ctx, bob) + mc.entDB[group.GetId().GetResource()] = []*v2.Entitlement{ent} + mc.grantDB[group.GetId().GetResource()] = []*v2.Grant{g1, g2} + mc.etagByResource[group.GetId().GetResource()] = `W/"etag-v1"` + + sync1 := filepath.Join(tmpDir, "sync1.c1z") + sync2 := filepath.Join(tmpDir, "sync2.c1z") + + runSourceCacheSync(ctx, t, mc, sync1, "", tmpDir, workerOpts...) + require.Zero(t, mc.lookupHits, "sync 1 has no previous sync to hit") + + // Upstream unchanged: sync 2 must replay. + runSourceCacheSync(ctx, t, mc, sync2, sync1, tmpDir, workerOpts...) + require.NotZero(t, mc.lookupHits, "sync 2's lookup must hit the etag persisted by sync 1") + + grants := listGrantsInFile(ctx, t, sync2) + require.Contains(t, grants, g1.GetId()) + require.Contains(t, grants, g2.GetId()) + + // And sync 2's file is usable as the NEXT previous sync: manifest + // entry + stamped rows were carried forward (a third sync replays too). + sync3 := filepath.Join(tmpDir, "sync3.c1z") + hitsBefore := mc.lookupHits + runSourceCacheSync(ctx, t, mc, sync3, sync2, tmpDir, workerOpts...) + require.Greater(t, mc.lookupHits, hitsBefore, "a replay-only sync must remain usable as the next replay source") + grants3 := listGrantsInFile(ctx, t, sync3) + require.Contains(t, grants3, g1.GetId()) + require.Contains(t, grants3, g2.GetId()) +} + +// TestSourceCache_DeltaOverlayAndDeletes is the Microsoft Graph-shape +// test: the second sync replays the base, upserts changed rows on top, +// applies @removed tombstones, and rotates the token. +func TestSourceCache_DeltaOverlayAndDeletes(t *testing.T) { + for _, workers := range workerCounts { + t.Run(fmt.Sprintf("workers=%d", workers), func(t *testing.T) { + runDeltaOverlayAndDeletesScenario(t, workerSyncOpts(workers)) + }) + } +} + +func runDeltaOverlayAndDeletesScenario(t *testing.T, workerOpts []SyncOpt) { + ctx, err := logging.Init(t.Context()) + require.NoError(t, err) + tmpDir := t.TempDir() + + group, ent, alice, bob := sourceCacheTestFixtures(t) + carol, err := rs.NewUserResource("carol", userResourceType, "carol", nil, rs.WithAnnotation(&v2.SkipEntitlementsAndGrants{})) + require.NoError(t, err) + + g1 := gt.NewGrant(group, "member", alice) + g2 := gt.NewGrant(group, "member", bob) + g3 := gt.NewGrant(group, "member", carol) + + mc := newSourceCacheMockConnector() + mc.AddResource(ctx, group) + mc.AddResource(ctx, alice) + mc.AddResource(ctx, bob) + mc.AddResource(ctx, carol) + mc.entDB[group.GetId().GetResource()] = []*v2.Entitlement{ent} + mc.grantDB[group.GetId().GetResource()] = []*v2.Grant{g1, g2} + mc.etagByResource[group.GetId().GetResource()] = "delta-token-1" + + sync1 := filepath.Join(tmpDir, "sync1.c1z") + sync2 := filepath.Join(tmpDir, "sync2.c1z") + + runSourceCacheSync(ctx, t, mc, sync1, "", tmpDir, workerOpts...) + + // Delta round: bob removed (canonical-id tombstone on the replay + // annotation), carol added, token rotates. + mc.mu.Lock() + mc.deltaMode = true + mc.etagByResource[group.GetId().GetResource()] = "delta-token-2" + mc.deltaAdds[group.GetId().GetResource()] = []*v2.Grant{g3} + mc.deltaDeletedIDs[group.GetId().GetResource()] = []string{g2.GetId()} + mc.mu.Unlock() + + runSourceCacheSync(ctx, t, mc, sync2, sync1, tmpDir, workerOpts...) + require.NotZero(t, mc.lookupHits) + + grants := listGrantsInFile(ctx, t, sync2) + require.Contains(t, grants, g1.GetId(), "replayed base row must survive") + require.Contains(t, grants, g3.GetId(), "overlay row must be upserted") + require.NotContains(t, grants, g2.GetId(), "tombstoned row must be deleted") + + // Second delta round: carol removed by BARE PRINCIPAL ID via the + // scope annotation — no principal type, no canonical-id + // reconstruction (the Entra @removed shape). + sync3 := filepath.Join(tmpDir, "sync3.c1z") + mc.mu.Lock() + mc.etagByResource[group.GetId().GetResource()] = "delta-token-3" + mc.deltaAdds[group.GetId().GetResource()] = nil + mc.deltaDeletedIDs[group.GetId().GetResource()] = nil + mc.deltaDeletedPrincipals[group.GetId().GetResource()] = []string{"carol"} + mc.mu.Unlock() + + runSourceCacheSync(ctx, t, mc, sync3, sync2, tmpDir, workerOpts...) + + grants3 := listGrantsInFile(ctx, t, sync3) + require.Contains(t, grants3, g1.GetId(), "alice's replayed row must survive the principal tombstone") + require.NotContains(t, grants3, g3.GetId(), "carol's row must die from a bare principal-id tombstone") + require.NotContains(t, grants3, g2.GetId()) +} + +// TestSourceCache_ReplayArmsExpansion pins the expansion-arming contract: +// a sync in which EVERY grants page replays must still run grant +// expansion, because replayed rows carry needs_expansion but never pass +// GrantExpandable-annotated rows through the syncer's connector-response +// path (the only other thing that arms the expansion phase). +func TestSourceCache_ReplayArmsExpansion(t *testing.T) { + for _, workers := range workerCounts { + t.Run(fmt.Sprintf("workers=%d", workers), func(t *testing.T) { + runReplayArmsExpansionScenario(t, workerSyncOpts(workers)) + }) + } +} + +func runReplayArmsExpansionScenario(t *testing.T, workerOpts []SyncOpt) { + ctx, err := logging.Init(t.Context()) + require.NoError(t, err) + tmpDir := t.TempDir() + + mc := newSourceCacheMockConnector() + mc.rtDB = []*v2.ResourceType{groupResourceType, userResourceType} + + // Standard expansion topology: alice is a member of group2, and + // group2 is an expandable member of group1 — expansion derives + // alice's grant on group1's member entitlement. + group1, group1Ent, err := mc.AddGroup(ctx, "group_1") + require.NoError(t, err) + group2, group2Ent, err := mc.AddGroup(ctx, "group_2") + require.NoError(t, err) + alice, err := mc.AddUser(ctx, "alice") + require.NoError(t, err) + _ = mc.AddGroupMember(ctx, group2, alice) + _ = mc.AddGroupMember(ctx, group1, group2, group2Ent) + + mc.etagByResource[group1.GetId().GetResource()] = "etag-g1" + mc.etagByResource[group2.GetId().GetResource()] = "etag-g2" + + derivedGrantID := gt.NewGrant(group1, "member", alice).GetId() + + sync1 := filepath.Join(tmpDir, "sync1.c1z") + sync2 := filepath.Join(tmpDir, "sync2.c1z") + + runSourceCacheSync(ctx, t, mc, sync1, "", tmpDir, workerOpts...) + require.Contains(t, listGrantsInFile(ctx, t, sync1), derivedGrantID, + "sanity: sync 1's expansion must derive alice's group1 grant") + + // Sync 2 replays every grants page (upstream unchanged). Derived + // grants are never replayed (they carry no scope stamp), so if replay + // fails to arm expansion, SyncGrantExpansion is skipped and the + // derived grant vanishes from sync 2. + runSourceCacheSync(ctx, t, mc, sync2, sync1, tmpDir, workerOpts...) + require.NotZero(t, mc.lookupHits, "sanity: sync 2 must have replayed") + + grants2 := listGrantsInFile(ctx, t, sync2) + baseGrantID := gt.NewGrant(group1, "member", group2).GetId() + require.Contains(t, grants2, baseGrantID, "replayed expandable base grant missing") + require.Contains(t, grants2, derivedGrantID, + "sync 2 must run grant expansion over replayed rows: the derived grant is missing, "+ + "meaning SetNeedsExpansion was never armed by the replay path") + _ = group1Ent +} + +// TestSourceCache_ReplayWhileDegradedFails pins the fail-loud half of the +// error contract: a replay annotation arriving with no usable previous +// sync is a hard sync error, never a silent fallback (the connector has +// already skipped row generation). +func TestSourceCache_ReplayWhileDegradedFails(t *testing.T) { + ctx, err := logging.Init(t.Context()) + require.NoError(t, err) + tmpDir := t.TempDir() + + group, ent, alice, _ := sourceCacheTestFixtures(t) + g1 := gt.NewGrant(group, "member", alice) + + mc := newSourceCacheMockConnector() + mc.AddResource(ctx, group) + mc.AddResource(ctx, alice) + mc.entDB[group.GetId().GetResource()] = []*v2.Entitlement{ent} + mc.grantDB[group.GetId().GetResource()] = []*v2.Grant{g1} + mc.etagByResource[group.GetId().GetResource()] = "etag-v1" + + // Misbehaving connector: fabricate a lookup hit so it emits a replay + // annotation even though the syncer never installed a lookup with a + // previous source. + mc.SetSourceCache(ctx, staticLookup{etag: "etag-v1"}) + + store, err := dotc1z.NewStore(ctx, filepath.Join(tmpDir, "out.c1z"), + dotc1z.WithEngine(c1zstore.EnginePebble), + dotc1z.WithTmpDir(tmpDir), + ) + require.NoError(t, err) + syncer, err := NewSyncer(ctx, °radedLookupConnector{mc}, WithConnectorStore(store), WithTmpDir(tmpDir)) + require.NoError(t, err) + err = syncer.Sync(ctx) + require.Error(t, err, "replay with no previous sync must fail the sync loudly") + require.Contains(t, err.Error(), "source cache") + require.NoError(t, syncer.Close(ctx)) +} + +// stubSourceCacheStore fakes the store surface begin/finish touch, so the +// stats bookkeeping can be pinned without a real engine. +type stubSourceCacheStore struct { + dotc1z.SourceCacheStore + replayRows int64 + stamped []string +} + +func (s *stubSourceCacheStore) LookupSourceCacheEntry(context.Context, sourcecache.RowKind, string) (sourcecache.Entry, bool, error) { + return sourcecache.Entry{ETag: "prev-token"}, true, nil +} + +func (s *stubSourceCacheStore) ReplaySourceCache(context.Context, connectorstore.Reader, sourcecache.RowKind, string) (dotc1z.SourceCacheReplayResult, error) { + return dotc1z.SourceCacheReplayResult{Rows: s.replayRows}, nil +} + +func (s *stubSourceCacheStore) PutSourceCacheEntry(_ context.Context, _ sourcecache.RowKind, scopeHash, _ string) error { + s.stamped = append(s.stamped, scopeHash) + return nil +} + +// TestSourceCacheStats_OverlayRoundBooksOneHit pins two stats contracts: +// +// 1. Replay counters are recorded at FINISH, not begin — a page re-run +// after failing between replay and finish must not double-count. +// 2. A multi-page overlay round, whose rotated validator legally arrives +// on the FINAL page's scope annotation (no replay annotation there), +// books exactly one ScopesReplayed and zero ScopesStamped — not one +// of each, which would read a 100%-warm delta sync as 50%. +func TestSourceCacheStats_OverlayRoundBooksOneHit(t *testing.T) { + ctx, err := logging.Init(context.Background()) + require.NoError(t, err) + + store := &stubSourceCacheStore{replayRows: 2} + s := &syncer{ + state: newState(), + sourceCache: syncerSourceCache{ + enabled: true, + current: store, + prev: store, + replayedScopes: &replayedScopeSet{}, + }, + } + scope := sourcecache.HashScope("delta://groups") + + // Page 1: overlay replay, no etag yet (the delta token rotates on the + // final page). One overlay row rides along. + _, page1, err := s.beginSourceCachePage(ctx, sourcecache.RowKindGrants, + annotations.New(v2.SourceCacheReplay_builder{ScopeHash: scope, Overlay: true}.Build()), 1) + require.NoError(t, err) + require.True(t, page1.replayed) + + if stats := s.state.SourceCacheStatsSnapshot(); stats != nil { + require.Zero(t, stats.ScopesReplayed, "replay counters must not be recorded at begin (re-run double-count)") + } + + require.NoError(t, s.finishSourceCachePage(ctx, page1)) + + // Final page of the round: scope annotation with the NEW token, no + // replay annotation — this is a stamp write, but of the round's own + // scope, not a cold miss. + _, page2, err := s.beginSourceCachePage(ctx, sourcecache.RowKindGrants, + annotations.New(v2.SourceCacheScope_builder{ScopeHash: scope, Etag: "delta-token-2"}.Build()), 0) + require.NoError(t, err) + require.False(t, page2.replayed) + require.NoError(t, s.finishSourceCachePage(ctx, page2)) + require.Equal(t, []string{scope}, store.stamped, "the rotated validator must still be persisted") + + // A genuinely cold scope still counts as stamped. + coldScope := sourcecache.HashScope("delta://users") + _, page3, err := s.beginSourceCachePage(ctx, sourcecache.RowKindGrants, + annotations.New(v2.SourceCacheScope_builder{ScopeHash: coldScope, Etag: "etag-1"}.Build()), 3) + require.NoError(t, err) + require.NoError(t, s.finishSourceCachePage(ctx, page3)) + + stats := s.state.SourceCacheStatsSnapshot() + require.NotNil(t, stats) + require.Equal(t, int64(1), stats.ScopesReplayed, "one warm round, one hit") + require.Equal(t, int64(1), stats.ScopesStamped, "only the cold scope counts as stamped") + require.Equal(t, int64(2), stats.RowsReplayed[string(sourcecache.RowKindGrants)]) + require.Equal(t, int64(1), stats.OverlayRows) +} + +// staticLookup always hits with a fixed etag — used to simulate a +// connector that violates the only-replay-what-you-looked-up invariant. +type staticLookup struct{ etag string } + +func (s staticLookup) LookupPreviousSourceCache(context.Context, sourcecache.RowKind, string) (sourcecache.Entry, bool, error) { + return sourcecache.Entry{ETag: s.etag}, true, nil +} + +// degradedLookupConnector shadows SetSourceCache so the syncer cannot +// overwrite the fabricated lookup: the connector keeps "hitting" while the +// syncer's source-cache read side is degraded. +type degradedLookupConnector struct { + *sourceCacheMockConnector +} + +func (d *degradedLookupConnector) SetSourceCache(context.Context, sourcecache.Lookup) { + // Deliberately ignore the syncer's install; the mock keeps its + // fabricated staticLookup. +} diff --git a/pkg/sync/spawn_cursors_test.go b/pkg/sync/spawn_cursors_test.go new file mode 100644 index 000000000..82dc7147a --- /dev/null +++ b/pkg/sync/spawn_cursors_test.go @@ -0,0 +1,580 @@ +package sync //nolint:revive,nolintlint // we can't change the package name for backwards compatibility + +// Per-resource SpawnCursors: parallel warm revalidation for page-numbered +// APIs (the GitHub shape). On page one of a resource's grant collection the +// connector already knows every other page's URL and stored validator, so +// it answers page one and spawns pages 2..N as sibling actions; each +// spawned page independently hits or misses its source-cache lookup. +// +// Under test: +// - SpawnCursors on a per-resource grants response fans out sibling +// actions carrying the resource identity (no warning, no drops); +// - spawned pages are ordinary pages: replay on hit, cold fetch on miss +// (page boundary shifted), and a cold spawned page may CHAIN to a +// fresh page via NextPageToken (probe past a full tail); +// - reader-surface equivalence against a cold control sync, at +// workers=0 (sequential) and workers=4 (parallel). + +import ( + "context" + "fmt" + "math/rand" + "path/filepath" + "slices" + "sync" + "testing" + + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + "github.com/conductorone/baton-sdk/pkg/annotations" + "github.com/conductorone/baton-sdk/pkg/dotc1z" + "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" + "github.com/conductorone/baton-sdk/pkg/logging" + "github.com/conductorone/baton-sdk/pkg/sourcecache" + gt "github.com/conductorone/baton-sdk/pkg/types/grant" + rs "github.com/conductorone/baton-sdk/pkg/types/resource" +) + +const spawnPageSize = 4 + +// pagedSpawnMockConnector serves each group's member grants through +// numbered pages of size spawnPageSize. Page 1 carries its own rows plus +// SpawnCursors for every other page the previous sync recorded (warm) or +// that exist upstream (cold). Every page is its own source-cache scope +// (page-URL style). Membership mutations shift page contents, so mutated +// pages miss their lookup and refetch cold — including discovering fresh +// tail pages a warm spawn never knew about. +type pagedSpawnMockConnector struct { + *mockConnector + + mu sync.Mutex + lookup sourcecache.Lookup + + groupIDs []string + members map[string][]string // groupID -> userIDs, page k = members[k*4:(k+1)*4] + // prevMembers models the connector's knowledge of the STORED page + // chain (what the previous sync recorded): the spawn horizon on warm + // syncs, and the row counts behind the full-tail probe decision. The + // test snapshots it after each sync, mirroring what a real connector + // derives from its stored validators. + prevMembers map[string][]string + resByID map[string]*v2.Resource + + replayPages int + coldPages int + probePages int + + // failPageOnce injects a one-shot error when serving "gid/page" — + // simulating a crash while spawned siblings are pending in the + // checkpointed action stack. + failPageOnce map[string]bool + // servedPages counts servePage attempts per "gid/page" (including the + // failed attempt), so resume tests can see which pages re-executed. + servedPages map[string]int +} + +func newPagedSpawnMockConnector(t *testing.T, groupCount, membersPerGroup int) *pagedSpawnMockConnector { + t.Helper() + mc := &pagedSpawnMockConnector{ + mockConnector: newMockConnector(), + members: map[string][]string{}, + prevMembers: map[string][]string{}, + resByID: map[string]*v2.Resource{}, + failPageOnce: map[string]bool{}, + servedPages: map[string]int{}, + } + mc.rtDB = append(mc.rtDB, groupResourceType, userResourceType) + + for i := 0; i < 30; i++ { + uid := fmt.Sprintf("user-%02d", i) + u, err := rs.NewUserResource(uid, userResourceType, uid, nil, rs.WithAnnotation(&v2.SkipEntitlementsAndGrants{})) + require.NoError(t, err) + mc.AddResource(context.Background(), u) + mc.resByID[uid] = u + } + for i := 0; i < groupCount; i++ { + gid := fmt.Sprintf("group-%02d", i) + g, err := rs.NewGroupResource(gid, groupResourceType, gid, nil) + require.NoError(t, err) + mc.AddResource(context.Background(), g) + mc.resByID[gid] = g + mc.groupIDs = append(mc.groupIDs, gid) + mc.entDB[gid] = []*v2.Entitlement{mkMemberEnt(g)} + for m := 0; m < membersPerGroup; m++ { + mc.members[gid] = append(mc.members[gid], fmt.Sprintf("user-%02d", (i*3+m)%30)) + } + } + return mc +} + +func (mc *pagedSpawnMockConnector) SetSourceCache(_ context.Context, lookup sourcecache.Lookup) { + mc.mu.Lock() + defer mc.mu.Unlock() + if lookup == nil { + lookup = sourcecache.NoopLookup{} + } + mc.lookup = lookup +} + +func (mc *pagedSpawnMockConnector) Validate(context.Context, *v2.ConnectorServiceValidateRequest, ...grpc.CallOption) (*v2.ConnectorServiceValidateResponse, error) { + return v2.ConnectorServiceValidateResponse_builder{ + Annotations: annotations.New(v2.SourceCacheCapability_builder{ + Mode: v2.SourceCacheCapability_MODE_READ_WRITE, + }.Build()), + }.Build(), nil +} + +func pageScopeFor(gid string, page int) string { + return fmt.Sprintf("groups/%s/members?page=%d", gid, page) +} + +func (mc *pagedSpawnMockConnector) pageRows(gid string, page int) []string { + lo := page * spawnPageSize + hi := lo + spawnPageSize + all := mc.members[gid] + if lo >= len(all) { + return nil + } + if hi > len(all) { + hi = len(all) + } + return all[lo:hi] +} + +func (mc *pagedSpawnMockConnector) grantsFor(gid string, uids []string) []*v2.Grant { + out := make([]*v2.Grant, 0, len(uids)) + for _, uid := range uids { + out = append(out, gt.NewGrant(mc.resByID[gid], "member", mc.resByID[uid].GetId())) + } + return out +} + +// pageEtag is a content-hash-style validator: stable across syncs when the +// page's rows are unchanged, different otherwise. Growth always changes +// the stored tail page's content (rows shift into it), so the last stored +// page misses its lookup and chains — which is how fresh tail pages get +// discovered on warm syncs. +func pageEtag(gid string, page int, rows []string) string { + return fmt.Sprintf("etag:%s:%d:%v", gid, page, rows) +} + +// prevPageCount is the connector's stored-chain knowledge: how many pages +// the previous sync recorded for gid. The test snapshots members → +// prevMembers after each sync, mirroring what a real connector derives +// from its stored validators. +func (mc *pagedSpawnMockConnector) prevPageCount(gid string) int { + return (len(mc.prevMembers[gid]) + spawnPageSize - 1) / spawnPageSize +} + +func (mc *pagedSpawnMockConnector) snapshotPrev() { + mc.mu.Lock() + defer mc.mu.Unlock() + mc.prevMembers = map[string][]string{} + for gid, uids := range mc.members { + mc.prevMembers[gid] = append([]string(nil), uids...) + } +} + +// servePage answers one numbered page: lookup hit with matching validator → +// replay (no rows); miss/changed → cold rows + scope. A cold page chains to +// page+1 only when page+1 is NOT covered by a spawned sibling (i.e. beyond +// the stored chain) — that's the probe-past-tail path discovering pages the +// previous sync never had. +func (mc *pagedSpawnMockConnector) servePage(ctx context.Context, gid string, page int) (*v2.GrantsServiceListGrantsResponse, error) { + pageKey := fmt.Sprintf("%s/%d", gid, page) + mc.mu.Lock() + mc.servedPages[pageKey]++ + if mc.failPageOnce[pageKey] { + delete(mc.failPageOnce, pageKey) + mc.mu.Unlock() + return nil, fmt.Errorf("injected crash serving %s", pageKey) + } + lookup := mc.lookup + rows := mc.pageRows(gid, page) + etag := pageEtag(gid, page, rows) + prevPages := mc.prevPageCount(gid) + hasNextUpstream := len(mc.pageRows(gid, page+1)) > 0 + mc.mu.Unlock() + if lookup == nil { + lookup = sourcecache.NoopLookup{} + } + scope := pageScopeFor(gid, page) + + entry, found, err := lookup.LookupPreviousSourceCache(ctx, sourcecache.RowKindGrants, scope) + if err != nil { + return nil, err + } + if found && entry.ETag == etag { + mc.mu.Lock() + mc.replayPages++ + mc.mu.Unlock() + return v2.GrantsServiceListGrantsResponse_builder{ + Annotations: annotations.New(v2.SourceCacheReplay_builder{ + ScopeHash: scope, + Etag: etag, + }.Build()), + }.Build(), nil + } + + next := "" + if hasNextUpstream && page+1 >= prevPages { + // page+1 exists upstream but was never stored, so no sibling was + // spawned for it — this cold page's chain is its only route in. + next = fmt.Sprintf("page=%d", page+1) + } + mc.mu.Lock() + mc.coldPages++ + if page >= prevPages && prevPages > 0 { + mc.probePages++ + } + mc.mu.Unlock() + return v2.GrantsServiceListGrantsResponse_builder{ + List: mc.grantsFor(gid, rows), + Annotations: annotations.New(v2.SourceCacheScope_builder{ + ScopeHash: scope, + Etag: etag, + }.Build()), + NextPageToken: next, + }.Build(), nil +} + +func (mc *pagedSpawnMockConnector) ListGrants(ctx context.Context, in *v2.GrantsServiceListGrantsRequest, _ ...grpc.CallOption) (*v2.GrantsServiceListGrantsResponse, error) { + gid := in.GetResource().GetId().GetResource() + if in.GetResource().GetId().GetResourceType() != groupResourceType.GetId() { + return v2.GrantsServiceListGrantsResponse_builder{}.Build(), nil + } + + tok := in.GetPageToken() + if tok != "" { + var page int + if _, err := fmt.Sscanf(tok, "page=%d", &page); err != nil { + return nil, fmt.Errorf("bad page token %q", tok) + } + return mc.servePage(ctx, gid, page) + } + + // Page one: serve it, then spawn a sibling for every OTHER page of the + // STORED chain (the pages whose URLs and validators the previous sync + // recorded). Cold first syncs have no stored chain and just paginate + // serially; pages beyond the stored tail are discovered by the last + // stored page missing its lookup and chaining (see servePage). + resp, err := mc.servePage(ctx, gid, 0) + if err != nil { + return nil, err + } + mc.mu.Lock() + prevPages := mc.prevPageCount(gid) + mc.mu.Unlock() + + var tokens []string + for p := 1; p < prevPages; p++ { + tokens = append(tokens, fmt.Sprintf("page=%d", p)) + } + if len(tokens) > 0 { + annos := annotations.Annotations(resp.GetAnnotations()) + annos.Update(v2.SpawnCursors_builder{PageTokens: tokens}.Build()) + resp.SetAnnotations(annos) + // Page one's own chain ends here; siblings carry the rest. + resp.SetNextPageToken("") + } + return resp, nil +} + +func mkMemberEnt(group *v2.Resource) *v2.Entitlement { + ent := v2.Entitlement_builder{ + Id: fmt.Sprintf("%s:%s:member", group.GetId().GetResourceType(), group.GetId().GetResource()), + Resource: group, + DisplayName: "member", + Slug: "member", + GrantableTo: []*v2.ResourceType{userResourceType}, + }.Build() + return ent +} + +// runSpawnSync runs one sync and returns the grant-coverage counter for +// the group type — the number the "more grant resources than resources" +// anomaly warning compares against the type's resource total. With spawn +// fan-out it must equal the group count exactly (one per resource), not +// one per spawned page. +func runSpawnSync(ctx context.Context, t *testing.T, mc *pagedSpawnMockConnector, path, prevPath, tmpDir string, workers int) int { + t.Helper() + store, err := dotc1z.NewStore(ctx, path, + dotc1z.WithEngine(c1zstore.EnginePebble), + dotc1z.WithTmpDir(tmpDir), + ) + require.NoError(t, err) + opts := []SyncOpt{WithConnectorStore(store), WithTmpDir(tmpDir)} + if prevPath != "" { + opts = append(opts, WithPreviousSyncC1ZPath(prevPath)) + } + if workers > 0 { + opts = append(opts, WithWorkerCount(workers)) + } + sc, err := NewSyncer(ctx, mc, opts...) + require.NoError(t, err) + require.NoError(t, sc.Sync(ctx)) + impl, ok := sc.(*syncer) + require.True(t, ok) + covered := impl.counts.GrantsProgress(groupResourceType.GetId()) + require.NoError(t, sc.Close(ctx)) + return covered +} + +func TestPerResourceSpawnCursorsWarmRevalidation(t *testing.T) { + for _, workers := range []int{0, 4} { + t.Run(fmt.Sprintf("workers=%d", workers), func(t *testing.T) { + ctx, err := logging.Init(context.Background()) + require.NoError(t, err) + tmpDir := t.TempDir() + + // 6 groups × 11 members = 3 pages each (4/4/3), 66 grants. + mc := newPagedSpawnMockConnector(t, 6, 11) + + // --- Sync 1: cold — no stored chain, serial pagination. ------- + sync1 := filepath.Join(tmpDir, "sync1.c1z") + covered := runSpawnSync(ctx, t, mc, sync1, "", tmpDir, workers) + require.Equal(t, 6, covered, "cold: each group counts once toward grant coverage") + grants1 := listGrantsInFile(ctx, t, sync1) + require.Len(t, grants1, 66) + require.Zero(t, mc.replayPages) + require.Equal(t, 18, mc.coldPages, "6 groups × 3 pages fetch cold") + mc.snapshotPrev() + + // --- Sync 2: warm, unchanged — page 1 spawns pages 2..3 per + // group and EVERY page replays. -------------------------------- + sync2 := filepath.Join(tmpDir, "sync2.c1z") + covered = runSpawnSync(ctx, t, mc, sync2, sync1, tmpDir, workers) + require.Equal(t, 6, covered, + "warm fan-out: a group with 3 spawned/replayed pages still counts ONCE — spawned siblings must not inflate coverage past the resource total") + require.Equal(t, 18, mc.replayPages, "every page of every group replays") + require.Equal(t, 18, mc.coldPages, "no cold fetches on the unchanged warm sync") + requireSameGrantIDs(t, grants1, listGrantsInFile(ctx, t, sync2), "warm vs cold") + mc.snapshotPrev() + + // --- Sync 3: warm with churn: + // group-00 gains 2 members → its stored tail page (page 2) + // changes content, misses its lookup, fetches cold, and CHAINS + // into a brand-new page 3 no sibling was spawned for; + // group-01 loses 3 members → its stored page 2 now returns + // zero rows (spawned probe of a vanished tail). -------------- + mc.mu.Lock() + mc.members["group-00"] = append(mc.members["group-00"], "user-20", "user-21") + mc.members["group-01"] = mc.members["group-01"][:8] // exactly 2 pages now + mc.mu.Unlock() + + sync3 := filepath.Join(tmpDir, "sync3.c1z") + runSpawnSync(ctx, t, mc, sync3, sync2, tmpDir, workers) + grants3 := listGrantsInFile(ctx, t, sync3) + require.Positive(t, mc.probePages, "the fresh tail page must be reached by a cold page chaining past the stored horizon") + + // Cold control against the same tenant state. + control := filepath.Join(tmpDir, "control.c1z") + runSpawnSync(ctx, t, mc, control, "", tmpDir, workers) + requireSameGrantIDs(t, listGrantsInFile(ctx, t, control), grants3, "churned warm vs cold control") + }) + } +} + +// TestPerResourceSpawnCursorsResumeWithPendingSiblings pins suspend/resume +// across a spawn fan-out. A warm sync crashes while serving one of a +// group's spawned sibling pages and is then resumed on the same c1z. +// Checkpoints are rate-limited (minCheckpointInterval), so resume +// granularity is at-least-once for everything since the last persisted +// checkpoint — possibly the whole grants phase in a fast sync. The +// invariants are therefore: +// +// - no DROPS: the crashed sibling and the sibling spawned alongside it +// both execute by the time the resumed sync finishes; +// - no DUPES on double-execution: pages that committed before the crash +// re-run after resume and must be idempotent — replay re-copies, cold +// re-upserts, and the final file is grant-for-grant identical to a +// cold control sync. +// +// (The complementary serialization half — a checkpoint can never hold +// "page finished but siblings missing", and the Spawned marker survives +// the round trip — is pinned by TestSpawnedActionsSurviveCheckpoint.) +func TestPerResourceSpawnCursorsResumeWithPendingSiblings(t *testing.T) { + for _, workers := range []int{0, 4} { + t.Run(fmt.Sprintf("workers=%d", workers), func(t *testing.T) { + ctx, err := logging.Init(context.Background()) + require.NoError(t, err) + tmpDir := t.TempDir() + + // 6 groups × 11 members = 3 pages each. + mc := newPagedSpawnMockConnector(t, 6, 11) + + sync1 := filepath.Join(tmpDir, "sync1.c1z") + runSpawnSync(ctx, t, mc, sync1, "", tmpDir, workers) + mc.snapshotPrev() + + // Churn group-01 so the warm sync mixes replayed and cold + // spawned pages across the crash boundary. + mc.mu.Lock() + mc.servedPages = map[string]int{} // count only the warm attempts + mc.members["group-01"] = mc.members["group-01"][:9] + // Crash while serving group-00's spawned page 1: page 0 (the + // spawner) has already finished, so pages 1 and 2 are pending + // Spawned actions in the checkpoint when the sync dies. + mc.failPageOnce["group-00/1"] = true + mc.mu.Unlock() + + opts := []SyncOpt{WithTmpDir(tmpDir), WithPreviousSyncC1ZPath(sync1)} + if workers > 0 { + opts = append(opts, WithWorkerCount(workers)) + } + + sync2 := filepath.Join(tmpDir, "sync2.c1z") + crashed, err := NewSyncer(ctx, mc, append(opts, WithC1ZPath(sync2))...) + require.NoError(t, err) + err = crashed.Sync(ctx) + require.ErrorContains(t, err, "injected crash", "the warm sync must die mid-fan-out") + // Close on a cancelled context so the in-flight sync stays + // resumable instead of being finalized (the established + // interrupt pattern from the child-resources resume test). + cancelledCtx, cancel := context.WithCancel(ctx) + cancel() + require.NoError(t, crashed.Close(cancelledCtx)) + + resumed, err := NewSyncer(ctx, mc, append(opts, WithC1ZPath(sync2))...) + require.NoError(t, err) + require.NoError(t, resumed.Sync(ctx)) + require.NoError(t, resumed.Close(ctx)) + + mc.mu.Lock() + require.GreaterOrEqual(t, mc.servedPages["group-00/1"], 2, + "the crashed sibling must be re-executed on resume") + require.GreaterOrEqual(t, mc.servedPages["group-00/2"], 1, + "the sibling spawned alongside the crashed one must not be dropped") + mc.mu.Unlock() + + // No drops, no duplicates: cold control equivalence. Note the + // resumed file also re-executed pages that committed before the + // crash — this passing IS the double-execution idempotency + // assertion. + control := filepath.Join(tmpDir, "control.c1z") + runSpawnSync(ctx, t, mc, control, "", tmpDir, workers) + requireSameGrantIDs(t, listGrantsInFile(ctx, t, control), listGrantsInFile(ctx, t, sync2), + "resumed warm sync vs cold control") + }) + } +} + +// TestSpawnedActionsSurviveCheckpoint pins the state-level half of +// spawn-fan-out resume: the exact sequence syncGrantsForResource performs +// (push siblings, then finish the spawner — one state mutation window, so +// no checkpoint can see "finished but siblings missing") marshals into a +// sync token from which a fresh state pops every sibling with its Spawned +// marker and resource identity intact. If the marker were dropped in the +// round trip, a resumed sync would double-count grant coverage for every +// pending sibling. +func TestSpawnedActionsSurviveCheckpoint(t *testing.T) { + ctx := context.Background() + st := newState() + require.NoError(t, st.Unmarshal("")) + + // Drain the implicit init action and stand up a grants action for one + // resource, as the syncer would. + st.FinishAction(ctx, st.Current()) + st.PushAction(ctx, Action{Op: SyncGrantsOp, ResourceTypeID: "group", ResourceID: "group-00"}) + spawner := st.Current() + + // nextPageOrFinishAction order: push spawned siblings, then finish the + // spawning action. + for _, tok := range []string{"page=1", "page=2"} { + st.PushAction(ctx, Action{ + Op: SyncGrantsOp, + ResourceTypeID: "group", + ResourceID: "group-00", + PageToken: tok, + Spawned: true, + }) + } + st.FinishAction(ctx, spawner) + + token, err := st.Marshal() + require.NoError(t, err) + + resumed := newState() + require.NoError(t, resumed.Unmarshal(token)) + + var siblings []Action + for resumed.Current() != nil { + a := *resumed.Current() + siblings = append(siblings, a) + resumed.FinishAction(ctx, &a) + } + require.Len(t, siblings, 2, "both pending siblings must survive the checkpoint round trip") + tokens := map[string]bool{} + for _, a := range siblings { + require.True(t, a.Spawned, "the Spawned marker must survive serialization (progress accounting depends on it)") + require.Equal(t, "group-00", a.ResourceID, "the resource identity rides the action across resume") + require.Equal(t, SyncGrantsOp, a.Op) + tokens[a.PageToken] = true + } + require.Equal(t, map[string]bool{"page=1": true, "page=2": true}, tokens) +} + +// TestPerResourceSpawnCursorsChurnSoak is the randomized soak (the shape +// that caught real bugs in both the Entra and GitHub POCs): seeded rounds +// of membership churn against the paged-spawn connector, each round +// running a warm chained sync at workers=4 and asserting reader-surface +// equivalence against a cold control sync of the same tenant state. Churn +// shifts page boundaries arbitrarily, so every round mixes replayed pages, +// cold refetches of shifted pages, vanished tails (zero-row probes), and +// fresh tails discovered by chaining — concurrently. +func TestPerResourceSpawnCursorsChurnSoak(t *testing.T) { + if testing.Short() { + t.Skip("soak test") + } + for _, seed := range []int64{1, 7} { + t.Run(fmt.Sprintf("seed=%d", seed), func(t *testing.T) { + ctx, err := logging.Init(context.Background()) + require.NoError(t, err) + tmpDir := t.TempDir() + rng := rand.New(rand.NewSource(seed)) //nolint:gosec // deterministic test seed + + mc := newPagedSpawnMockConnector(t, 8, 9) + + prev := filepath.Join(tmpDir, "sync0.c1z") + runSpawnSync(ctx, t, mc, prev, "", tmpDir, 4) + mc.snapshotPrev() + + for round := 1; round <= 6; round++ { + // Churn: random adds/removes across random groups. Removes + // from the middle shift every following page of the group; + // adds can grow a fresh tail page. + mc.mu.Lock() + for i := 0; i < 1+rng.Intn(4); i++ { + gid := mc.groupIDs[rng.Intn(len(mc.groupIDs))] + cur := mc.members[gid] + if rng.Intn(2) == 0 && len(cur) > 1 { + k := rng.Intn(len(cur)) + mc.members[gid] = append(append([]string(nil), cur[:k]...), cur[k+1:]...) + } else { + uid := fmt.Sprintf("user-%02d", rng.Intn(30)) + if !slices.Contains(cur, uid) { + mc.members[gid] = append(cur, uid) + } + } + } + mc.mu.Unlock() + + warm := filepath.Join(tmpDir, fmt.Sprintf("warm-%d.c1z", round)) + runSpawnSync(ctx, t, mc, warm, prev, tmpDir, 4) + warmGrants := listGrantsInFile(ctx, t, warm) + + control := filepath.Join(tmpDir, fmt.Sprintf("control-%d.c1z", round)) + runSpawnSync(ctx, t, mc, control, "", tmpDir, 4) + requireSameGrantIDs(t, listGrantsInFile(ctx, t, control), warmGrants, + fmt.Sprintf("round %d: warm chain vs cold control", round)) + + mc.snapshotPrev() + prev = warm + } + require.Positive(t, mc.replayPages, "soak must actually exercise replay") + require.Positive(t, mc.coldPages, "soak must actually exercise cold refetch") + }) + } +} diff --git a/pkg/sync/state.go b/pkg/sync/state.go index f1e3cc08c..34ba47a85 100644 --- a/pkg/sync/state.go +++ b/pkg/sync/state.go @@ -46,6 +46,13 @@ type State interface { RecordSessionOp(op string, duration time.Duration, opErr error, timedOut bool) MergeSessionStat(op string, add SessionStoreStat) SessionStoreStats() map[string]SessionStoreStat + // AddSourceCacheStats merges delta into the sync's source-cache + // counters (replay effectiveness, tombstones, spawn fan-out, lookup + // continuation). Token-persisted, so a resumed sync keeps its counts. + AddSourceCacheStats(delta SourceCacheStats) + // SourceCacheStatsSnapshot returns a copy of the accumulated + // source-cache counters, or nil when nothing was recorded. + SourceCacheStatsSnapshot() *SourceCacheStats // CheckAndSetExclusionGroupResourceType records that exclusionGroupID is // used on resourceTypeID. If the group was already recorded against a // different resource type, the prior value is returned with conflict=true @@ -210,6 +217,15 @@ type Action struct { ResourceID string `json:"resource_id,omitempty"` ParentResourceTypeID string `json:"parent_resource_type_id,omitempty"` ParentResourceID string `json:"parent_resource_id,omitempty"` + + // Spawned marks an action enqueued by a connector's SpawnCursors + // annotation (a sibling cursor) rather than by the syncer's own + // planners. Progress accounting uses it: per-resource grant coverage + // counts a resource once, when its ORIGIN action's chain ends — + // spawned siblings (and their NextPage continuations, which inherit + // the action) don't count, or one resource would count N times. + // omitempty keeps old checkpointed sync tokens decoding unchanged. + Spawned bool `json:"spawned,omitempty"` } var _ State = &state{} @@ -233,6 +249,7 @@ type state struct { stepDurationsMs map[string]int64 connectorCallStats map[string]*ConnectorCallStat sessionStoreStats map[string]*SessionStoreStat + sourceCacheStats *SourceCacheStats // compaction is provenance written by the sync compactor via // BuildCompactedToken; the syncer itself never sets it. Kept on the // state so Unmarshal→Marshal round trips (e.g. expansion replay @@ -291,6 +308,7 @@ type serializedTokenV1 struct { StepDurationsMs map[string]int64 `json:"step_durations_ms,omitempty"` ConnectorCallStats map[string]*ConnectorCallStat `json:"connector_call_stats,omitempty"` SessionStoreStats map[string]*SessionStoreStat `json:"session_store_stats,omitempty"` + SourceCacheStats *SourceCacheStats `json:"source_cache_stats,omitempty"` Compaction *CompactionTokenStats `json:"compaction,omitempty"` Version uint64 `json:"version"` } @@ -456,6 +474,7 @@ func (st *state) Unmarshal(input string) error { if st.sessionStoreStats == nil { st.sessionStoreStats = make(map[string]*SessionStoreStat) } + st.sourceCacheStats = token.SourceCacheStats st.compaction = token.Compaction } else { st.actions = make(map[string]Action) @@ -476,6 +495,7 @@ func (st *state) Unmarshal(input string) error { st.stepDurationsMs = make(map[string]int64) st.connectorCallStats = make(map[string]*ConnectorCallStat) st.sessionStoreStats = make(map[string]*SessionStoreStat) + st.sourceCacheStats = nil st.compaction = nil } @@ -504,6 +524,7 @@ func (st *state) Marshal() (string, error) { StepDurationsMs: st.stepDurationsMs, ConnectorCallStats: st.connectorCallStats, SessionStoreStats: st.sessionStoreStats, + SourceCacheStats: st.sourceCacheStats, Compaction: st.compaction, Version: 1, }) diff --git a/pkg/sync/syncer.go b/pkg/sync/syncer.go index 39e7adc88..548d05d31 100644 --- a/pkg/sync/syncer.go +++ b/pkg/sync/syncer.go @@ -20,6 +20,7 @@ import ( "github.com/conductorone/baton-sdk/pkg/bid" "github.com/conductorone/baton-sdk/pkg/dotc1z" "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" + "github.com/conductorone/baton-sdk/pkg/sourcecache" "github.com/conductorone/baton-sdk/pkg/sync/expand" "github.com/conductorone/baton-sdk/pkg/types/entitlement" batonGrant "github.com/conductorone/baton-sdk/pkg/types/grant" @@ -158,6 +159,7 @@ type syncer struct { syncID string skipEGForResourceType syncMap[string, bool] skipEntitlementsForResourceType syncMap[string, bool] + typeScopedGrantsForResourceType syncMap[string, bool] skipEntitlementsAndGrants bool skipGrants bool resourceTypeTraits syncMap[string, []v2.ResourceType_Trait] @@ -169,6 +171,7 @@ type syncer struct { metricsHandler metrics.Handler syncIdentity uotel.SyncIdentity recordStats bool + sourceCache syncerSourceCache } var _ Syncer = (*syncer)(nil) @@ -496,6 +499,9 @@ func (s *syncer) syncSummaryFields(span trace.Span) []zap.Field { if logSession { fields = append(fields, zap.Any("session_store_stats", sessionForLog)) } + if scStats := s.state.SourceCacheStatsSnapshot(); scStats != nil { + fields = append(fields, zap.Any("source_cache_stats", scStats)) + } return fields } @@ -770,6 +776,11 @@ func (s *syncer) Sync(ctx context.Context) error { } } + err = s.configureSourceCache(ctx, resp) + if err != nil { + return err + } + syncResourceTypeMap := make(map[string]bool) if len(s.syncResourceTypes) > 0 { for _, rt := range s.syncResourceTypes { @@ -895,6 +906,7 @@ func (s *syncer) Sync(ctx context.Context) error { return s.returnSyncError(l, span, err) } + s.logSourceCacheStats(ctx) if s.recordStats { l.Info("Sync complete.", s.syncSummaryFields(span)...) } else { @@ -1087,11 +1099,7 @@ func (s *syncer) hasChildResources(resource *v2.Resource) bool { // At sync scale (100k+ resources per trace) the span overhead and trace bloat // outweighed any debugging value. func (s *syncer) getSubResources(ctx context.Context, parent *v2.Resource) error { - syncResourceTypeMap := make(map[string]bool) - for _, rt := range s.syncResourceTypes { - syncResourceTypeMap[rt] = true - } - + var childTypeIDs []string for _, a := range parent.GetAnnotations() { if a.MessageIs((*v2.ChildResourceType)(nil)) { crt := &v2.ChildResourceType{} @@ -1099,24 +1107,32 @@ func (s *syncer) getSubResources(ctx context.Context, parent *v2.Resource) error if err != nil { return err } - if len(s.syncResourceTypes) > 0 { - if shouldSync := syncResourceTypeMap[crt.GetResourceTypeId()]; !shouldSync { - continue - } - } - childAction := Action{ - Op: SyncResourcesOp, - ResourceTypeID: crt.GetResourceTypeId(), - ParentResourceID: parent.GetId().GetResource(), - ParentResourceTypeID: parent.GetId().GetResourceType(), - } - s.state.PushAction(ctx, childAction) + childTypeIDs = append(childTypeIDs, crt.GetResourceTypeId()) } } - + s.pushChildResourceActions(ctx, childTypeIDs, parent.GetId().GetResourceType(), parent.GetId().GetResource()) return nil } +// pushChildResourceActions queues child resource syncs for one parent, +// honoring the sync's resource-type filter. Shared by getSubResources +// (response rows) and the source-cache replay path, which must schedule +// children for replayed parents itself (a replayed page has no response +// rows for getSubResources to inspect). +func (s *syncer) pushChildResourceActions(ctx context.Context, childTypeIDs []string, parentTypeID, parentID string) { + for _, childTypeID := range childTypeIDs { + if len(s.syncResourceTypes) > 0 && !slices.Contains(s.syncResourceTypes, childTypeID) { + continue + } + s.state.PushAction(ctx, Action{ + Op: SyncResourcesOp, + ResourceTypeID: childTypeID, + ParentResourceID: parentID, + ParentResourceTypeID: parentTypeID, + }) + } +} + func (s *syncer) getResourceFromConnector(ctx context.Context, resourceID *v2.ResourceId, parentResourceID *v2.ResourceId) (*v2.Resource, error) { ctx, span := tracer.Start(ctx, "syncer.getResource") var err error @@ -1269,26 +1285,57 @@ func (s *syncer) SyncResources(ctx context.Context, action *Action) error { // owns a span — the duplicate inflated trace span counts without adding // information. func (s *syncer) syncResources(ctx context.Context, action *Action) error { - req := v2.ResourcesServiceListResourcesRequest_builder{ - ResourceTypeId: action.ResourceTypeID, - PageToken: action.PageToken, - ActiveSyncId: s.getActiveSyncID(), - }.Build() - if action.ParentResourceTypeID != "" && action.ParentResourceID != "" { - req.SetParentResourceId(v2.ResourceId_builder{ - ResourceType: action.ParentResourceTypeID, - Resource: action.ParentResourceID, - }.Build()) + var resp *v2.ResourcesServiceListResourcesResponse + err := s.withSourceCacheContinuation(ctx, "sync-resources", func(extra annotations.Annotations, attempt int) (listAttempt, error) { + req := v2.ResourcesServiceListResourcesRequest_builder{ + ResourceTypeId: action.ResourceTypeID, + PageToken: action.PageToken, + ActiveSyncId: s.getActiveSyncID(), + Annotations: extra, + }.Build() + if action.ParentResourceTypeID != "" && action.ParentResourceID != "" { + req.SetParentResourceId(v2.ResourceId_builder{ + ResourceType: action.ParentResourceTypeID, + Resource: action.ParentResourceID, + }.Build()) + } + start := time.Now() + r, err := s.connector.ListResources(ctx, req) + s.observeConnectorCall(ctx, continuationCallMethod("list-resources", attempt), start, action.ResourceTypeID, action.ResourceID) + if err != nil { + return listAttempt{}, err + } + s.recordSessionUsage(r.GetAnnotations()) + resp = r + return listAttempt{ + annos: annotations.Annotations(r.GetAnnotations()), + rows: len(r.GetList()), + nextToken: r.GetNextPageToken(), + }, nil + }) + if err != nil { + return err } - start := time.Now() - resp, err := s.connector.ListResources(ctx, req) - s.observeConnectorCall(ctx, "list-resources", start, action.ResourceTypeID, action.ResourceID) - s.recordSessionUsage(resp.GetAnnotations()) + putCtx, scPage, err := s.beginSourceCachePage(ctx, sourcecache.RowKindResources, annotations.Annotations(resp.GetAnnotations()), len(resp.GetList())) if err != nil { return err } + // On any source-cache-scoped page the "already synced this sync" + // dedupe below is wrong — scoped pages are upsert streams whose rows + // are always authoritative: + // - Replayed rounds copy the previous sync's rows in on the round's + // FIRST page, so an overlay row's identity ALWAYS hits the store, + // on every page of the round — but the stored row is the stale + // base and the overlay row is the update that must overwrite it. + // Keying off the replay annotation alone would only protect page + // one (the annotation fires once per round). + // - Cold delta enumerations may legally return the same object + // multiple times (changed mid-walk), later occurrences + // authoritative; deduping would keep the STALE first occurrence. + pageScoped := scPage != nil + bulkPutResoruces := []*v2.Resource{} for _, r := range resp.GetList() { validatedResource := false @@ -1305,7 +1352,7 @@ func (s *syncer) syncResources(ctx context.Context, action *Action) error { validatedResource = true // We must *ALSO* check if we have any child resources. - if !s.hasChildResources(r) { + if !pageScoped && !s.hasChildResources(r) { // Since we only have the resource type IDs of child resources, // we can't tell if we already have synced those child resources. // Those children may also have their own child resources, @@ -1334,12 +1381,16 @@ func (s *syncer) syncResources(ctx context.Context, action *Action) error { } if len(bulkPutResoruces) > 0 { - err = s.store.PutResources(ctx, bulkPutResoruces...) + err = s.store.PutResources(putCtx, bulkPutResoruces...) if err != nil { return err } } + if err := s.finishSourceCachePage(ctx, scPage); err != nil { + return err + } + s.handleProgress(ctx, action, len(resp.GetList())) s.counts.AddResources(action.ResourceTypeID, len(resp.GetList())) if resp.GetNextPageToken() == "" { @@ -1547,25 +1598,48 @@ func (s *syncer) syncEntitlementsForResource(ctx context.Context, action *Action resource := resourceResponse.GetResource() - start := time.Now() - resp, err := s.connector.ListEntitlements(ctx, v2.EntitlementsServiceListEntitlementsRequest_builder{ - Resource: resource, - PageToken: action.PageToken, - ActiveSyncId: s.getActiveSyncID(), - }.Build()) - s.observeConnectorCall(ctx, "list-entitlements", start, action.ResourceTypeID, action.ResourceID) - s.recordSessionUsage(resp.GetAnnotations()) + var resp *v2.EntitlementsServiceListEntitlementsResponse + err = s.withSourceCacheContinuation(ctx, "sync-entitlements", func(extra annotations.Annotations, attempt int) (listAttempt, error) { + start := time.Now() + r, err := s.connector.ListEntitlements(ctx, v2.EntitlementsServiceListEntitlementsRequest_builder{ + Resource: resource, + PageToken: action.PageToken, + ActiveSyncId: s.getActiveSyncID(), + Annotations: extra, + }.Build()) + s.observeConnectorCall(ctx, continuationCallMethod("list-entitlements", attempt), start, action.ResourceTypeID, action.ResourceID) + if err != nil { + return listAttempt{}, err + } + s.recordSessionUsage(r.GetAnnotations()) + resp = r + return listAttempt{ + annos: annotations.Annotations(r.GetAnnotations()), + rows: len(r.GetList()), + nextToken: r.GetNextPageToken(), + }, nil + }) if err != nil { return err } if err := s.validateEntitlementExclusionGroups(resp.GetList()); err != nil { return err } - err = s.store.PutEntitlements(ctx, resp.GetList()...) + + putCtx, scPage, err := s.beginSourceCachePage(ctx, sourcecache.RowKindEntitlements, annotations.Annotations(resp.GetAnnotations()), len(resp.GetList())) + if err != nil { + return err + } + + err = s.store.PutEntitlements(putCtx, resp.GetList()...) if err != nil { return err } + if err := s.finishSourceCachePage(ctx, scPage); err != nil { + return err + } + s.handleProgress(ctx, action, len(resp.GetList())) if resp.GetNextPageToken() == "" { s.counts.AddEntitlementsProgress(resourceID.ResourceType, 1) @@ -1999,6 +2073,9 @@ func (s *syncer) fixEntitlementGraphCycles(ctx context.Context, graph *expand.En // SyncGrants fetches the grants for each resource from the connector. It iterates each resource // from the datastore, and pushes a new action to sync the grants for each individual resource. +// Resource types annotated with TypeScopedGrants are excluded from the per-resource fan-out and +// get a single type-scoped action instead (empty ResourceID); the connector enumerates the whole +// type, optionally spawning additional cursors via the SpawnCursors annotation. func (s *syncer) SyncGrants(ctx context.Context, action *Action) error { ctx, span := uotel.StartWithLink(ctx, tracer, "syncer.SyncGrants") uotel.SetSyncIdentityAttrs(ctx, span) @@ -2006,9 +2083,20 @@ func (s *syncer) SyncGrants(ctx context.Context, action *Action) error { defer func() { uotel.EndSpanWithError(span, err) }() if action.ResourceTypeID == "" && action.ResourceID == "" { + actions := make([]Action, 0) if action.PageToken == "" { ctxzap.Extract(ctx).Info("Syncing grants...") s.handleInitialActionForStep(ctx, *action) + + // One type-scoped action per annotated resource type, enqueued + // exactly once (the planner's first page). + typeScoped, err := s.typeScopedGrantsResourceTypes(ctx) + if err != nil { + return fmt.Errorf("sync-grants: error listing type-scoped resource types: %w", err) + } + for _, rtID := range typeScoped { + actions = append(actions, Action{Op: SyncGrantsOp, ResourceTypeID: rtID}) + } } resp, err := s.store.ListResources(ctx, v2.ResourcesServiceListResourcesRequest_builder{ @@ -2019,16 +2107,25 @@ func (s *syncer) SyncGrants(ctx context.Context, action *Action) error { return fmt.Errorf("sync-grants: error listing resources: %w", err) } - actions := make([]Action, 0) for _, r := range resp.GetList() { shouldSkip, err := s.shouldSkipGrants(ctx, r) if err != nil { return err } - if shouldSkip { continue } + + // Types with type-scoped grants are excluded from the + // per-resource fan-out; their single action is enqueued above. + typeScoped, err := s.resourceTypeHasTypeScopedGrants(ctx, r.GetId().GetResourceType()) + if err != nil { + return err + } + if typeScoped { + continue + } + actions = append(actions, Action{Op: SyncGrantsOp, ResourceID: r.GetId().GetResource(), ResourceTypeID: r.GetId().GetResourceType()}) } @@ -2042,30 +2139,110 @@ func (s *syncer) SyncGrants(ctx context.Context, action *Action) error { return nil } +// resourceTypeHasTypeScopedGrants reports (cached per sync) whether the +// resource type carries the TypeScopedGrants annotation. +func (s *syncer) resourceTypeHasTypeScopedGrants(ctx context.Context, resourceTypeID string) (bool, error) { + if v, ok := s.typeScopedGrantsForResourceType.Load(resourceTypeID); ok { + return v, nil + } + rt, err := s.store.GetResourceType(ctx, reader_v2.ResourceTypesReaderServiceGetResourceTypeRequest_builder{ + ResourceTypeId: resourceTypeID, + }.Build()) + if err != nil { + return false, err + } + rtAnnos := annotations.Annotations(rt.GetResourceType().GetAnnotations()) + typeScoped := rtAnnos.Contains(&v2.TypeScopedGrants{}) + s.typeScopedGrantsForResourceType.Store(resourceTypeID, typeScoped) + return typeScoped, nil +} + +// typeScopedGrantsResourceTypes lists every synced resource type annotated +// with TypeScopedGrants. +func (s *syncer) typeScopedGrantsResourceTypes(ctx context.Context) ([]string, error) { + var out []string + pageToken := "" + for { + resp, err := s.store.ListResourceTypes(ctx, v2.ResourceTypesServiceListResourceTypesRequest_builder{ + PageToken: pageToken, + }.Build()) + if err != nil { + return nil, err + } + for _, rt := range resp.GetList() { + rtAnnos := annotations.Annotations(rt.GetAnnotations()) + typeScoped := rtAnnos.Contains(&v2.TypeScopedGrants{}) + s.typeScopedGrantsForResourceType.Store(rt.GetId(), typeScoped) + if typeScoped { + out = append(out, rt.GetId()) + } + } + pageToken = resp.GetNextPageToken() + if pageToken == "" { + return out, nil + } + } +} + // syncGrantsForResource fetches the grants for a specific resource from the connector. +// An action with an empty ResourceID is a TYPE-SCOPED grants cursor: the connector +// enumerates grants for the whole resource type (no single resource backs the call), +// and may spawn sibling cursors via the SpawnCursors response annotation. // No span here: only call site is SyncGrants, which already owns a span. func (s *syncer) syncGrantsForResource(ctx context.Context, action *Action) error { + typeScoped := action.ResourceID == "" resourceID := v2.ResourceId_builder{ ResourceType: action.ResourceTypeID, Resource: action.ResourceID, }.Build() - resourceResponse, err := s.store.GetResource(ctx, reader_v2.ResourcesReaderServiceGetResourceRequest_builder{ - ResourceId: resourceID, - }.Build()) - if err != nil { - return fmt.Errorf("sync-grants-for-resource: error getting resource: %w", err) - } - resource := resourceResponse.GetResource() + var resource *v2.Resource + var reqAnnos annotations.Annotations + if typeScoped { + // Wire validation requires a non-empty resource id, so the stub is + // self-referential ({type, type}) and the request carries the + // TypeScopedGrants annotation as the routing marker. + resource = v2.Resource_builder{ + Id: v2.ResourceId_builder{ + ResourceType: action.ResourceTypeID, + Resource: action.ResourceTypeID, + }.Build(), + }.Build() + reqAnnos.Update(&v2.TypeScopedGrants{}) + } else { + resourceResponse, err := s.store.GetResource(ctx, reader_v2.ResourcesReaderServiceGetResourceRequest_builder{ + ResourceId: resourceID, + }.Build()) + if err != nil { + return fmt.Errorf("sync-grants-for-resource: error getting resource: %w", err) + } + resource = resourceResponse.GetResource() + } - start := time.Now() - resp, err := s.connector.ListGrants(ctx, v2.GrantsServiceListGrantsRequest_builder{ - Resource: resource, - PageToken: action.PageToken, - ActiveSyncId: s.getActiveSyncID(), - }.Build()) - s.observeConnectorCall(ctx, "list-grants", start, action.ResourceTypeID, action.ResourceID) - s.recordSessionUsage(resp.GetAnnotations()) + var resp *v2.GrantsServiceListGrantsResponse + err := s.withSourceCacheContinuation(ctx, "sync-grants-for-resource", func(extra annotations.Annotations, attempt int) (listAttempt, error) { + annos := make(annotations.Annotations, 0, len(reqAnnos)+len(extra)) + annos = append(annos, reqAnnos...) + annos = append(annos, extra...) + start := time.Now() + r, err := s.connector.ListGrants(ctx, v2.GrantsServiceListGrantsRequest_builder{ + Resource: resource, + PageToken: action.PageToken, + ActiveSyncId: s.getActiveSyncID(), + Annotations: annos, + }.Build()) + s.observeConnectorCall(ctx, continuationCallMethod("list-grants", attempt), start, action.ResourceTypeID, action.ResourceID) + if err != nil { + return listAttempt{}, err + } + s.recordSessionUsage(r.GetAnnotations()) + resp = r + return listAttempt{ + annos: annotations.Annotations(r.GetAnnotations()), + rows: len(r.GetList()), + nextToken: r.GetNextPageToken(), + }, nil + }) if err != nil { return fmt.Errorf("sync-grants-for-resource: error listing grants: %w", err) } @@ -2077,6 +2254,15 @@ func (s *syncer) syncGrantsForResource(ctx context.Context, action *Action) erro respAnnos := annotations.Annotations(resp.GetAnnotations()) insertResourceGrants := respAnnos.Contains(&v2.InsertResourceGrants{}) + // Source-cache replay/stamping for this grants page. putCtx applies + // ONLY to the PutGrants call below — the related-resource PutResources + // writes in this function are resource rows and must not inherit a + // grants-scope stamp. + putCtx, scPage, err := s.beginSourceCachePage(ctx, sourcecache.RowKindGrants, respAnnos, len(grants)) + if err != nil { + return fmt.Errorf("sync-grants-for-resource: %w", err) + } + // Stamp InsertResourceGrants per-grant so the slim-blob writer's // gate sees it. The annotation is response-level, but the writer // needs it per-row to avoid stripping the Resource this path @@ -2159,19 +2345,99 @@ func (s *syncer) syncGrantsForResource(ctx context.Context, action *Action) erro } } - err = s.store.PutGrants(ctx, grants...) + err = s.store.PutGrants(putCtx, grants...) if err != nil { return fmt.Errorf("sync-grants-for-resource: error putting grants: %w", err) } + if err := s.finishSourceCachePage(ctx, scPage); err != nil { + return fmt.Errorf("sync-grants-for-resource: %w", err) + } + s.handleProgress(ctx, action, len(grants)) - if resp.GetNextPageToken() == "" { + if typeScoped { + // Cursors don't map 1:1 to resources (one cursor can cover many + // groups and may also emit cross-type grants), so the per-resource + // "N of M resources covered" accounting is meaningless here and + // would trip the "more grant resources than resources" warning on + // healthy syncs. Count raw grant rows instead. + s.counts.SetGrantsCountOnly(resourceID.GetResourceType()) + s.counts.AddGrantsProgress(resourceID.GetResourceType(), len(grants)) + s.counts.LogGrantsProgress(ctx, resourceID.GetResourceType()) + } else if resp.GetNextPageToken() == "" && !action.Spawned { + // A resource counts as covered exactly once: when its ORIGIN + // action's page chain ends. Spawned sibling cursors for the same + // resource also end with an empty token but must not count, or a + // resource with N spawned pages would count N times and trip the + // progress anomaly warning on healthy syncs. s.counts.AddGrantsProgress(resourceID.GetResourceType(), 1) s.counts.LogGrantsProgress(ctx, resourceID.GetResourceType()) } - return s.nextPageOrFinishAction(ctx, action, resp.GetNextPageToken()) + // SpawnCursors: the response may enqueue sibling cursors — each runs + // as its own action, scheduled by the worker pool, rate-limited, and + // checkpointed like any other pagination. + // + // - Type-scoped calls: one cursor per connector-defined shard (e.g. + // 50-id delta chunks); actions carry only the resource type. + // - Per-resource calls: parallel page fan-out for page-numbered APIs + // (the connector knows every page URL from page one, so a warm + // sync can revalidate all pages concurrently instead of serially); + // actions carry the resource identity. + // + // Spawned cursors are ORDINARY pages that happen to be enqueued + // eagerly: the SDK assumes nothing about replay — a spawned page may + // hit its lookup and replay, miss and fetch cold (page boundary + // shifted since last sync), and may chain further via NextPageToken. + spawn := &v2.SpawnCursors{} + hasSpawn, err := respAnnos.Pick(spawn) + if err != nil { + return fmt.Errorf("sync-grants-for-resource: error parsing spawn-cursors annotation: %w", err) + } + var spawned []Action + if hasSpawn { + // Every spawned action is persisted in the checkpointed state + // token, so an unbounded fan-out (a buggy planner emitting + // millions of tokens, or a spawn-inside-spawn loop) would bloat + // every checkpoint write before anything else fails. Cap + // per-response and fail loudly, mirroring the lookup bounce cap. + // Legitimate patterns sit far below this: Entra spawns ~35 chunk + // cursors per planning page, GitHub one sibling per stored page. + if len(spawn.GetPageTokens()) > maxSpawnCursorsPerResponse { + return fmt.Errorf( + "sync-grants-for-resource: SpawnCursors carried %d page tokens (max %d per response); chain additional spawns across pages instead", + len(spawn.GetPageTokens()), maxSpawnCursorsPerResponse) + } + for _, tok := range spawn.GetPageTokens() { + if tok == "" { + continue + } + if len(tok) > maxSpawnCursorTokenBytes { + return fmt.Errorf( + "sync-grants-for-resource: SpawnCursors page token is %d bytes (max %d)", + len(tok), maxSpawnCursorTokenBytes) + } + spawned = append(spawned, Action{ + Op: SyncGrantsOp, + ResourceTypeID: action.ResourceTypeID, + ResourceID: action.ResourceID, // empty on type-scoped actions + PageToken: tok, + Spawned: true, + }) + } + if len(spawned) > 0 { + s.state.AddSourceCacheStats(SourceCacheStats{SpawnedCursors: int64(len(spawned))}) + } + l.Debug("sync-grants-for-resource: spawned sibling grant cursors", + zap.String("resource_type_id", action.ResourceTypeID), + zap.String("resource_id", action.ResourceID), + zap.Bool("type_scoped", typeScoped), + zap.Int("cursors", len(spawned)), + zap.Int64("estimated_total", spawn.GetEstimatedTotal())) + } + + return s.nextPageOrFinishAction(ctx, action, resp.GetNextPageToken(), spawned...) } func (s *syncer) SyncExternalResources(ctx context.Context, action *Action) error { @@ -2580,7 +2846,37 @@ func (s *syncer) processGrantsWithExternalPrincipals(ctx context.Context, princi } grantsToDelete := make([]*v2.Grant, 0) - expandedGrants := make([]*v2.Grant, 0) + // expanded dedupes transformed grants by id with DETERMINISTIC + // collision resolution. Overlapping match rules (e.g. MatchID and + // MatchAll on one entitlement resolving to the same principal) + // produce the same grant id with different annotations; naive + // last-writer-wins depends on store iteration order, which differs + // between a cold sync (iterating source grants) and a warm sync + // (iterating replayed transformed grants) — breaking replay + // equivalence. The more specific rule wins (MatchID > key/value + // Match > MatchAll); equal-rank conflicts keep the smaller + // deterministic encoding. Each entry also carries its SOURCE grant's + // source-cache scope: the source grant is deleted below, and a scope + // left holding neither source nor transformed rows would silently + // replay empty on the next sync. + const ( + matchRankAll = iota + 1 + matchRankKeyValue + matchRankID + ) + type expandedEntry struct { + grant *v2.Grant + scope string + rank int + } + expanded := make(map[string]*expandedEntry) + // matchAllSeen guards MatchAll reprocessing. On a warm sync the + // previous sync's transformed grants replay and re-enter this loop, + // each still carrying the original MatchAll annotation; without + // dedupe, N transformed grants would each regenerate all N + // principals' grants (O(N²)). One regeneration per (entitlement, + // trait) produces the complete set. + matchAllSeen := make(map[string]struct{}) for ga, err := range s.store.Grants().ListWithAnnotations(ctx) { if err != nil { @@ -2592,6 +2888,23 @@ func (s *syncer) processGrantsWithExternalPrincipals(ctx context.Context, princi if !annos.ContainsAny(&v2.ExternalResourceMatchAll{}, &v2.ExternalResourceMatch{}, &v2.ExternalResourceMatchID{}) { continue } + sourceScope := ga.SourceScopeHash + addExpanded := func(rank int, g *v2.Grant) { + entry := &expandedEntry{grant: g, scope: sourceScope, rank: rank} + prev, ok := expanded[g.GetId()] + if !ok || entry.rank > prev.rank { + expanded[g.GetId()] = entry + return + } + if entry.rank < prev.rank || prev.grant == g { + // Lower specificity loses; re-adding the same object (the + // MatchID+expandable path mutates and re-adds) is a no-op. + return + } + if grantCollisionKey(entry.scope, g) < grantCollisionKey(prev.scope, prev.grant) { + expanded[g.GetId()] = entry + } + } // Match all matchResourceMatchAllAnno, err := GetExternalResourceMatchAllAnnotation(annos) @@ -2599,18 +2912,22 @@ func (s *syncer) processGrantsWithExternalPrincipals(ctx context.Context, princi return err } if matchResourceMatchAllAnno != nil { - var processPrincipals []*v2.Resource - switch matchResourceMatchAllAnno.GetResourceType() { - case v2.ResourceType_TRAIT_USER: - processPrincipals = userPrincipals - case v2.ResourceType_TRAIT_GROUP: - processPrincipals = groupPrincipals - default: - l.Error("unexpected external resource type trait", zap.Any("trait", matchResourceMatchAllAnno.GetResourceType())) - } - for _, principal := range processPrincipals { - newGrant := newGrantForExternalPrincipal(grant, principal) - expandedGrants = append(expandedGrants, newGrant) + matchAllKey := grant.GetEntitlement().GetId() + "\x00" + matchResourceMatchAllAnno.GetResourceType().String() + if _, done := matchAllSeen[matchAllKey]; !done { + matchAllSeen[matchAllKey] = struct{}{} + var processPrincipals []*v2.Resource + switch matchResourceMatchAllAnno.GetResourceType() { + case v2.ResourceType_TRAIT_USER: + processPrincipals = userPrincipals + case v2.ResourceType_TRAIT_GROUP: + processPrincipals = groupPrincipals + default: + l.Error("unexpected external resource type trait", zap.Any("trait", matchResourceMatchAllAnno.GetResourceType())) + } + for _, principal := range processPrincipals { + newGrant := newGrantForExternalPrincipal(grant, principal) + addExpanded(matchRankAll, newGrant) + } } grantsToDelete = append(grantsToDelete, grant) continue @@ -2649,7 +2966,7 @@ func (s *syncer) processGrantsWithExternalPrincipals(ctx context.Context, princi if matchResourceMatchIDAnno != nil { if principal, ok := principalMap[matchResourceMatchIDAnno.GetId()]; ok { newGrant := newGrantForExternalPrincipal(grant, principal) - expandedGrants = append(expandedGrants, newGrant) + addExpanded(matchRankID, newGrant) newGrantAnnos := annotations.Annotations(newGrant.GetAnnotations()) @@ -2682,7 +2999,7 @@ func (s *syncer) processGrantsWithExternalPrincipals(ctx context.Context, princi }.Build() newGrantAnnos.Update(newExpandableAnno) newGrant.SetAnnotations(newGrantAnnos) - expandedGrants = append(expandedGrants, newGrant) + addExpanded(matchRankID, newGrant) } } @@ -2709,7 +3026,7 @@ func (s *syncer) processGrantsWithExternalPrincipals(ctx context.Context, princi if matchExternalResource.GetKey() == "email" { if userTraitContainsEmail(userTrait.GetEmails(), matchExternalResource.GetValue()) { newGrant := newGrantForExternalPrincipal(grant, userPrincipal) - expandedGrants = append(expandedGrants, newGrant) + addExpanded(matchRankKeyValue, newGrant) // continue to next principal since we found an email match continue } @@ -2717,7 +3034,7 @@ func (s *syncer) processGrantsWithExternalPrincipals(ctx context.Context, princi profileVal, ok := resource.GetProfileStringValue(userTrait.GetProfile(), matchExternalResource.GetKey()) if ok && strings.EqualFold(profileVal, matchExternalResource.GetValue()) { newGrant := newGrantForExternalPrincipal(grant, userPrincipal) - expandedGrants = append(expandedGrants, newGrant) + addExpanded(matchRankKeyValue, newGrant) } } case v2.ResourceType_TRAIT_GROUP: @@ -2761,7 +3078,7 @@ func (s *syncer) processGrantsWithExternalPrincipals(ctx context.Context, princi }.Build() newGrantAnnos.Update(newExpandableAnno) newGrant.SetAnnotations(newGrantAnnos) - expandedGrants = append(expandedGrants, newGrant) + addExpanded(matchRankKeyValue, newGrant) } } } @@ -2775,13 +3092,24 @@ func (s *syncer) processGrantsWithExternalPrincipals(ctx context.Context, princi } newGrantIDs := mapset.NewSet[string]() - for _, ng := range expandedGrants { - newGrantIDs.Add(ng.GetId()) + expandedByScope := make(map[string][]*v2.Grant) + for id, e := range expanded { + newGrantIDs.Add(id) + expandedByScope[e.scope] = append(expandedByScope[e.scope], e.grant) } - err = s.store.PutGrants(ctx, expandedGrants...) - if err != nil { - return err + // Stamped groups write under their source grant's scope so the next + // sync's replay of that scope carries the transformed grants forward; + // the "" group is the plain unstamped write. + for scope, grants := range expandedByScope { + putCtx := ctx + if scope != "" { + putCtx = sourcecache.WithScope(ctx, scope) + } + err = s.store.PutGrants(putCtx, grants...) + if err != nil { + return err + } } // Prefer the refs-based delete (exact structural identity) when the @@ -2818,6 +3146,17 @@ type grantByRefsDeleter interface { DeleteGrantByRefs(ctx context.Context, grant *v2.Grant) error } +// grantCollisionKey gives equal-rank transformed-grant collisions a +// deterministic total order that does not depend on store iteration +// order (which differs between cold syncs and warm replayed syncs). +func grantCollisionKey(scope string, g *v2.Grant) string { + b, err := proto.MarshalOptions{Deterministic: true}.Marshal(g) + if err != nil { + return scope + } + return scope + "\x00" + string(b) +} + func newGrantForExternalPrincipal(grant *v2.Grant, principal *v2.Resource) *v2.Grant { newGrant := v2.Grant_builder{ Entitlement: grant.GetEntitlement(), @@ -2984,6 +3323,10 @@ func (s *syncer) Close(ctx context.Context) error { var errs []error + // Detach the source-cache lookup before the stores go away so a late + // connector RPC can't read a store the syncer no longer owns. + s.clearSourceCacheLookup(ctx) + var storeCloseErr error if s.store != nil { storeCloseErr = s.store.Close(finalizeCtx) diff --git a/pkg/sync/type_scoped_grants_test.go b/pkg/sync/type_scoped_grants_test.go new file mode 100644 index 000000000..041d44b82 --- /dev/null +++ b/pkg/sync/type_scoped_grants_test.go @@ -0,0 +1,534 @@ +package sync //nolint:revive,nolintlint // we can't change the package name for backwards compatibility + +// End-to-end harness for TYPE-SCOPED grants (v2.TypeScopedGrants + +// SpawnCursors), modeled on Microsoft Graph's chunked groups/delta: +// membership grants for a whole resource type are served through +// connector-defined 50-id chunk cursors instead of one ListGrants call per +// group. Each chunk is its own source-cache scope, so warm syncs replay +// unchanged chunks and overlay/tombstone the changed ones. +// +// The syncer-side behaviors under test: +// - the grants planner excludes the annotated type from the per-resource +// fan-out and enqueues exactly one type-scoped action, +// - the planning call's SpawnCursors annotation enqueues one action per +// chunk, each checkpointed/paginated independently, +// - chunk scopes replay warm with overlay adds and canonical-id +// tombstones, and equivalence against a cold control sync holds. + +import ( + "context" + "fmt" + "path/filepath" + "sort" + "strings" + "sync" + "testing" + + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + "github.com/conductorone/baton-sdk/pkg/annotations" + "github.com/conductorone/baton-sdk/pkg/dotc1z" + "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" + "github.com/conductorone/baton-sdk/pkg/logging" + "github.com/conductorone/baton-sdk/pkg/sourcecache" + et "github.com/conductorone/baton-sdk/pkg/types/entitlement" + gt "github.com/conductorone/baton-sdk/pkg/types/grant" + rs "github.com/conductorone/baton-sdk/pkg/types/resource" +) + +const chunkSize = 50 + +// chunkedGrantsMockConnector serves group-membership grants exclusively +// through type-scoped cursors, mimicking chunked Graph delta: +// +// - planning call (empty page token): no rows; SpawnCursors with one +// token per 50-group chunk. +// - chunk cursor, lookup miss (cold): enumerate the chunk's grants, +// paginated, final page carrying the chunk's validator. +// - chunk cursor, lookup hit (warm): replay + overlay adds + canonical +// grant-id tombstones + rotated validator. +type chunkedGrantsMockConnector struct { + *mockConnector + + mu sync.Mutex + lookup sourcecache.Lookup + + groupIDs []string // sorted; chunk i = groupIDs[i*50:(i+1)*50] + members map[string][]string // groupID -> userIDs + entByID map[string]*v2.Entitlement + resByID map[string]*v2.Resource + + tokenByChunk map[int]string // last validator issued per chunk + generation int + + // per-sync delta state, applied on warm rounds + addsByChunk map[int][]*v2.Grant + deletedIDsByChunk map[int][]string + + // unstampedExtra, when set, chains one UNSTAMPED page after chunk 0's + // scoped pages — cold and warm alike. Models the Okta shape: a + // type-scoped cursor whose members pages are scope-stamped, followed + // by fresh pages with no change signal (group role assignments) that + // carry no source-cache annotations and must be re-emitted every sync. + unstampedExtra bool + + // observability for assertions + planningCalls int + coldChunkRounds int + warmChunkRounds int + perResourceGroupLG int + unstampedPages int +} + +var typeScopedGroupRT = v2.ResourceType_builder{ + Id: "group", + DisplayName: "Group", + Traits: []v2.ResourceType_Trait{v2.ResourceType_TRAIT_GROUP}, + Annotations: annotations.New(&v2.TypeScopedGrants{}), +}.Build() + +func newChunkedGrantsMockConnector(t *testing.T, groupCount int, membersPerGroup int) *chunkedGrantsMockConnector { + t.Helper() + mc := &chunkedGrantsMockConnector{ + mockConnector: newMockConnector(), + members: map[string][]string{}, + entByID: map[string]*v2.Entitlement{}, + resByID: map[string]*v2.Resource{}, + tokenByChunk: map[int]string{}, + addsByChunk: map[int][]*v2.Grant{}, + deletedIDsByChunk: map[int][]string{}, + } + mc.rtDB = append(mc.rtDB, typeScopedGroupRT, userResourceType) + + // A pool of users, each carrying SkipEntitlementsAndGrants so only the + // group type participates in the grants phase. + userIDs := make([]string, 0, 10) + for i := 0; i < 10; i++ { + uid := fmt.Sprintf("user-%02d", i) + u, err := rs.NewUserResource(uid, userResourceType, uid, nil, rs.WithAnnotation(&v2.SkipEntitlementsAndGrants{})) + require.NoError(t, err) + mc.AddResource(context.Background(), u) + mc.resByID[uid] = u + userIDs = append(userIDs, uid) + } + + for i := 0; i < groupCount; i++ { + gid := fmt.Sprintf("group-%03d", i) + g, err := rs.NewGroupResource(gid, typeScopedGroupRT, gid, nil) + require.NoError(t, err) + mc.AddResource(context.Background(), g) + mc.resByID[gid] = g + mc.groupIDs = append(mc.groupIDs, gid) + + ent := et.NewAssignmentEntitlement(g, "member", et.WithGrantableTo(userResourceType)) + ent.SetSlug("member") + mc.entByID[gid] = ent + mc.entDB[gid] = []*v2.Entitlement{ent} + + for m := 0; m < membersPerGroup; m++ { + mc.members[gid] = append(mc.members[gid], userIDs[(i+m)%len(userIDs)]) + } + } + sort.Strings(mc.groupIDs) + return mc +} + +func (mc *chunkedGrantsMockConnector) SetSourceCache(_ context.Context, lookup sourcecache.Lookup) { + mc.mu.Lock() + defer mc.mu.Unlock() + if lookup == nil { + lookup = sourcecache.NoopLookup{} + } + mc.lookup = lookup +} + +func (mc *chunkedGrantsMockConnector) Validate(context.Context, *v2.ConnectorServiceValidateRequest, ...grpc.CallOption) (*v2.ConnectorServiceValidateResponse, error) { + return v2.ConnectorServiceValidateResponse_builder{ + Annotations: annotations.New(v2.SourceCacheCapability_builder{ + Mode: v2.SourceCacheCapability_MODE_READ_WRITE, + }.Build()), + }.Build(), nil +} + +func (mc *chunkedGrantsMockConnector) chunkCount() int { + return (len(mc.groupIDs) + chunkSize - 1) / chunkSize +} + +func (mc *chunkedGrantsMockConnector) chunkGroups(chunk int) []string { + lo := chunk * chunkSize + hi := lo + chunkSize + if hi > len(mc.groupIDs) { + hi = len(mc.groupIDs) + } + return mc.groupIDs[lo:hi] +} + +func chunkScope(chunk int) string { + return fmt.Sprintf("groups/delta/chunk/%03d?sig=testsig", chunk) +} + +func (mc *chunkedGrantsMockConnector) memberGrant(gid, uid string) *v2.Grant { + return gt.NewGrant(mc.resByID[gid], "member", mc.resByID[uid].GetId()) +} + +func (mc *chunkedGrantsMockConnector) chunkGrants(chunk int) []*v2.Grant { + var out []*v2.Grant + for _, gid := range mc.chunkGroups(chunk) { + for _, uid := range mc.members[gid] { + out = append(out, mc.memberGrant(gid, uid)) + } + } + return out +} + +// mutateAddMember journals a warm-round overlay add and updates live state +// (so a cold control sync sees the same tenant). +func (mc *chunkedGrantsMockConnector) mutateAddMember(gid, uid string) { + mc.mu.Lock() + defer mc.mu.Unlock() + mc.members[gid] = append(mc.members[gid], uid) + chunk := sort.SearchStrings(mc.groupIDs, gid) / chunkSize + mc.addsByChunk[chunk] = append(mc.addsByChunk[chunk], mc.memberGrant(gid, uid)) +} + +// mutateRemoveMember journals a warm-round tombstone and updates live state. +func (mc *chunkedGrantsMockConnector) mutateRemoveMember(gid, uid string) { + mc.mu.Lock() + defer mc.mu.Unlock() + kept := mc.members[gid][:0] + for _, m := range mc.members[gid] { + if m != uid { + kept = append(kept, m) + } + } + mc.members[gid] = kept + chunk := sort.SearchStrings(mc.groupIDs, gid) / chunkSize + mc.deletedIDsByChunk[chunk] = append(mc.deletedIDsByChunk[chunk], mc.memberGrant(gid, uid).GetId()) +} + +func (mc *chunkedGrantsMockConnector) clearDeltas() { + mc.mu.Lock() + defer mc.mu.Unlock() + mc.addsByChunk = map[int][]*v2.Grant{} + mc.deletedIDsByChunk = map[int][]string{} +} + +func (mc *chunkedGrantsMockConnector) ListGrants(ctx context.Context, in *v2.GrantsServiceListGrantsRequest, _ ...grpc.CallOption) (*v2.GrantsServiceListGrantsResponse, error) { + rid := in.GetResource().GetId() + if rid.GetResourceType() != typeScopedGroupRT.GetId() { + // Users are annotated SkipEntitlementsAndGrants; nothing else + // should arrive here. + return v2.GrantsServiceListGrantsResponse_builder{}.Build(), nil + } + reqAnnos := annotations.Annotations(in.GetAnnotations()) + if !reqAnnos.Contains(&v2.TypeScopedGrants{}) { + // The planner must never fan out per-resource for the annotated + // type; recorded and asserted in the test. + mc.mu.Lock() + mc.perResourceGroupLG++ + mc.mu.Unlock() + return v2.GrantsServiceListGrantsResponse_builder{}.Build(), nil + } + + // --- type-scoped call --- + tok := in.GetPageToken() + if tok == "" { + // Planning call: spawn one cursor per chunk, no rows. + mc.mu.Lock() + mc.planningCalls++ + n := mc.chunkCount() + mc.mu.Unlock() + tokens := make([]string, 0, n) + for i := 0; i < n; i++ { + tokens = append(tokens, fmt.Sprintf("chunk=%d", i)) + } + return v2.GrantsServiceListGrantsResponse_builder{ + Annotations: annotations.New(v2.SpawnCursors_builder{ + PageTokens: tokens, + EstimatedTotal: int64(len(mc.groupIDs)), + }.Build()), + }.Build(), nil + } + + if tok == "chunk=0&extra" { + // The unstamped trailing page: fresh rows, NO source-cache + // annotations, emitted on every sync (there is no validator for + // them, so there is no legal replay). + mc.mu.Lock() + mc.unstampedPages++ + mc.mu.Unlock() + return v2.GrantsServiceListGrantsResponse_builder{ + List: mc.unstampedExtraGrants(), + }.Build(), nil + } + + var chunk, page int + if n, err := fmt.Sscanf(tok, "chunk=%d&page=%d", &chunk, &page); err != nil || n != 2 { + if _, err := fmt.Sscanf(tok, "chunk=%d", &chunk); err != nil { + return nil, fmt.Errorf("bad page token %q", tok) + } + page = 0 + } + scope := chunkScope(chunk) + + mc.mu.Lock() + lookup := mc.lookup + lastTok := mc.tokenByChunk[chunk] + adds := mc.addsByChunk[chunk] + deleted := mc.deletedIDsByChunk[chunk] + gen := mc.generation + mc.mu.Unlock() + if lookup == nil { + lookup = sourcecache.NoopLookup{} + } + + newTok := fmt.Sprintf("tok-c%d-g%d", chunk, gen) + rotate := func() { + mc.mu.Lock() + mc.tokenByChunk[chunk] = newTok + mc.mu.Unlock() + } + + if page == 0 { + entry, found, err := lookup.LookupPreviousSourceCache(ctx, sourcecache.RowKindGrants, scope) + if err != nil { + return nil, err + } + if found && lastTok != "" && entry.ETag == lastTok { + // Warm round: one page — replay, overlay adds, tombstones, + // rotated validator. With unstampedExtra set, chunk 0 chains + // its unstamped trailing page after the replay page: scoped + // and unscoped pages coexisting in one cursor's stream. + mc.mu.Lock() + mc.warmChunkRounds++ + mc.mu.Unlock() + rotate() + next := "" + if mc.unstampedExtra && chunk == 0 { + next = "chunk=0&extra" + } + return v2.GrantsServiceListGrantsResponse_builder{ + List: adds, + Annotations: annotations.New(v2.SourceCacheReplay_builder{ + ScopeHash: scope, + Etag: newTok, + Overlay: true, + DeletedIds: deleted, + }.Build()), + NextPageToken: next, + }.Build(), nil + } + } + + // Cold enumeration for this chunk, two pages to exercise cursor + // checkpointing: page 0 = first half + interim scope (no etag), + // page 1 = rest + validator. + grants := mc.chunkGrants(chunk) + half := len(grants) / 2 + if page == 0 { + mc.mu.Lock() + mc.coldChunkRounds++ + mc.mu.Unlock() + return v2.GrantsServiceListGrantsResponse_builder{ + List: grants[:half], + Annotations: annotations.New(v2.SourceCacheScope_builder{ + ScopeHash: scope, + }.Build()), + NextPageToken: fmt.Sprintf("chunk=%d&page=1", chunk), + }.Build(), nil + } + rotate() + next := "" + if mc.unstampedExtra && chunk == 0 { + next = "chunk=0&extra" + } + return v2.GrantsServiceListGrantsResponse_builder{ + List: grants[half:], + Annotations: annotations.New(v2.SourceCacheScope_builder{ + ScopeHash: scope, + Etag: newTok, + }.Build()), + NextPageToken: next, + }.Build(), nil +} + +// unstampedExtraGrants are the rows served by the unstamped trailing page: +// "admin" grants on group-000 for two fixed users. They exist only while +// the connector emits them — nothing replays them. +func (mc *chunkedGrantsMockConnector) unstampedExtraGrants() []*v2.Grant { + g := mc.resByID[mc.groupIDs[0]] + return []*v2.Grant{ + gt.NewGrant(g, "admin", mc.resByID["user-08"].GetId()), + gt.NewGrant(g, "admin", mc.resByID["user-09"].GetId()), + } +} + +func requireSameGrantIDs(t *testing.T, want, got map[string]*v2.Grant, msg string) { + t.Helper() + var missing, extra []string + for id := range want { + if _, ok := got[id]; !ok { + missing = append(missing, id) + } + } + for id := range got { + if _, ok := want[id]; !ok { + extra = append(extra, id) + } + } + sort.Strings(missing) + sort.Strings(extra) + require.Emptyf(t, missing, "%s: grants missing: %s", msg, strings.Join(missing, ", ")) + require.Emptyf(t, extra, "%s: unexpected grants: %s", msg, strings.Join(extra, ", ")) +} + +func runChunkedSync(ctx context.Context, t *testing.T, mc *chunkedGrantsMockConnector, path, prevPath, tmpDir string, extraOpts ...SyncOpt) { + t.Helper() + store, err := dotc1z.NewStore(ctx, path, + dotc1z.WithEngine(c1zstore.EnginePebble), + dotc1z.WithTmpDir(tmpDir), + ) + require.NoError(t, err) + opts := []SyncOpt{WithConnectorStore(store), WithTmpDir(tmpDir)} + if prevPath != "" { + opts = append(opts, WithPreviousSyncC1ZPath(prevPath)) + } + opts = append(opts, extraOpts...) + syncer, err := NewSyncer(ctx, mc, opts...) + require.NoError(t, err) + require.NoError(t, syncer.Sync(ctx)) + require.NoError(t, syncer.Close(ctx)) +} + +// TestTypeScopedGrantsMixedStampedUnstampedCursor pins a contract point +// proven in the field by the baton-okta POC: within ONE type-scoped +// cursor's page stream, scope-stamped pages (members, replayable) and +// unstamped fresh pages (rows with no change signal, e.g. group role +// assignments) coexist. Stamping is per-page context, not per-cursor +// state, so: +// +// - warm syncs replay the stamped pages while the unstamped pages are +// re-emitted fresh by the connector, and the union matches a cold +// sync exactly; +// - unstamped rows never ride replay: the sync only contains them while +// the connector emits them. +func TestTypeScopedGrantsMixedStampedUnstampedCursor(t *testing.T) { + ctx, err := logging.Init(context.Background()) + require.NoError(t, err) + tmpDir := t.TempDir() + + // 10 groups => 1 chunk, 3 members each => 30 member grants, plus 2 + // unstamped "admin" grants from the trailing extra page. + mc := newChunkedGrantsMockConnector(t, 10, 3) + mc.unstampedExtra = true + require.Equal(t, 1, mc.chunkCount()) + + // --- Sync 1: cold — scoped member pages then the unstamped page ---- + sync1 := filepath.Join(tmpDir, "sync1.c1z") + runChunkedSync(ctx, t, mc, sync1, "", tmpDir) + require.Equal(t, 1, mc.coldChunkRounds) + require.Equal(t, 1, mc.unstampedPages, "cold sync serves the unstamped trailing page") + + grants1 := listGrantsInFile(ctx, t, sync1) + require.Len(t, grants1, 32) + + // --- Sync 2: warm — replay page chains into the unstamped page ----- + mc.generation = 1 + sync2 := filepath.Join(tmpDir, "sync2.c1z") + runChunkedSync(ctx, t, mc, sync2, sync1, tmpDir) + require.Equal(t, 1, mc.warmChunkRounds, "member pages replay warm") + require.Equal(t, 1, mc.coldChunkRounds, "no cold re-enumeration") + require.Equal(t, 2, mc.unstampedPages, "unstamped page is re-served on the warm sync — it never replays") + + grants2 := listGrantsInFile(ctx, t, sync2) + requireSameGrantIDs(t, grants1, grants2, "warm sync with mixed pages vs cold") + + // --- Sync 3: warm, connector stops emitting the unstamped page ----- + // The admin grants must vanish: they were never scope-stamped, so + // nothing replays them. If they survive, unstamped rows are leaking + // into replay. + mc.unstampedExtra = false + mc.generation = 2 + sync3 := filepath.Join(tmpDir, "sync3.c1z") + runChunkedSync(ctx, t, mc, sync3, sync2, tmpDir) + require.Equal(t, 2, mc.warmChunkRounds) + require.Equal(t, 2, mc.unstampedPages, "no unstamped page served on sync 3") + + grants3 := listGrantsInFile(ctx, t, sync3) + require.Len(t, grants3, 30, "unstamped rows must not ride replay") + for _, g := range mc.unstampedExtraGrants() { + require.NotContains(t, grants3, g.GetId()) + } +} + +func TestTypeScopedGrantsChunkedDelta(t *testing.T) { + for _, workers := range []int{0, 4} { + t.Run(fmt.Sprintf("workers=%d", workers), func(t *testing.T) { + runChunkedDeltaScenario(t, workers) + }) + } +} + +func runChunkedDeltaScenario(t *testing.T, workers int) { + ctx, err := logging.Init(context.Background()) + require.NoError(t, err) + tmpDir := t.TempDir() + + var workerOpts []SyncOpt + if workers > 0 { + workerOpts = append(workerOpts, WithWorkerCount(workers)) + } + + // 120 groups => 3 chunks (50/50/20), 3 members each => 360 grants. + mc := newChunkedGrantsMockConnector(t, 120, 3) + require.Equal(t, 3, mc.chunkCount()) + + // --- Sync 1: cold --------------------------------------------------- + sync1 := filepath.Join(tmpDir, "sync1.c1z") + runChunkedSync(ctx, t, mc, sync1, "", tmpDir, workerOpts...) + + require.Equal(t, 1, mc.planningCalls, "one planning call per sync") + require.Equal(t, 3, mc.coldChunkRounds, "every chunk enumerates cold on the first sync") + require.Zero(t, mc.warmChunkRounds) + require.Zero(t, mc.perResourceGroupLG, "planner must not fan out per-resource for the annotated type") + + grants1 := listGrantsInFile(ctx, t, sync1) + require.Len(t, grants1, 360) + + // --- Sync 2: warm with changes in two chunks ------------------------ + mc.generation = 1 + mc.mutateAddMember("group-071", "user-00") // chunk 1 + mc.mutateRemoveMember("group-115", mc.members["group-115"][0]) // chunk 2 + + sync2 := filepath.Join(tmpDir, "sync2.c1z") + runChunkedSync(ctx, t, mc, sync2, sync1, tmpDir, workerOpts...) + + require.Equal(t, 2, mc.planningCalls) + require.Equal(t, 3, mc.coldChunkRounds, "no chunk re-enumerates on the warm sync") + require.Equal(t, 3, mc.warmChunkRounds, "every chunk replays warm") + require.Zero(t, mc.perResourceGroupLG) + + grants2 := listGrantsInFile(ctx, t, sync2) + require.Len(t, grants2, 360, "one add and one remove cancel out") + require.Contains(t, grants2, mc.memberGrant("group-071", "user-00").GetId(), "overlay add must land") + + // Equivalence: a cold control sync against the same tenant state must + // produce the same grant set. + mc.clearDeltas() + control := filepath.Join(tmpDir, "control.c1z") + runChunkedSync(ctx, t, mc, control, "", tmpDir, workerOpts...) + require.Equal(t, 6, mc.coldChunkRounds, "control (no previous c1z) re-enumerates every chunk") + controlGrants := listGrantsInFile(ctx, t, control) + requireSameGrantIDs(t, controlGrants, grants2, "warm sync vs cold control") + + // --- Sync 3: no-op warm chained off sync 2 -------------------------- + mc.generation = 2 + sync3 := filepath.Join(tmpDir, "sync3.c1z") + runChunkedSync(ctx, t, mc, sync3, sync2, tmpDir, workerOpts...) + require.Equal(t, 6, mc.warmChunkRounds, "no-op round replays all chunks (validators from sync 2 rotated)") + require.Equal(t, 6, mc.coldChunkRounds, "no cold rounds on the no-op sync") + grants3 := listGrantsInFile(ctx, t, sync3) + requireSameGrantIDs(t, grants2, grants3, "no-op warm sync vs previous") +} diff --git a/pkg/synccompactor/compactor_pebble.go b/pkg/synccompactor/compactor_pebble.go index 9983ba315..7bedbeafc 100644 --- a/pkg/synccompactor/compactor_pebble.go +++ b/pkg/synccompactor/compactor_pebble.go @@ -647,6 +647,18 @@ func (c *Compactor) compactPebbleFold(ctx context.Context) (string, error) { l.Info("compactPebbleFold: no grant writes; base grant digest state left untouched") } + // Drop the source-cache manifest inherited from the base copy. The + // fold is a keep-newer UPSERT merge — base rows the partials deleted + // survive — so the output is not a faithful snapshot of any input + // sync and NO input's validator may be allowed to 304-validate it. + // With the manifest empty, a syncer using this artifact as its + // previous sync misses every lookup and fetches fresh: cold-correct. + // (The rebuild path is already clean: its bucket specs never carry + // source-cache entries into the fresh output.) + if err := destEng.ClearSourceCacheEntries(ctx); err != nil { + return "", fmt.Errorf("compactPebbleFold: clear source-cache manifest: %w", err) + } + // Optionally compact the folded LSM before save. Off by default: // a compaction rewrites the SSTs that overlap the partials' writes // — for scattered overrides that is most of the base — which both @@ -680,6 +692,10 @@ func (c *Compactor) compactPebbleFold(ctx context.Context) (string, error) { baseRec.SetSyncId(newSyncID) baseRec.SetParentSyncId("") baseRec.SetType(unionType) + // Compaction provenance: this artifact is a keep-newer merge, not a + // connector run — no input's source-cache validators describe it, so + // it must never serve as a replay source (see SyncRunRecord.compacted). + baseRec.SetCompacted(true) if !maxEnded.IsZero() { baseRec.SetEndedAt(timestamppb.New(maxEnded)) } @@ -1186,6 +1202,9 @@ func (c *Compactor) compactPebble(ctx context.Context, newSyncId string) error { return fmt.Errorf("compactPebble: load dest sync_run: %w", err) } rec.SetType(unionType) + // Compaction provenance: rebuild outputs are keep-newer merges too — + // never replay sources (see SyncRunRecord.compacted). + rec.SetCompacted(true) if !maxEnded.IsZero() { rec.SetEndedAt(timestamppb.New(maxEnded)) } diff --git a/pkg/synccompactor/pebble/bucket_plans.go b/pkg/synccompactor/pebble/bucket_plans.go index 33d9be541..7b63bdac9 100644 --- a/pkg/synccompactor/pebble/bucket_plans.go +++ b/pkg/synccompactor/pebble/bucket_plans.go @@ -60,6 +60,34 @@ func buildBucketPlans() []bucketPlan { lower: enginepkg.GrantByNeedsExpansionLowerBound(), upper: enginepkg.GrantByNeedsExpansionUpperBound(), }, + // Source-cache state MUST fold with its rows. Excising these + // ranges alongside the record buckets replaces the base sync's + // manifest/index entries with the applied sync's: scopes the + // applied sync touched keep a consistent (entry, rows) pair, and + // scopes it never touched become clean lookup misses. Leaving + // them out would strand the BASE sync's validators and index + // entries on top of the applied sync's rows — a stale manifest + // hit could then replay rows that no longer belong to the scope. + { + name: "grant_by_source_scope", + lower: enginepkg.GrantBySourceScopeLowerBound(), + upper: enginepkg.GrantBySourceScopeUpperBound(), + }, + { + name: "entitlement_by_source_scope", + lower: enginepkg.EntitlementBySourceScopeLowerBound(), + upper: enginepkg.EntitlementBySourceScopeUpperBound(), + }, + { + name: "resource_by_source_scope", + lower: enginepkg.ResourceBySourceScopeLowerBound(), + upper: enginepkg.ResourceBySourceScopeUpperBound(), + }, + { + name: "source_cache_entry", + lower: enginepkg.SourceCacheEntryLowerBound(), + upper: enginepkg.SourceCacheEntryUpperBound(), + }, { name: "grant_by_entitlement_principal_hash", lower: enginepkg.GrantByEntPrincHashLowerBound(), diff --git a/pkg/synccompactor/source_cache_replay_test.go b/pkg/synccompactor/source_cache_replay_test.go new file mode 100644 index 000000000..3200dfda5 --- /dev/null +++ b/pkg/synccompactor/source_cache_replay_test.go @@ -0,0 +1,180 @@ +package synccompactor + +// Compacted artifacts and source-cache replay. +// +// The production compaction paths (fold's keep-newer MergeInto, the +// overlay/kway rebuild) are UPSERT-merges: base rows the newer sync +// didn't touch survive, and deletions don't propagate. A compacted +// artifact is therefore NOT a faithful snapshot of its newest input +// sync. Carrying the newest sync's source-cache manifest entries into +// such an artifact would be dangerous: a future sync could 304-validate +// a scope against the carried etag and replay a row set that silently +// includes resurrected base rows — stale data presented as current. +// +// The safe semantic — and the one these tests PIN — is that compacted +// artifacts carry NO source-cache manifest entries: every lookup against +// a compacted replay source misses, the connector fetches fresh, and the +// sync is cold-correct. (The IngestAndExcise compactor in +// pkg/synccompactor/pebble replaces whole key ranges and could preserve +// entries soundly — see its bucket_plans.go — but it is not the +// production path.) +// +// Consequence worth knowing operationally: if the previous-sync c1z fed +// to a syncer is a compaction artifact, ETag replay silently never +// engages. Feed raw sync files to WithPreviousSyncC1ZPath when replay +// matters. + +import ( + "context" + "fmt" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + "github.com/conductorone/baton-sdk/pkg/connectorstore" + "github.com/conductorone/baton-sdk/pkg/dotc1z" + "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" + pebbleengine "github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble" + "github.com/conductorone/baton-sdk/pkg/sourcecache" + gt "github.com/conductorone/baton-sdk/pkg/types/grant" + rs "github.com/conductorone/baton-sdk/pkg/types/resource" +) + +var ( + scGroupRT = v2.ResourceType_builder{Id: "group", DisplayName: "Group", Traits: []v2.ResourceType_Trait{v2.ResourceType_TRAIT_GROUP}}.Build() + scUserRT = v2.ResourceType_builder{Id: "user", DisplayName: "User", Traits: []v2.ResourceType_Trait{v2.ResourceType_TRAIT_USER}}.Build() +) + +// writeScopedSyncFile writes one finished Pebble sync whose grants are +// stamped with scope and whose manifest carries (grants, scope, etag). +func writeScopedSyncFile(ctx context.Context, t *testing.T, path, tmpDir, scope, etag string, memberIDs []string) string { + t.Helper() + store, err := dotc1z.NewStore(ctx, path, + dotc1z.WithEngine(c1zstore.EnginePebble), + dotc1z.WithTmpDir(tmpDir), + ) + require.NoError(t, err) + + syncID, isNew, err := store.StartOrResumeSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + require.True(t, isNew) + + group, err := rs.NewGroupResource("g1", scGroupRT, "g1", nil) + require.NoError(t, err) + require.NoError(t, store.PutResourceTypes(ctx, scGroupRT, scUserRT)) + require.NoError(t, store.PutResources(ctx, group)) + + grants := make([]*v2.Grant, 0, len(memberIDs)) + for _, uid := range memberIDs { + user, err := rs.NewUserResource(uid, scUserRT, uid, nil) + require.NoError(t, err) + require.NoError(t, store.PutResources(ctx, user)) + grants = append(grants, gt.NewGrant(group, "member", user)) + } + putCtx := sourcecache.WithScope(ctx, scope) + require.NoError(t, store.PutGrants(putCtx, grants...)) + + scStore, ok := any(store).(dotc1z.SourceCacheStore) + require.True(t, ok) + require.NoError(t, scStore.PutSourceCacheEntry(ctx, sourcecache.RowKindGrants, scope, etag)) + + require.NoError(t, store.EndSync(ctx)) + require.NoError(t, store.Close(ctx)) + return syncID +} + +// TestCompactedFileIsColdReplaySource pins the compaction/replay safety +// contract for both production Pebble modes: the compacted output must +// carry NO source-cache manifest entries, so a syncer replaying from it +// misses every lookup and goes cold — never a 304 against a merged row +// set that upsert-merge semantics cannot guarantee matches the etag. +func TestCompactedFileIsColdReplaySource(t *testing.T) { + for _, mode := range []PebbleCompactorMode{PebbleCompactorModeFold, PebbleCompactorModeOverlay} { + t.Run(string(mode), func(t *testing.T) { + ctx := t.Context() + tmpDir := t.TempDir() + scope := sourcecache.HashScope("mock://grants/g1") + + olderPath := filepath.Join(tmpDir, "older.c1z") + newerPath := filepath.Join(tmpDir, "newer.c1z") + olderID := writeScopedSyncFile(ctx, t, olderPath, tmpDir, scope, "v1", []string{"u1", "u2"}) + newerID := writeScopedSyncFile(ctx, t, newerPath, tmpDir, scope, "v2", []string{"u1", "u3"}) + + // Sanity: the inputs DO carry manifest entries, or the + // empty-output assertion below tests nothing. + for _, p := range []string{olderPath, newerPath} { + in, err := dotc1z.NewStore(ctx, p, dotc1z.WithReadOnly(true), dotc1z.WithTmpDir(tmpDir)) + require.NoError(t, err) + holder, ok := any(in).(interface{ PebbleEngine() *pebbleengine.Engine }) + require.True(t, ok) + manifest, err := holder.PebbleEngine().SourceCacheManifestSnapshot(ctx) + require.NoError(t, err) + require.Len(t, manifest, 1, "sanity: input %s must carry its manifest entry", p) + require.NoError(t, in.Close(ctx)) + } + + outDir := filepath.Join(tmpDir, "out") + require.NoError(t, os.MkdirAll(outDir, 0o755)) + compactor, cleanup, err := NewCompactor(ctx, outDir, []*CompactableSync{ + {FilePath: olderPath, SyncID: olderID}, + {FilePath: newerPath, SyncID: newerID}, + }, WithTmpDir(tmpDir), WithPebbleCompactorMode(mode)) + require.NoError(t, err) + defer func() { require.NoError(t, cleanup()) }() + out, err := compactor.Compact(ctx) + require.NoError(t, err) + require.NotNil(t, out) + + compacted, err := dotc1z.NewStore(ctx, out.FilePath, + dotc1z.WithReadOnly(true), + dotc1z.WithTmpDir(tmpDir), + ) + require.NoError(t, err) + defer func() { _ = compacted.Close(ctx) }() + run, err := compacted.SyncMeta().LatestFinishedSyncOfAnyType(ctx) + require.NoError(t, err) + require.NotNil(t, run) + require.NoError(t, compacted.SetCurrentSync(ctx, run.ID)) + + // Provenance metadata: the artifact must SAY it was + // compacted, so orchestrators (and the syncer's replay gate) + // can answer "can this be used for replay?" without + // inspecting keyspaces. The answer is no. + require.True(t, run.Compacted, + "compaction must stamp the output sync run's compacted flag") + + // The safety property: NO manifest entries survive, so a + // syncer using this artifact as its previous sync misses + // every lookup (cold-correct), never replays a merged row + // set against a carried validator. + holder, ok := any(compacted).(interface{ PebbleEngine() *pebbleengine.Engine }) + require.True(t, ok, "compacted output must be a pebble store") + manifest, err := holder.PebbleEngine().SourceCacheManifestSnapshot(ctx) + require.NoError(t, err) + require.Empty(t, manifest, + "compacted artifacts must carry no source-cache manifest entries: upsert-merge output cannot be safely validated by any input's etag") + + scStore, ok := any(compacted).(dotc1z.SourceCacheStore) + require.True(t, ok) + _, found, err := scStore.LookupSourceCacheEntry(ctx, sourcecache.RowKindGrants, scope) + require.NoError(t, err) + require.False(t, found, "a lookup against a compacted replay source must miss") + + // The merged DATA is still there (upsert semantics: newest + // wins per key; base-only rows survive). + grants, err := compacted.ListGrants(ctx, v2.GrantsServiceListGrantsRequest_builder{}.Build()) + require.NoError(t, err) + ids := map[string]bool{} + for _, g := range grants.GetList() { + ids[g.GetId()] = true + } + for _, uid := range []string{"u1", "u3"} { + require.True(t, ids[fmt.Sprintf("group:g1:member:user:%s", uid)], + "newest sync's row for %s must be present", uid) + } + }) + } +} diff --git a/pkg/tasks/local/syncer.go b/pkg/tasks/local/syncer.go index 5d469ca50..916b166db 100644 --- a/pkg/tasks/local/syncer.go +++ b/pkg/tasks/local/syncer.go @@ -24,6 +24,7 @@ type localSyncer struct { o sync.Once tmpDir string externalResourceC1Z string + previousSyncC1Z string externalResourceEntitlementIdFilter string targetedSyncResources []*v2.Resource skipEntitlementsAndGrants bool @@ -47,6 +48,12 @@ func WithExternalResourceC1Z(externalResourceC1Z string) Option { } } +func WithPreviousSyncC1Z(previousSyncC1Z string) Option { + return func(m *localSyncer) { + m.previousSyncC1Z = previousSyncC1Z + } +} + func WithExternalResourceEntitlementIdFilter(entitlementId string) Option { return func(m *localSyncer) { m.externalResourceEntitlementIdFilter = entitlementId @@ -122,6 +129,7 @@ func (m *localSyncer) Process(ctx context.Context, task *v1.Task, cc types.Conne sdkSync.WithC1ZPath(m.dbPath), sdkSync.WithTmpDir(m.tmpDir), sdkSync.WithExternalResourceC1ZPath(m.externalResourceC1Z), + sdkSync.WithPreviousSyncC1ZPath(m.previousSyncC1Z), sdkSync.WithExternalResourceEntitlementIdFilter(m.externalResourceEntitlementIdFilter), sdkSync.WithTargetedSyncResources(m.targetedSyncResources), sdkSync.WithSkipEntitlementsAndGrants(m.skipEntitlementsAndGrants), diff --git a/pkg/types/resource/resource.go b/pkg/types/resource/resource.go index 16585ed3f..49a01e0f1 100644 --- a/pkg/types/resource/resource.go +++ b/pkg/types/resource/resource.go @@ -8,6 +8,7 @@ import ( v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" "github.com/conductorone/baton-sdk/pkg/annotations" "github.com/conductorone/baton-sdk/pkg/pagination" + "github.com/conductorone/baton-sdk/pkg/sourcecache" "github.com/conductorone/baton-sdk/pkg/types/sessions" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -524,9 +525,14 @@ func NewManagedDeviceResource( } type SyncOpAttrs struct { - Session sessions.SessionStore - SyncID string - PageToken pagination.Token + Session sessions.SessionStore + // SourceCache resolves a scope's previous-sync validator (etag / + // delta token) for source-cache replay. Never nil for framework-built + // connectors: when source cache is disabled or degraded the SDK + // supplies sourcecache.NoopLookup, which always misses. + SourceCache sourcecache.Lookup + SyncID string + PageToken pagination.Token } type SyncOpResults struct { diff --git a/proto/c1/connector/v2/annotation_source_cache.proto b/proto/c1/connector/v2/annotation_source_cache.proto new file mode 100644 index 000000000..a139fff1c --- /dev/null +++ b/proto/c1/connector/v2/annotation_source_cache.proto @@ -0,0 +1,250 @@ +syntax = "proto3"; + +package c1.connector.v2; + +import "validate/validate.proto"; + +option go_package = "github.com/conductorone/baton-sdk/pb/c1/connector/v2"; + +// Source-cache replay annotations. +// +// A connector that talks to an upstream API with cheap change detection +// (HTTP conditional requests, delta/watermark queries) can opt in to +// source-cache replay: the SDK persists which rows each upstream +// request/scope produced, and on the next sync the connector can tell the +// SDK to replay a scope's previous rows instead of re-emitting them. +// +// The connector owns scope computation. A scope is any stable partition of +// upstream data the connector can cheaply revalidate as a unit: a page URL +// for GitHub-style conditional GETs, a whole collection for Microsoft +// Graph-style delta queries. The SDK never computes provider scopes; it +// only keys storage by the connector-supplied scope_hash. +// +// The etag is an opaque validator persisted per scope: a literal HTTP ETag +// for conditional requests, a delta/skip token for delta queries. The SDK +// hands it back via the source-cache lookup on the next sync and never +// interprets it. +// +// Flow per sync, per scope: +// +// 1. Connector looks up the previous etag for (row_kind, scope_hash) via +// the SDK-provided lookup. Miss => fetch fresh, emit rows, attach +// SourceCacheScope. Hit => revalidate upstream with the etag. +// 2. Unchanged (HTTP 304): attach SourceCacheReplay with overlay=false and +// the (unchanged) etag; emit no rows. +// 3. Changed-with-diff (delta query): attach SourceCacheReplay with +// overlay=true and any removed ids; emit ONLY the changed rows (tagged +// with SourceCacheScope); the final page's SourceCacheScope carries the +// new token as etag. +// 4. Changed-without-diff (plain conditional GET): fetch fresh, emit rows, +// attach SourceCacheScope with the new etag. Nothing is replayed. +// +// Replay is only honored when the connector declared +// SourceCacheCapability MODE_READ_WRITE on its Validate response AND the +// sync is running on a storage engine that supports source-cache replay +// with a usable previous sync. When either condition fails the SDK +// installs a lookup that always misses, so a well-behaved connector +// (invariant: only emit SourceCacheReplay for a scope whose etag came +// from this sync's lookup) naturally degrades to full fetch. +// +// "This sync's lookup" means the lookup happened during THIS sync — not +// necessarily during the same call that emits the replay. A planning call +// may batch-resolve validators for many scopes and hand the verdicts to +// sibling cursors via SpawnCursors page tokens; a replay emitted by such a +// cursor satisfies the invariant. What the invariant forbids is validators +// that outlive the sync (connector-side caches, config, upstream echoes): +// the lookup result must originate from the sync that consumes it. + +// SourceCacheCapability is attached to ConnectorServiceValidateResponse +// annotations to opt in to source-cache replay. Absent or any mode other +// than MODE_READ_WRITE means all source-cache annotations are ignored. +message SourceCacheCapability { + enum Mode { + MODE_UNSPECIFIED = 0; + MODE_DISABLED = 1; + MODE_READ_WRITE = 2; + } + Mode mode = 1; +} + +// SourceCacheScope is attached to a list-response page whose rows were +// freshly fetched from upstream. The SDK stamps the page's rows with +// scope_hash so a future sync can replay them as a unit. +message SourceCacheScope { + // Connector-computed stable identifier for the canonical scope. Must be + // byte-stable across syncs for the same logical scope. Prefer an ORDERED + // natural identifier (e.g. the request URL, or "groups/{id}/members") + // over a random hash: scope-index writes lead with this value, so + // identifiers that correlate with fetch order keep index writes nearly + // append-ordered at scale. sourcecache.HashScope is the fallback when no + // compact natural form exists. + string scope_hash = 1; + + // Opaque validator to persist for this scope. May be empty on interim + // pages of a multi-page scope (e.g. Graph @odata.nextLink pages); the + // SDK writes the scope's manifest entry when a non-empty etag arrives. + // A 200 response with zero rows still persists the entry. + string etag = 2; + + // Tombstones applied after this page's rows commit. Lets every page of + // a multi-page delta round carry its own deletions as the provider + // delivers them, instead of buffering a whole round onto the first + // (replay-annotated) page. Same formats as SourceCacheReplay. + // + // PRECONDITION for tombstones anywhere in a round: the provider's delta + // must be coalesced — at most one add-or-tombstone per object per round + // (Microsoft Graph guarantees this by returning final object state). + // With interleaved add/remove events for one object, per-page ordering + // is deterministic (a page's rows upsert before its deletions apply) + // but cross-page re-adds after a tombstone are the connector's + // responsibility to order. + repeated string deleted_ids = 3; + + // Principal-scoped grant tombstones: for RowKindGrants pages, each + // entry deletes EVERY grant row stamped with this scope whose principal + // id equals the entry — no principal resource type and no canonical + // grant-id reconstruction required (delta tombstones usually carry only + // a bare object id, and the object may no longer exist to look up). + // + // PRECONDITION: the scope must be partitioned so that "principal + // removed from scope" means every grant they have in the scope is gone + // — one scope per navigation with independent removal semantics (e.g. + // members and owners of a group are separate scopes, or membership + // removal would take the owner grant with it). + // + // For RowKindResources pages, each entry deletes the resource row(s) + // stamped with this scope whose resource id equals the entry, any + // resource type. + repeated string deleted_principal_ids = 4; +} + +// SourceCacheReplay is attached to a list-response page to tell the SDK to +// copy the previous sync's rows for scope_hash into the current sync. +// +// The row kind (resources, entitlements, grants) is determined by which +// RPC the annotation arrived on, never by the annotation itself. +// +// The connector must only emit this for a scope whose etag it received +// from the SDK's source-cache lookup during this same sync. A replay for +// an unknown scope fails the sync: the connector has already skipped row +// generation, so there is nothing to fall back to. +message SourceCacheReplay { + string scope_hash = 1; + + // Validator to persist for this scope in the current sync. For an HTTP + // 304 this is the unchanged etag. For a delta query this is the NEW + // token; it may instead be supplied by the final overlay page's + // SourceCacheScope.etag, in which case this field may be left empty. + string etag = 2; + + // When true, the response (and subsequent pages carrying + // SourceCacheScope with the same scope_hash) contains changed rows to + // upsert on top of the replayed base. When false the response must + // contain no rows for this scope. + bool overlay = 3; + + // Public canonical IDs (grant/entitlement IDs, or resource BIDs for + // RowKindResources) to delete from the current sync after the replay + // copy and this page's upserts. Used for delta-query tombstones (e.g. + // Microsoft Graph @removed entries). Subsequent pages of the round + // carry their tombstones on SourceCacheScope.deleted_ids. + repeated string deleted_ids = 4; + + // Principal-scoped tombstones for this page; see + // SourceCacheScope.deleted_principal_ids for semantics and + // preconditions. + repeated string deleted_principal_ids = 5; +} + +// ----------------------------------------------------------------------- +// Lookup continuation (ask/answer) — the source-cache lookup for runtimes +// that cannot call back to the syncer (single-shot request/response +// transports, e.g. gRPC-over-Lambda). See +// docs/tasks/source-cache-lambda-lookup.md. +// +// syncer connector +// ── list req + LookupOffer ───────▶ +// lookup deferred: collect scopes +// ◀── resp{SourceCacheLookupAsk} ─── +// resolve locally (previous c1z) +// ── SAME req + LookupOffer +// + SourceCacheLookupAnswers ─▶ +// lookup served from answers +// ◀── rows + Scope / Replay ──────── (normal page handling) +// +// The connector must only defer (emit an ask) when the request carried +// SourceCacheLookupOffer — old syncers never send it, so version skew +// degrades to a cold sync instead of a misinterpreted response. +// ----------------------------------------------------------------------- + +// SourceCacheLookupOffer is attached by the SDK to list REQUESTS when the +// syncer can answer source-cache lookup asks: the connector declared +// SourceCacheCapability and a warm previous-sync lookup is installed. +// Its presence is the connector's permission to answer with +// SourceCacheLookupAsk instead of rows. Connectors with a direct lookup +// (in-process, subprocess) never need to defer and may ignore it. +message SourceCacheLookupOffer {} + +// SourceCacheLookupAsk is attached to a list RESPONSE in place of rows: +// the connector needs previous-sync validators before it can serve the +// page. An ask response must carry NO rows, NO next page token, and no +// other source-cache annotations; the syncer consumes it (it never +// reaches page handling), resolves every query, and re-invokes the same +// request with SourceCacheLookupAnswers attached. +// +// Only legal on responses to requests that carried +// SourceCacheLookupOffer. Bounces are capped per REQUEST (a page-token +// advance is a new request and resets the cap, so a multi-page action +// may legally ask once per page); a connector that keeps asking without +// progressing past the cap fails the sync loudly. +message SourceCacheLookupAsk { + message Query { + // One of the sourcecache.RowKind values: "resources", + // "entitlements", "grants". + string row_kind = 1 [(validate.rules).string = { + min_bytes: 1 + max_bytes: 64 + }]; + string scope_hash = 2 [(validate.rules).string = { + min_bytes: 1 + max_bytes: 256 + }]; + } + repeated Query queries = 1 [(validate.rules).repeated = { + min_items: 1 + max_items: 4096 + }]; +} + +// SourceCacheLookupAnswers is attached by the SDK to the re-invoked +// REQUEST, carrying the resolution of a prior ask's queries. +// +// Every query of the prior ask is answered — found=true with the +// previous sync's etag, or found=false meaning fetch fresh. A found +// answer whose etag does not fit the per-request answer size budget is +// delivered as found=false: the scope degrades to a cold fetch +// (correct, just slower) instead of staying unresolved, because answers +// accumulate across bounces and an etag that did not fit once can never +// fit a later re-invoke. An ABSENT query (a phase-2 lookup for a scope +// the connector did not ask before) means unresolved: the connector may +// ask again, subject to the bounce cap. +message SourceCacheLookupAnswers { + message Answer { + string row_kind = 1 [(validate.rules).string = { + min_bytes: 1 + max_bytes: 64 + }]; + string scope_hash = 2 [(validate.rules).string = { + min_bytes: 1 + max_bytes: 256 + }]; + bool found = 3; + // The previous sync's validator; empty when found is false. Cap + // matches the lookup RPC (Graph delta tokens run long). + string etag = 4 [(validate.rules).string = { + ignore_empty: true + max_bytes: 65536 + }]; + } + repeated Answer answers = 1; +} diff --git a/proto/c1/connector/v2/annotation_type_scoped_grants.proto b/proto/c1/connector/v2/annotation_type_scoped_grants.proto new file mode 100644 index 000000000..f48dc747b --- /dev/null +++ b/proto/c1/connector/v2/annotation_type_scoped_grants.proto @@ -0,0 +1,87 @@ +syntax = "proto3"; + +package c1.connector.v2; + +import "validate/validate.proto"; + +option go_package = "github.com/conductorone/baton-sdk/pb/c1/connector/v2"; + +// Type-scoped grant ingestion. +// +// Some providers expose change/enumeration APIs whose natural unit is a +// whole collection (or a connector-defined shard of one), not a single +// resource — e.g. Microsoft Graph delta queries, where one stream yields +// membership changes for up to 50 groups. Forcing that shape through the +// per-resource ListGrants fan-out costs one request per resource even when +// nothing changed. +// +// A resource type annotated with TypeScopedGrants is EXCLUDED from the +// per-resource grants fan-out. Instead the syncer issues ListGrants calls +// whose resource carries ONLY the resource type (empty resource id); the +// connector answers with grants for the whole type, across as many +// paginated cursors as it chooses to spawn (see SpawnCursors). +// +// This is the grants-phase analogue of StaticEntitlements: the connector +// takes over enumeration for the type, and every downstream behavior +// (storage, source-cache scopes/replay/tombstones, grant expansion, rate +// limiting, page-token checkpointing) is unchanged because grants are +// self-describing rows. +// +// COMPLETENESS CONTRACT: a full sync of an annotated type must emit (or +// replay, via source-cache annotations) EVERY grant of that type. The +// syncer no longer visits each resource, so completeness rests entirely on +// the connector's enumeration. +message TypeScopedGrants {} + +// SpawnCursors is attached to a ListGrants response to enqueue additional +// independent sibling cursors. Each token is delivered back to the +// connector as the page token of its own action — scheduled by the +// syncer's worker pool, rate-limited, and checkpointed like any other +// pagination. Honored on BOTH type-scoped and per-resource ListGrants +// responses; the spawned actions inherit the response's resource identity +// (type only for type-scoped calls, type+resource for per-resource calls). +// +// Typical uses: +// +// - Type-scoped planning: the first call computes shard assignments — +// e.g. one 50-id delta filter chunk per cursor — returns no rows, and +// spawns one cursor per shard. Cold enumeration then parallelizes +// across shards instead of serializing through one stream. +// - Per-resource parallel warm revalidation (page-numbered APIs): on the +// first page of a collection the connector already knows every other +// page's URL and stored validator, so it answers page one and spawns +// pages 2..N; the worker pool revalidates them concurrently instead +// of paying one round trip per page serially. +// +// Spawned cursors are ORDINARY pages that happen to be enqueued eagerly. +// The SDK assumes nothing about how they resolve: a spawned page may hit +// its source-cache lookup and replay, miss and fetch cold (e.g. a page +// boundary shifted since the last sync), and may continue a chain via its +// response's next page token. Any page of a fan-out may itself spawn. +// +// Progress accounting counts a resource as covered when its ORIGIN +// action's chain ends; spawned siblings never double-count it. +// +// Tokens are opaque to the SDK. They must be self-contained: a cursor may +// execute after a suspend/resume, on a different worker, so anything the +// connector needs to serve the cursor must be inside the token (or +// re-derivable from it) — for per-resource spawns the resource identity +// rides the action, so tokens only need the page coordinate. +message SpawnCursors { + // Page tokens for the sibling cursors to enqueue, one action each. + // Capped per response (spawned actions persist in the checkpointed + // state token); chain additional spawns across pages to fan out wider. + repeated string page_tokens = 1 [(validate.rules).repeated = { + max_items: 1024 + items: { + string: { + min_len: 1 + max_len: 1048576 + } + } + }]; + + // Optional connector estimate of total grants across all cursors of this + // type, for progress reporting. Zero means unknown. + int64 estimated_total = 2; +} diff --git a/proto/c1/connector/v2/resource.proto b/proto/c1/connector/v2/resource.proto index 4a4cecd33..2b68838e4 100644 --- a/proto/c1/connector/v2/resource.proto +++ b/proto/c1/connector/v2/resource.proto @@ -41,7 +41,9 @@ message ResourceType { } repeated Trait traits = 3 [(validate.rules).repeated = { unique: true - items: {enum: {defined_only: true}} + items: { + enum: {defined_only: true} + } }]; repeated google.protobuf.Any annotations = 4; string description = 5 [(validate.rules).string = { diff --git a/proto/c1/connectorapi/baton/v1/source_cache.proto b/proto/c1/connectorapi/baton/v1/source_cache.proto new file mode 100644 index 000000000..5180996be --- /dev/null +++ b/proto/c1/connectorapi/baton/v1/source_cache.proto @@ -0,0 +1,55 @@ +syntax = "proto3"; + +package c1.connectorapi.baton.v1; + +import "validate/validate.proto"; + +option go_package = "gitlab.com/ductone/c1/pkg/pb/c1/connectorapi/baton/v1"; + +// BatonSourceCacheService is the dedicated parent-side RPC the connector +// subprocess uses to ask "do I have a previous validator (etag / delta +// token) for this scope?" before revalidating upstream. +// +// This is intentionally NOT routed through the session-store gRPC service: +// session data goes through the connector's local MemorySessionCache +// (otter), which would subject sync-scoped validator state to generic +// TTL/eviction policies and burn bounded cache weight on it. A dedicated +// service keeps the message shape explicit and the path uncached. +// +// The parent has exactly one active lookup registered at a time (set +// per-sync, cleared at sync end), so no sync_id travels on the wire. +service BatonSourceCacheService { + rpc Lookup(LookupRequest) returns (LookupResponse) {} +} + +message LookupRequest { + // Row kind: resources / entitlements / grants + // (pkg/sourcecache.RowKind values). Entries are partitioned by row + // kind, so one scope hash can carry a different validator per kind. + string row_kind = 1 [(validate.rules).string = { + min_len: 1 + max_len: 64 + }]; + + // Connector-defined stable scope identifier (conventionally a hex hash + // of the canonical scope; see pkg/sourcecache.HashScope). Opaque to the + // parent; matched verbatim against the previous sync's source-cache + // entries. + string scope_hash = 2 [(validate.rules).string = { + min_len: 1 + max_len: 256 + }]; +} + +message LookupResponse { + // False means no prior entry exists for (row_kind, scope_hash): the + // connector must fetch fresh and must not emit SourceCacheReplay for + // this scope. + bool found = 1; + + // The opaque validator the previous sync recorded for this scope (HTTP + // ETag, delta token, ...). Empty when found is false. The cap is a + // sanity bound sized for Microsoft Graph delta tokens, which are known + // to run to thousands of characters; storage imposes no limit. + string etag = 2 [(validate.rules).string = {max_len: 65536}]; +} diff --git a/proto/c1/storage/v3/records.proto b/proto/c1/storage/v3/records.proto index bdd167640..97d39b84e 100644 --- a/proto/c1/storage/v3/records.proto +++ b/proto/c1/storage/v3/records.proto @@ -103,6 +103,13 @@ message ResourceRecord { // into `annotations` as Any (the v2 connector contract — see // pkg/types/resource/user_trait.go). The annotations field above // round-trips them lossless; no extra field is needed. + + // Source-cache scope stamp (see annotation_source_cache.proto). + // Empty for rows not produced under a source-cache scope. + string source_scope_hash = 9 [(index) = { + name: "by_source_scope" + partial: "source_scope_hash != ''" + }]; } message EntitlementRecord { @@ -142,6 +149,13 @@ message EntitlementRecord { // resource_types table when needed. Consumed by // pkg/sync/syncer.go's principal-type narrowing. repeated string grantable_to_resource_type_ids = 10; + + // Source-cache scope stamp (see annotation_source_cache.proto). + // Empty for rows not produced under a source-cache scope. + string source_scope_hash = 11 [(index) = { + name: "by_source_scope" + partial: "source_scope_hash != ''" + }]; } message GrantRecord { @@ -188,6 +202,16 @@ message GrantRecord { // map — same wire shape as // c1.connector.v2.GrantSources.sources. map sources = 9; + + // Source-cache scope stamp (see annotation_source_cache.proto). + // Empty for rows not produced under a source-cache scope — notably + // expander-derived grants, which are recreated by expansion each sync + // and never replayed. StoreExpandedGrants preserves this field on + // rewrites of existing rows, exactly like expansion/needs_expansion. + string source_scope_hash = 10 [(index) = { + name: "by_source_scope" + partial: "source_scope_hash != ''" + }]; } message AssetRecord { @@ -220,6 +244,17 @@ message SyncRunRecord { string sync_token = 6; bool supports_diff = 7; string linked_sync_id = 8; + + // compacted marks a sync produced by compaction (fold or rebuild) + // rather than by a real connector run. Compacted artifacts are + // keep-newer UPSERT merges — base rows a newer input deleted survive — + // so no input sync's upstream validators (source-cache etags) describe + // their contents. Consequence, enforced by the syncer: a compacted + // sync CANNOT be used as a source-cache replay source; feeding one to + // WithPreviousSyncC1ZPath degrades to a cold (full-fetch) sync. + // Orchestrators deciding which artifact to materialize as the previous + // sync should read this flag instead of guessing from provenance. + bool compacted = 9; } // SyncStatsRecord is the engine-internal sidecar populated at @@ -250,10 +285,41 @@ message SyncStatsRecord { message SessionRecord { option (table) = { name: "sessions" - primary_key: ["sync_id", "key"] + primary_key: [ + "sync_id", + "key" + ] }; string sync_id = 1; string key = 2; bytes value = 3; } + +// SourceCacheEntryRecord is the per-scope manifest entry for source-cache +// replay (see c1/connector/v2/annotation_source_cache.proto). One entry +// per (row_kind, scope_hash), written for every freshly fetched scope — +// including zero-row responses — and rewritten on replay with the scope's +// current validator. The previous sync's entries (read from the previous +// c1z) are the lookup surface connectors revalidate against. +message SourceCacheEntryRecord { + option (table) = { + name: "source_cache_entries" + primary_key: [ + "row_kind", + "scope_hash" + ] + }; + + // Row kind partition: "resources", "entitlements", or "grants" + // (pkg/sourcecache.RowKind values). + string row_kind = 1; + + // Connector-computed scope hash (lowercase hex). + string scope_hash = 2; + + // Opaque upstream validator: HTTP ETag, delta token, etc. + string etag = 3; + + google.protobuf.Timestamp discovered_at = 4; +} From 4d2e51a3851a2e3d65193475c5c0bb61a05feb97 Mon Sep 17 00:00:00 2001 From: Matt Kaniaris Date: Tue, 14 Jul 2026 10:57:22 -0600 Subject: [PATCH 02/17] explicit interfaces in compacted artifacts for the metadata concerning cache invalidation --- pb/c1/storage/v3/records.pb.go | 28 ++++++---- pb/c1/storage/v3/records_protoopaque.pb.go | 14 +++-- pkg/dotc1z/c1zstore/sync_meta.go | 23 ++++++++- pkg/sync/source_cache.go | 46 ++++++++++------- pkg/sync/source_cache_sideeffects_test.go | 59 ++++++++++++++++++++++ proto/c1/storage/v3/records.proto | 14 +++-- 6 files changed, 144 insertions(+), 40 deletions(-) diff --git a/pb/c1/storage/v3/records.pb.go b/pb/c1/storage/v3/records.pb.go index 90030dbe4..e02826a96 100644 --- a/pb/c1/storage/v3/records.pb.go +++ b/pb/c1/storage/v3/records.pb.go @@ -1250,11 +1250,15 @@ type SyncRunRecord struct { // rather than by a real connector run. Compacted artifacts are // keep-newer UPSERT merges — base rows a newer input deleted survive — // so no input sync's upstream validators (source-cache etags) describe - // their contents. Consequence, enforced by the syncer: a compacted - // sync CANNOT be used as a source-cache replay source; feeding one to - // WithPreviousSyncC1ZPath degrades to a cold (full-fetch) sync. - // Orchestrators deciding which artifact to materialize as the previous - // sync should read this flag instead of guessing from provenance. + // their contents. + // + // "Can this sync be used as a source-cache replay source?" is the + // predicate c1zstore.SyncRun.UsableAsReplaySource: type == FULL and + // !compacted. The syncer enforces it — feeding a compacted or + // partial/derived sync to WithPreviousSyncC1ZPath degrades to a cold + // (full-fetch) sync. Orchestrators deciding which artifact to + // materialize as the previous sync should apply the same predicate + // instead of guessing from provenance. Compacted bool `protobuf:"varint,9,opt,name=compacted,proto3" json:"compacted,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache @@ -1421,11 +1425,15 @@ type SyncRunRecord_builder struct { // rather than by a real connector run. Compacted artifacts are // keep-newer UPSERT merges — base rows a newer input deleted survive — // so no input sync's upstream validators (source-cache etags) describe - // their contents. Consequence, enforced by the syncer: a compacted - // sync CANNOT be used as a source-cache replay source; feeding one to - // WithPreviousSyncC1ZPath degrades to a cold (full-fetch) sync. - // Orchestrators deciding which artifact to materialize as the previous - // sync should read this flag instead of guessing from provenance. + // their contents. + // + // "Can this sync be used as a source-cache replay source?" is the + // predicate c1zstore.SyncRun.UsableAsReplaySource: type == FULL and + // !compacted. The syncer enforces it — feeding a compacted or + // partial/derived sync to WithPreviousSyncC1ZPath degrades to a cold + // (full-fetch) sync. Orchestrators deciding which artifact to + // materialize as the previous sync should apply the same predicate + // instead of guessing from provenance. Compacted bool } diff --git a/pb/c1/storage/v3/records_protoopaque.pb.go b/pb/c1/storage/v3/records_protoopaque.pb.go index 4830bbcfc..21f9567b2 100644 --- a/pb/c1/storage/v3/records_protoopaque.pb.go +++ b/pb/c1/storage/v3/records_protoopaque.pb.go @@ -1378,11 +1378,15 @@ type SyncRunRecord_builder struct { // rather than by a real connector run. Compacted artifacts are // keep-newer UPSERT merges — base rows a newer input deleted survive — // so no input sync's upstream validators (source-cache etags) describe - // their contents. Consequence, enforced by the syncer: a compacted - // sync CANNOT be used as a source-cache replay source; feeding one to - // WithPreviousSyncC1ZPath degrades to a cold (full-fetch) sync. - // Orchestrators deciding which artifact to materialize as the previous - // sync should read this flag instead of guessing from provenance. + // their contents. + // + // "Can this sync be used as a source-cache replay source?" is the + // predicate c1zstore.SyncRun.UsableAsReplaySource: type == FULL and + // !compacted. The syncer enforces it — feeding a compacted or + // partial/derived sync to WithPreviousSyncC1ZPath degrades to a cold + // (full-fetch) sync. Orchestrators deciding which artifact to + // materialize as the previous sync should apply the same predicate + // instead of guessing from provenance. Compacted bool } diff --git a/pkg/dotc1z/c1zstore/sync_meta.go b/pkg/dotc1z/c1zstore/sync_meta.go index 2904bbd16..174fd168a 100644 --- a/pkg/dotc1z/c1zstore/sync_meta.go +++ b/pkg/dotc1z/c1zstore/sync_meta.go @@ -62,9 +62,30 @@ type SyncRun struct { // no input's source-cache validators (etags) describe their contents: // the syncer refuses to use a compacted sync as a replay source // (degrades to a cold sync), and orchestrators choosing which - // artifact to materialize as the previous sync should read this flag. + // artifact to materialize as the previous sync should read + // UsableAsReplaySource instead of guessing from provenance. // Always false on the SQLite engine (v1 files cannot be replay // sources regardless). Compacted bool Stats *reader_v2.SyncStats } + +// UsableAsReplaySource answers "can this sync serve source-cache replay?" +// — the question an orchestrator must ask before materializing an +// artifact as a syncer's previous sync. The predicate is deliberately +// narrow: only a FULL, non-compacted sync is a faithful snapshot that its +// own recorded validators describe. +// +// - Partial/targeted, resources-only, and diff-typed syncs are subsets +// or derivations — their rows do not cover the scopes any validator +// would vouch for (they also never write manifest entries, but the +// type gate makes the refusal explicit rather than incidental). +// - Compacted syncs are keep-newer merges of multiple runs; no single +// input's validators describe the merged row set. +// +// The syncer enforces this same predicate when a previous-sync c1z is +// supplied (see pkg/sync's configureSourceCache): a non-qualifying file +// degrades to a cold sync rather than replaying. +func (s *SyncRun) UsableAsReplaySource() bool { + return s != nil && s.Type == connectorstore.SyncTypeFull && !s.Compacted +} diff --git a/pkg/sync/source_cache.go b/pkg/sync/source_cache.go index adf2ee880..27c44ab44 100644 --- a/pkg/sync/source_cache.go +++ b/pkg/sync/source_cache.go @@ -168,17 +168,18 @@ func (s *syncer) configureSourceCache(ctx context.Context, resp *v2.ConnectorSer readReason = "no previous-sync c1z configured" } else if prev, ok := s.previousSyncReader.(dotc1z.SourceCacheStore); !ok { readReason = "previous-sync store engine does not support source cache" - } else if compacted, err := previousSyncIsCompacted(ctx, s.previousSyncReader); err != nil { + } else if reason, err := previousSyncReplayUnusableReason(ctx, s.previousSyncReader); err != nil { return fmt.Errorf("error reading previous sync's run metadata: %w", err) - } else if compacted { + } else if reason != "" { // The explicit metadata gate, independent of the manifest's - // contents: a compacted artifact is a keep-newer merge, so no - // input sync's upstream validators describe it — a 304 against a - // carried etag would replay a row set the provider never vouched - // for. Compaction also clears the manifest (belt), but the flag - // is the contract (suspenders): "did this sync go through - // compaction" answers "can it be used for replay" with NO. - readReason = "previous sync is a compaction artifact; upstream validators do not describe its merged contents" + // contents (c1zstore.SyncRun.UsableAsReplaySource): compacted + // artifacts are keep-newer merges no input's validators describe, + // and partial/derived syncs are subsets whose rows do not cover + // the scopes any validator vouches for. Those paths also leave no + // manifest to hit (belt); the run-record gate is the contract + // (suspenders) — "what kind of sync is this" answers "can it be + // used for replay" without inspecting keyspaces. + readReason = reason } else { s.sourceCache.prev = prev s.sourceCache.prevReader = s.previousSyncReader @@ -204,24 +205,31 @@ func (s *syncer) configureSourceCache(ctx context.Context, resp *v2.ConnectorSer return nil } -// previousSyncIsCompacted reports whether the previous-sync reader's -// newest finished sync carries the compaction provenance flag -// (SyncRunRecord.compacted). A missing SyncMeta surface or absent run -// record degrades to "not compacted" — the manifest-based guards still +// previousSyncReplayUnusableReason inspects the previous-sync reader's +// newest finished run record against the replay-source predicate +// (c1zstore.SyncRun.UsableAsReplaySource) and returns a human-readable +// reason when it fails, or "" when the sync qualifies. A reader without a +// SyncMeta surface degrades to "" — the manifest-based guards still // apply — but a readable record is authoritative. -func previousSyncIsCompacted(ctx context.Context, reader connectorstore.Reader) (bool, error) { +func previousSyncReplayUnusableReason(ctx context.Context, reader connectorstore.Reader) (string, error) { metaHolder, ok := reader.(interface{ SyncMeta() c1zstore.SyncMeta }) if !ok { - return false, nil + return "", nil } run, err := metaHolder.SyncMeta().LatestFinishedSyncOfAnyType(ctx) if err != nil { - return false, err + return "", err } - if run == nil { - return false, nil + switch { + case run == nil: + return "previous-sync c1z has no finished sync run", nil + case run.UsableAsReplaySource(): + return "", nil + case run.Compacted: + return "previous sync is a compaction artifact; upstream validators do not describe its merged contents", nil + default: + return fmt.Sprintf("previous sync is type %q, not a full sync; its rows do not cover the scopes any validator vouches for", run.Type), nil } - return run.Compacted, nil } // clearSourceCacheLookup detaches the per-sync lookup so a late RPC from diff --git a/pkg/sync/source_cache_sideeffects_test.go b/pkg/sync/source_cache_sideeffects_test.go index 6ab04bf9b..811befa6f 100644 --- a/pkg/sync/source_cache_sideeffects_test.go +++ b/pkg/sync/source_cache_sideeffects_test.go @@ -17,6 +17,7 @@ import ( v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" "github.com/conductorone/baton-sdk/pkg/annotations" + "github.com/conductorone/baton-sdk/pkg/connectorstore" "github.com/conductorone/baton-sdk/pkg/dotc1z" "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" pebbleengine "github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble" @@ -613,6 +614,64 @@ func TestSourceCache_PartialSyncDegradesToCold(t *testing.T) { "sanity: the connector must have looked up and missed (no-op lookup installed)") } +// --------------------------------------------------------------------- +// Non-full previous syncs must not replay. +// --------------------------------------------------------------------- + +// TestSourceCache_PartialPreviousSyncDegradesToCold pins the type half of +// the replay-source predicate (UsableAsReplaySource): a PARTIAL-typed +// previous sync must never serve replay, even when its file carries a +// manifest entry over stamped rows. The fixture plants exactly that — +// a partial sync with a valid-looking manifest — so a pass proves the +// run-record TYPE gate causes the degradation, not an incidentally empty +// manifest. Partial syncs are subsets: their rows do not cover the +// scopes any validator vouches for. +func TestSourceCache_PartialPreviousSyncDegradesToCold(t *testing.T) { + ctx, err := logging.Init(t.Context()) + require.NoError(t, err) + tmpDir := t.TempDir() + + group, ent, alice, _ := sourceCacheTestFixtures(t) + g1 := gt.NewGrant(group, "member", alice) + grantsScope := grantScopeForResource(group.GetId().GetResource()) + + // Hand-build a PARTIAL sync whose grants are stamped with the scope + // the connector will look up, manifest entry included. + sync1 := filepath.Join(tmpDir, "partial-prev.c1z") + prevStore, err := dotc1z.NewStore(ctx, sync1, + dotc1z.WithEngine(c1zstore.EnginePebble), + dotc1z.WithTmpDir(tmpDir), + ) + require.NoError(t, err) + _, isNew, err := prevStore.StartOrResumeSync(ctx, connectorstore.SyncTypePartial, "") + require.NoError(t, err) + require.True(t, isNew) + require.NoError(t, prevStore.PutResourceTypes(ctx, groupResourceType, userResourceType)) + require.NoError(t, prevStore.PutResources(ctx, group, alice)) + require.NoError(t, prevStore.PutEntitlements(ctx, ent)) + require.NoError(t, prevStore.PutGrants(sourcecache.WithScope(ctx, grantsScope), g1)) + scPrev, ok := any(prevStore).(dotc1z.SourceCacheStore) + require.True(t, ok) + require.NoError(t, scPrev.PutSourceCacheEntry(ctx, sourcecache.RowKindGrants, grantsScope, `W/"etag-v1"`)) + require.NoError(t, prevStore.EndSync(ctx)) + require.NoError(t, prevStore.Close(ctx)) + + mc := newSourceCacheMockConnector() + mc.AddResource(ctx, group) + mc.AddResource(ctx, alice) + mc.entDB[group.GetId().GetResource()] = []*v2.Entitlement{ent} + mc.grantDB[group.GetId().GetResource()] = []*v2.Grant{g1} + mc.etagByResource[group.GetId().GetResource()] = `W/"etag-v1"` + + sync2 := filepath.Join(tmpDir, "sync2.c1z") + missesBefore := mc.lookupMisses + runSourceCacheSync(ctx, t, mc, sync2, sync1, tmpDir) + require.Zero(t, mc.lookupHits, + "a partial-typed previous sync must never produce a source-cache lookup hit, even with a planted manifest entry") + require.Greater(t, mc.lookupMisses, missesBefore, + "sanity: the connector must have looked up and missed (no-op lookup installed)") +} + // --------------------------------------------------------------------- // Compacted previous syncs must not replay. // --------------------------------------------------------------------- diff --git a/proto/c1/storage/v3/records.proto b/proto/c1/storage/v3/records.proto index 97d39b84e..fe87a5b05 100644 --- a/proto/c1/storage/v3/records.proto +++ b/proto/c1/storage/v3/records.proto @@ -249,11 +249,15 @@ message SyncRunRecord { // rather than by a real connector run. Compacted artifacts are // keep-newer UPSERT merges — base rows a newer input deleted survive — // so no input sync's upstream validators (source-cache etags) describe - // their contents. Consequence, enforced by the syncer: a compacted - // sync CANNOT be used as a source-cache replay source; feeding one to - // WithPreviousSyncC1ZPath degrades to a cold (full-fetch) sync. - // Orchestrators deciding which artifact to materialize as the previous - // sync should read this flag instead of guessing from provenance. + // their contents. + // + // "Can this sync be used as a source-cache replay source?" is the + // predicate c1zstore.SyncRun.UsableAsReplaySource: type == FULL and + // !compacted. The syncer enforces it — feeding a compacted or + // partial/derived sync to WithPreviousSyncC1ZPath degrades to a cold + // (full-fetch) sync. Orchestrators deciding which artifact to + // materialize as the previous sync should apply the same predicate + // instead of guessing from provenance. bool compacted = 9; } From 158c64c85ecd77f01dde9effa6784856730fbcb8 Mon Sep 17 00:00:00 2001 From: Matt Kaniaris Date: Tue, 14 Jul 2026 14:17:36 -0600 Subject: [PATCH 03/17] type scoped entitlements, and localized metamorphic sync invariants --- .../annotation_type_scoped_entitlements.pb.go | 159 ++++++ ...on_type_scoped_entitlements.pb.validate.go | 138 +++++ ...type_scoped_entitlements_protoopaque.pb.go | 159 ++++++ .../v2/annotation_type_scoped_grants.pb.go | 35 +- ...ation_type_scoped_grants_protoopaque.pb.go | 33 +- pkg/connectorbuilder/resource_syncer.go | 49 +- .../type_scoped_entitlements_test.go | 122 ++++ pkg/dotc1z/engine/pebble/cleanup.go | 5 + pkg/dotc1z/engine/pebble/engine.go | 12 + pkg/dotc1z/engine/pebble/grants.go | 5 + pkg/dotc1z/engine/pebble/ingest_facts.go | 207 +++++++ pkg/dotc1z/engine/pebble/raw_records.go | 41 -- pkg/dotc1z/engine/pebble/source_cache.go | 60 +- .../engine/pebble/source_cache_engine_test.go | 5 +- pkg/dotc1z/source_cache.go | 43 ++ pkg/sync/ingest_invariants.go | 338 ++++++++++++ pkg/sync/ingest_invariants_test.go | 37 ++ pkg/sync/progresslog/progresslog.go | 70 ++- pkg/sync/source_cache.go | 43 +- pkg/sync/source_cache_continuation_test.go | 142 +++++ pkg/sync/source_cache_equivalence_test.go | 80 ++- pkg/sync/source_cache_sideeffects_test.go | 2 +- pkg/sync/source_cache_test.go | 2 +- pkg/sync/state.go | 9 +- pkg/sync/syncer.go | 371 +++++++++---- pkg/sync/syncer_test.go | 78 ++- pkg/sync/type_scoped_entitlements_test.go | 520 ++++++++++++++++++ .../annotation_type_scoped_entitlements.proto | 57 ++ .../v2/annotation_type_scoped_grants.proto | 33 +- 29 files changed, 2507 insertions(+), 348 deletions(-) create mode 100644 pb/c1/connector/v2/annotation_type_scoped_entitlements.pb.go create mode 100644 pb/c1/connector/v2/annotation_type_scoped_entitlements.pb.validate.go create mode 100644 pb/c1/connector/v2/annotation_type_scoped_entitlements_protoopaque.pb.go create mode 100644 pkg/connectorbuilder/type_scoped_entitlements_test.go create mode 100644 pkg/dotc1z/engine/pebble/ingest_facts.go create mode 100644 pkg/sync/ingest_invariants.go create mode 100644 pkg/sync/ingest_invariants_test.go create mode 100644 pkg/sync/type_scoped_entitlements_test.go create mode 100644 proto/c1/connector/v2/annotation_type_scoped_entitlements.proto diff --git a/pb/c1/connector/v2/annotation_type_scoped_entitlements.pb.go b/pb/c1/connector/v2/annotation_type_scoped_entitlements.pb.go new file mode 100644 index 000000000..8507dd430 --- /dev/null +++ b/pb/c1/connector/v2/annotation_type_scoped_entitlements.pb.go @@ -0,0 +1,159 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.10 +// protoc (unknown) +// source: c1/connector/v2/annotation_type_scoped_entitlements.proto + +//go:build !protoopaque + +package v2 + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// Type-scoped entitlement ingestion. +// +// Some providers expose change/enumeration APIs whose natural unit is a +// whole collection (or a connector-defined shard of one), not a single +// resource — e.g. Microsoft Graph delta queries, where one stream yields +// entitlement definitions for up to 50 groups. Forcing that shape through +// the per-resource ListEntitlements fan-out costs one request per resource +// even when nothing changed. +// +// A resource type annotated with TypeScopedEntitlements is EXCLUDED from +// the per-resource entitlements fan-out. Instead the syncer issues +// ListEntitlements calls whose resource carries ONLY the resource type +// (empty resource id); the connector answers with entitlements for the +// whole type, across as many paginated cursors as it chooses to spawn +// (see SpawnCursors in annotation_type_scoped_grants.proto). +// +// This is the entitlements-phase analogue of TypeScopedGrants: the +// connector takes over enumeration for the type, and every downstream +// behavior (storage, source-cache scopes/replay/tombstones, exclusion- +// group validation, rate limiting, page-token checkpointing) is unchanged +// because entitlements are self-describing rows. +// +// COMPLETENESS CONTRACT: a full sync of an annotated type must emit (or +// replay, via source-cache annotations) EVERY entitlement of that type. +// The syncer no longer visits each resource, so completeness rests +// entirely on the connector's enumeration. +// +// SKIP ANNOTATIONS: for an annotated type the syncer does not consult +// shouldSkipEntitlements on the type-scoped path. The connector owns skip +// semantics for resources of that type (omit rows; do not rely on +// per-resource SkipEntitlements* annotations to suppress the type-scoped +// call). +// +// ZERO-RESOURCE TYPES: the planner still enqueues the type-scoped action +// when the store holds no resources of the type. The connector must +// return an empty page (or SpawnCursors that resolve empty), not an +// error — an empty type is a valid outcome. +// +// TARGETED SYNC: SyncTargetedResource must not enqueue a per-resource +// SyncEntitlementsOp for a type-scoped type. The type-scoped planner +// action is the only legal entitlements path; a targeted sync of one +// resource of an annotated type syncs the resource (and children) but +// leaves type-scoped entitlement enumeration to a full entitlements +// phase. +// +// ROUTING MARKER ONLY: TypeScopedEntitlements is not a stream-coupled +// side-effect annotation. It is absent from the ingestion-invariant +// side-effect coverage map for the same reason as TypeScopedGrants — +// it routes the ListEntitlements call; it does not imply post-ingest +// repair or validation beyond ordinary entitlement handling. +type TypeScopedEntitlements struct { + state protoimpl.MessageState `protogen:"hybrid.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *TypeScopedEntitlements) Reset() { + *x = TypeScopedEntitlements{} + mi := &file_c1_connector_v2_annotation_type_scoped_entitlements_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *TypeScopedEntitlements) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*TypeScopedEntitlements) ProtoMessage() {} + +func (x *TypeScopedEntitlements) ProtoReflect() protoreflect.Message { + mi := &file_c1_connector_v2_annotation_type_scoped_entitlements_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +type TypeScopedEntitlements_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + +} + +func (b0 TypeScopedEntitlements_builder) Build() *TypeScopedEntitlements { + m0 := &TypeScopedEntitlements{} + b, x := &b0, m0 + _, _ = b, x + return m0 +} + +var File_c1_connector_v2_annotation_type_scoped_entitlements_proto protoreflect.FileDescriptor + +const file_c1_connector_v2_annotation_type_scoped_entitlements_proto_rawDesc = "" + + "\n" + + "9c1/connector/v2/annotation_type_scoped_entitlements.proto\x12\x0fc1.connector.v2\"\x18\n" + + "\x16TypeScopedEntitlementsB6Z4github.com/conductorone/baton-sdk/pb/c1/connector/v2b\x06proto3" + +var file_c1_connector_v2_annotation_type_scoped_entitlements_proto_msgTypes = make([]protoimpl.MessageInfo, 1) +var file_c1_connector_v2_annotation_type_scoped_entitlements_proto_goTypes = []any{ + (*TypeScopedEntitlements)(nil), // 0: c1.connector.v2.TypeScopedEntitlements +} +var file_c1_connector_v2_annotation_type_scoped_entitlements_proto_depIdxs = []int32{ + 0, // [0:0] is the sub-list for method output_type + 0, // [0:0] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_c1_connector_v2_annotation_type_scoped_entitlements_proto_init() } +func file_c1_connector_v2_annotation_type_scoped_entitlements_proto_init() { + if File_c1_connector_v2_annotation_type_scoped_entitlements_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_c1_connector_v2_annotation_type_scoped_entitlements_proto_rawDesc), len(file_c1_connector_v2_annotation_type_scoped_entitlements_proto_rawDesc)), + NumEnums: 0, + NumMessages: 1, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_c1_connector_v2_annotation_type_scoped_entitlements_proto_goTypes, + DependencyIndexes: file_c1_connector_v2_annotation_type_scoped_entitlements_proto_depIdxs, + MessageInfos: file_c1_connector_v2_annotation_type_scoped_entitlements_proto_msgTypes, + }.Build() + File_c1_connector_v2_annotation_type_scoped_entitlements_proto = out.File + file_c1_connector_v2_annotation_type_scoped_entitlements_proto_goTypes = nil + file_c1_connector_v2_annotation_type_scoped_entitlements_proto_depIdxs = nil +} diff --git a/pb/c1/connector/v2/annotation_type_scoped_entitlements.pb.validate.go b/pb/c1/connector/v2/annotation_type_scoped_entitlements.pb.validate.go new file mode 100644 index 000000000..cbd3b3169 --- /dev/null +++ b/pb/c1/connector/v2/annotation_type_scoped_entitlements.pb.validate.go @@ -0,0 +1,138 @@ +// Code generated by protoc-gen-validate. DO NOT EDIT. +// source: c1/connector/v2/annotation_type_scoped_entitlements.proto + +package v2 + +import ( + "bytes" + "errors" + "fmt" + "net" + "net/mail" + "net/url" + "regexp" + "sort" + "strings" + "time" + "unicode/utf8" + + "google.golang.org/protobuf/types/known/anypb" +) + +// ensure the imports are used +var ( + _ = bytes.MinRead + _ = errors.New("") + _ = fmt.Print + _ = utf8.UTFMax + _ = (*regexp.Regexp)(nil) + _ = (*strings.Reader)(nil) + _ = net.IPv4len + _ = time.Duration(0) + _ = (*url.URL)(nil) + _ = (*mail.Address)(nil) + _ = anypb.Any{} + _ = sort.Sort +) + +// Validate checks the field values on TypeScopedEntitlements with the rules +// defined in the proto definition for this message. If any rules are +// violated, the first error encountered is returned, or nil if there are no violations. +func (m *TypeScopedEntitlements) Validate() error { + return m.validate(false) +} + +// ValidateAll checks the field values on TypeScopedEntitlements with the rules +// defined in the proto definition for this message. If any rules are +// violated, the result is a list of violation errors wrapped in +// TypeScopedEntitlementsMultiError, or nil if none found. +func (m *TypeScopedEntitlements) ValidateAll() error { + return m.validate(true) +} + +func (m *TypeScopedEntitlements) validate(all bool) error { + if m == nil { + return nil + } + + var errors []error + + if len(errors) > 0 { + return TypeScopedEntitlementsMultiError(errors) + } + + return nil +} + +// TypeScopedEntitlementsMultiError is an error wrapping multiple validation +// errors returned by TypeScopedEntitlements.ValidateAll() if the designated +// constraints aren't met. +type TypeScopedEntitlementsMultiError []error + +// Error returns a concatenation of all the error messages it wraps. +func (m TypeScopedEntitlementsMultiError) Error() string { + msgs := make([]string, 0, len(m)) + for _, err := range m { + msgs = append(msgs, err.Error()) + } + return strings.Join(msgs, "; ") +} + +// AllErrors returns a list of validation violation errors. +func (m TypeScopedEntitlementsMultiError) AllErrors() []error { return m } + +// TypeScopedEntitlementsValidationError is the validation error returned by +// TypeScopedEntitlements.Validate if the designated constraints aren't met. +type TypeScopedEntitlementsValidationError struct { + field string + reason string + cause error + key bool +} + +// Field function returns field value. +func (e TypeScopedEntitlementsValidationError) Field() string { return e.field } + +// Reason function returns reason value. +func (e TypeScopedEntitlementsValidationError) Reason() string { return e.reason } + +// Cause function returns cause value. +func (e TypeScopedEntitlementsValidationError) Cause() error { return e.cause } + +// Key function returns key value. +func (e TypeScopedEntitlementsValidationError) Key() bool { return e.key } + +// ErrorName returns error name. +func (e TypeScopedEntitlementsValidationError) ErrorName() string { + return "TypeScopedEntitlementsValidationError" +} + +// Error satisfies the builtin error interface +func (e TypeScopedEntitlementsValidationError) Error() string { + cause := "" + if e.cause != nil { + cause = fmt.Sprintf(" | caused by: %v", e.cause) + } + + key := "" + if e.key { + key = "key for " + } + + return fmt.Sprintf( + "invalid %sTypeScopedEntitlements.%s: %s%s", + key, + e.field, + e.reason, + cause) +} + +var _ error = TypeScopedEntitlementsValidationError{} + +var _ interface { + Field() string + Reason() string + Key() bool + Cause() error + ErrorName() string +} = TypeScopedEntitlementsValidationError{} diff --git a/pb/c1/connector/v2/annotation_type_scoped_entitlements_protoopaque.pb.go b/pb/c1/connector/v2/annotation_type_scoped_entitlements_protoopaque.pb.go new file mode 100644 index 000000000..29950644f --- /dev/null +++ b/pb/c1/connector/v2/annotation_type_scoped_entitlements_protoopaque.pb.go @@ -0,0 +1,159 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.10 +// protoc (unknown) +// source: c1/connector/v2/annotation_type_scoped_entitlements.proto + +//go:build protoopaque + +package v2 + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +// Type-scoped entitlement ingestion. +// +// Some providers expose change/enumeration APIs whose natural unit is a +// whole collection (or a connector-defined shard of one), not a single +// resource — e.g. Microsoft Graph delta queries, where one stream yields +// entitlement definitions for up to 50 groups. Forcing that shape through +// the per-resource ListEntitlements fan-out costs one request per resource +// even when nothing changed. +// +// A resource type annotated with TypeScopedEntitlements is EXCLUDED from +// the per-resource entitlements fan-out. Instead the syncer issues +// ListEntitlements calls whose resource carries ONLY the resource type +// (empty resource id); the connector answers with entitlements for the +// whole type, across as many paginated cursors as it chooses to spawn +// (see SpawnCursors in annotation_type_scoped_grants.proto). +// +// This is the entitlements-phase analogue of TypeScopedGrants: the +// connector takes over enumeration for the type, and every downstream +// behavior (storage, source-cache scopes/replay/tombstones, exclusion- +// group validation, rate limiting, page-token checkpointing) is unchanged +// because entitlements are self-describing rows. +// +// COMPLETENESS CONTRACT: a full sync of an annotated type must emit (or +// replay, via source-cache annotations) EVERY entitlement of that type. +// The syncer no longer visits each resource, so completeness rests +// entirely on the connector's enumeration. +// +// SKIP ANNOTATIONS: for an annotated type the syncer does not consult +// shouldSkipEntitlements on the type-scoped path. The connector owns skip +// semantics for resources of that type (omit rows; do not rely on +// per-resource SkipEntitlements* annotations to suppress the type-scoped +// call). +// +// ZERO-RESOURCE TYPES: the planner still enqueues the type-scoped action +// when the store holds no resources of the type. The connector must +// return an empty page (or SpawnCursors that resolve empty), not an +// error — an empty type is a valid outcome. +// +// TARGETED SYNC: SyncTargetedResource must not enqueue a per-resource +// SyncEntitlementsOp for a type-scoped type. The type-scoped planner +// action is the only legal entitlements path; a targeted sync of one +// resource of an annotated type syncs the resource (and children) but +// leaves type-scoped entitlement enumeration to a full entitlements +// phase. +// +// ROUTING MARKER ONLY: TypeScopedEntitlements is not a stream-coupled +// side-effect annotation. It is absent from the ingestion-invariant +// side-effect coverage map for the same reason as TypeScopedGrants — +// it routes the ListEntitlements call; it does not imply post-ingest +// repair or validation beyond ordinary entitlement handling. +type TypeScopedEntitlements struct { + state protoimpl.MessageState `protogen:"opaque.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *TypeScopedEntitlements) Reset() { + *x = TypeScopedEntitlements{} + mi := &file_c1_connector_v2_annotation_type_scoped_entitlements_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *TypeScopedEntitlements) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*TypeScopedEntitlements) ProtoMessage() {} + +func (x *TypeScopedEntitlements) ProtoReflect() protoreflect.Message { + mi := &file_c1_connector_v2_annotation_type_scoped_entitlements_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +type TypeScopedEntitlements_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + +} + +func (b0 TypeScopedEntitlements_builder) Build() *TypeScopedEntitlements { + m0 := &TypeScopedEntitlements{} + b, x := &b0, m0 + _, _ = b, x + return m0 +} + +var File_c1_connector_v2_annotation_type_scoped_entitlements_proto protoreflect.FileDescriptor + +const file_c1_connector_v2_annotation_type_scoped_entitlements_proto_rawDesc = "" + + "\n" + + "9c1/connector/v2/annotation_type_scoped_entitlements.proto\x12\x0fc1.connector.v2\"\x18\n" + + "\x16TypeScopedEntitlementsB6Z4github.com/conductorone/baton-sdk/pb/c1/connector/v2b\x06proto3" + +var file_c1_connector_v2_annotation_type_scoped_entitlements_proto_msgTypes = make([]protoimpl.MessageInfo, 1) +var file_c1_connector_v2_annotation_type_scoped_entitlements_proto_goTypes = []any{ + (*TypeScopedEntitlements)(nil), // 0: c1.connector.v2.TypeScopedEntitlements +} +var file_c1_connector_v2_annotation_type_scoped_entitlements_proto_depIdxs = []int32{ + 0, // [0:0] is the sub-list for method output_type + 0, // [0:0] is the sub-list for method input_type + 0, // [0:0] is the sub-list for extension type_name + 0, // [0:0] is the sub-list for extension extendee + 0, // [0:0] is the sub-list for field type_name +} + +func init() { file_c1_connector_v2_annotation_type_scoped_entitlements_proto_init() } +func file_c1_connector_v2_annotation_type_scoped_entitlements_proto_init() { + if File_c1_connector_v2_annotation_type_scoped_entitlements_proto != nil { + return + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_c1_connector_v2_annotation_type_scoped_entitlements_proto_rawDesc), len(file_c1_connector_v2_annotation_type_scoped_entitlements_proto_rawDesc)), + NumEnums: 0, + NumMessages: 1, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_c1_connector_v2_annotation_type_scoped_entitlements_proto_goTypes, + DependencyIndexes: file_c1_connector_v2_annotation_type_scoped_entitlements_proto_depIdxs, + MessageInfos: file_c1_connector_v2_annotation_type_scoped_entitlements_proto_msgTypes, + }.Build() + File_c1_connector_v2_annotation_type_scoped_entitlements_proto = out.File + file_c1_connector_v2_annotation_type_scoped_entitlements_proto_goTypes = nil + file_c1_connector_v2_annotation_type_scoped_entitlements_proto_depIdxs = nil +} diff --git a/pb/c1/connector/v2/annotation_type_scoped_grants.pb.go b/pb/c1/connector/v2/annotation_type_scoped_grants.pb.go index 12b1a230b..bee03140f 100644 --- a/pb/c1/connector/v2/annotation_type_scoped_grants.pb.go +++ b/pb/c1/connector/v2/annotation_type_scoped_grants.pb.go @@ -48,6 +48,22 @@ const ( // replay, via source-cache annotations) EVERY grant of that type. The // syncer no longer visits each resource, so completeness rests entirely on // the connector's enumeration. +// +// SKIP ANNOTATIONS: for an annotated type the syncer does not consult +// shouldSkipGrants on the type-scoped path. The connector owns skip +// semantics for resources of that type. +// +// ZERO-RESOURCE TYPES: the planner still enqueues the type-scoped action +// when the store holds no resources of the type. The connector must +// return an empty page (or SpawnCursors that resolve empty), not an error. +// +// TARGETED SYNC: SyncTargetedResource must not enqueue a per-resource +// SyncGrantsOp for a type-scoped type. Type-scoped grant enumeration is +// left to a full grants phase. +// +// ROUTING MARKER ONLY: TypeScopedGrants is absent from the ingestion- +// invariant side-effect coverage map — it routes ListGrants; it does not +// imply post-ingest repair beyond ordinary grant handling. type TypeScopedGrants struct { state protoimpl.MessageState `protogen:"hybrid.v1"` unknownFields protoimpl.UnknownFields @@ -91,13 +107,14 @@ func (b0 TypeScopedGrants_builder) Build() *TypeScopedGrants { return m0 } -// SpawnCursors is attached to a ListGrants response to enqueue additional -// independent sibling cursors. Each token is delivered back to the -// connector as the page token of its own action — scheduled by the -// syncer's worker pool, rate-limited, and checkpointed like any other -// pagination. Honored on BOTH type-scoped and per-resource ListGrants -// responses; the spawned actions inherit the response's resource identity -// (type only for type-scoped calls, type+resource for per-resource calls). +// SpawnCursors is attached to a ListGrants or ListEntitlements response to +// enqueue additional independent sibling cursors. Each token is delivered +// back to the connector as the page token of its own action — scheduled by +// the syncer's worker pool, rate-limited, and checkpointed like any other +// pagination. Honored on BOTH type-scoped and per-resource ListGrants / +// ListEntitlements responses; the spawned actions inherit the response's +// resource identity (type only for type-scoped calls, type+resource for +// per-resource calls). // // Typical uses: // @@ -131,7 +148,7 @@ type SpawnCursors struct { // Capped per response (spawned actions persist in the checkpointed // state token); chain additional spawns across pages to fan out wider. PageTokens []string `protobuf:"bytes,1,rep,name=page_tokens,json=pageTokens,proto3" json:"page_tokens,omitempty"` - // Optional connector estimate of total grants across all cursors of this + // Optional connector estimate of total rows across all cursors of this // type, for progress reporting. Zero means unknown. EstimatedTotal int64 `protobuf:"varint,2,opt,name=estimated_total,json=estimatedTotal,proto3" json:"estimated_total,omitempty"` unknownFields protoimpl.UnknownFields @@ -192,7 +209,7 @@ type SpawnCursors_builder struct { // Capped per response (spawned actions persist in the checkpointed // state token); chain additional spawns across pages to fan out wider. PageTokens []string - // Optional connector estimate of total grants across all cursors of this + // Optional connector estimate of total rows across all cursors of this // type, for progress reporting. Zero means unknown. EstimatedTotal int64 } diff --git a/pb/c1/connector/v2/annotation_type_scoped_grants_protoopaque.pb.go b/pb/c1/connector/v2/annotation_type_scoped_grants_protoopaque.pb.go index 499c83d81..5533c069b 100644 --- a/pb/c1/connector/v2/annotation_type_scoped_grants_protoopaque.pb.go +++ b/pb/c1/connector/v2/annotation_type_scoped_grants_protoopaque.pb.go @@ -48,6 +48,22 @@ const ( // replay, via source-cache annotations) EVERY grant of that type. The // syncer no longer visits each resource, so completeness rests entirely on // the connector's enumeration. +// +// SKIP ANNOTATIONS: for an annotated type the syncer does not consult +// shouldSkipGrants on the type-scoped path. The connector owns skip +// semantics for resources of that type. +// +// ZERO-RESOURCE TYPES: the planner still enqueues the type-scoped action +// when the store holds no resources of the type. The connector must +// return an empty page (or SpawnCursors that resolve empty), not an error. +// +// TARGETED SYNC: SyncTargetedResource must not enqueue a per-resource +// SyncGrantsOp for a type-scoped type. Type-scoped grant enumeration is +// left to a full grants phase. +// +// ROUTING MARKER ONLY: TypeScopedGrants is absent from the ingestion- +// invariant side-effect coverage map — it routes ListGrants; it does not +// imply post-ingest repair beyond ordinary grant handling. type TypeScopedGrants struct { state protoimpl.MessageState `protogen:"opaque.v1"` unknownFields protoimpl.UnknownFields @@ -91,13 +107,14 @@ func (b0 TypeScopedGrants_builder) Build() *TypeScopedGrants { return m0 } -// SpawnCursors is attached to a ListGrants response to enqueue additional -// independent sibling cursors. Each token is delivered back to the -// connector as the page token of its own action — scheduled by the -// syncer's worker pool, rate-limited, and checkpointed like any other -// pagination. Honored on BOTH type-scoped and per-resource ListGrants -// responses; the spawned actions inherit the response's resource identity -// (type only for type-scoped calls, type+resource for per-resource calls). +// SpawnCursors is attached to a ListGrants or ListEntitlements response to +// enqueue additional independent sibling cursors. Each token is delivered +// back to the connector as the page token of its own action — scheduled by +// the syncer's worker pool, rate-limited, and checkpointed like any other +// pagination. Honored on BOTH type-scoped and per-resource ListGrants / +// ListEntitlements responses; the spawned actions inherit the response's +// resource identity (type only for type-scoped calls, type+resource for +// per-resource calls). // // Typical uses: // @@ -187,7 +204,7 @@ type SpawnCursors_builder struct { // Capped per response (spawned actions persist in the checkpointed // state token); chain additional spawns across pages to fan out wider. PageTokens []string - // Optional connector estimate of total grants across all cursors of this + // Optional connector estimate of total rows across all cursors of this // type, for progress reporting. Zero means unknown. EstimatedTotal int64 } diff --git a/pkg/connectorbuilder/resource_syncer.go b/pkg/connectorbuilder/resource_syncer.go index 6e776850b..c5aa24f83 100644 --- a/pkg/connectorbuilder/resource_syncer.go +++ b/pkg/connectorbuilder/resource_syncer.go @@ -84,6 +84,18 @@ type TypeScopedGrantsSyncer interface { GrantsForResourceType(ctx context.Context, resourceTypeID string, opts resource.SyncOpAttrs) ([]*v2.Grant, *resource.SyncOpResults, error) } +// TypeScopedEntitlementsSyncer is the entitlements-phase analogue of +// TypeScopedGrantsSyncer: the connector enumerates entitlements for a WHOLE +// resource type instead of being called once per resource. Implement it on +// a resource syncer whose ResourceType carries the v2.TypeScopedEntitlements +// annotation; the syncer then issues ListEntitlements calls with an empty +// resource id, which the builder routes here. +// +// Planning / SpawnCursors / source-cache behavior matches TypeScopedGrantsSyncer. +type TypeScopedEntitlementsSyncer interface { + EntitlementsForResourceType(ctx context.Context, resourceTypeID string, opts resource.SyncOpAttrs) ([]*v2.Entitlement, *resource.SyncOpResults, error) +} + // syncOpAttrs assembles the per-call SyncOpAttrs handed to V2 resource // syncers. SourceCache is never nil: when the runner supplied no lookup // (source cache disabled/degraded) connectors see NoopLookup and every @@ -382,9 +394,17 @@ func (b *builder) ListEntitlements(ctx context.Context, request *v2.Entitlements start := b.nowFunc() tt := tasks.ListEntitlementsType - rb, ok := b.resourceSyncers[request.GetResource().GetId().GetResourceType()] + + if request.GetResource() == nil { + err = status.Error(codes.InvalidArgument, "error: list entitlements requires a resource") + b.m.RecordTaskFailure(ctx, tt, b.nowFunc().Sub(start), err) + return nil, err + } + + rid := request.GetResource().GetId() + rb, ok := b.resourceSyncers[rid.GetResourceType()] if !ok { - err = status.Errorf(codes.NotFound, "error: list entitlements with unknown resource type %s", request.GetResource().GetId().GetResourceType()) + err = status.Errorf(codes.NotFound, "error: list entitlements with unknown resource type %s", rid.GetResourceType()) b.m.RecordTaskFailure(ctx, tt, b.nowFunc().Sub(start), err) return nil, err } @@ -393,12 +413,33 @@ func (b *builder) ListEntitlements(ctx context.Context, request *v2.Entitlements Size: int(request.GetPageSize()), Token: request.GetPageToken(), } - opts, contLookup, err := b.continuationOpAttrs(request.GetActiveSyncId(), token, annotations.Annotations(request.GetAnnotations())) + reqAnnos := annotations.Annotations(request.GetAnnotations()) + opts, contLookup, err := b.continuationOpAttrs(request.GetActiveSyncId(), token, reqAnnos) if err != nil { b.m.RecordTaskFailure(ctx, tt, b.nowFunc().Sub(start), err) return nil, err } - out, retOptions, err := rb.Entitlements(ctx, request.GetResource(), opts) + + typeScoped := reqAnnos.Contains(&v2.TypeScopedEntitlements{}) + + var out []*v2.Entitlement + var retOptions *resource.SyncOpResults + if typeScoped { + // Type-scoped entitlements call (see v2.TypeScopedEntitlements): the + // request annotation is the routing marker (the resource is a + // self-referential {type, type} stub to satisfy wire validation). + // Only legal against a syncer that opted in. + tses, tsOk := rb.(TypeScopedEntitlementsSyncer) + if !tsOk { + err = status.Errorf(codes.InvalidArgument, + "error: type-scoped list entitlements for resource type %s, but its syncer does not implement TypeScopedEntitlementsSyncer", rid.GetResourceType()) + b.m.RecordTaskFailure(ctx, tt, b.nowFunc().Sub(start), err) + return nil, err + } + out, retOptions, err = tses.EntitlementsForResourceType(ctx, rid.GetResourceType(), opts) + } else { + out, retOptions, err = rb.Entitlements(ctx, request.GetResource(), opts) + } if retOptions == nil { retOptions = &resource.SyncOpResults{} } diff --git a/pkg/connectorbuilder/type_scoped_entitlements_test.go b/pkg/connectorbuilder/type_scoped_entitlements_test.go new file mode 100644 index 000000000..654f39ddb --- /dev/null +++ b/pkg/connectorbuilder/type_scoped_entitlements_test.go @@ -0,0 +1,122 @@ +package connectorbuilder + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + "github.com/conductorone/baton-sdk/pkg/annotations" + "github.com/conductorone/baton-sdk/pkg/sourcecache" + "github.com/conductorone/baton-sdk/pkg/types/resource" +) + +type typeScopedEntitlementsSyncer struct { + testResourceSyncerV2Simple + calls int + perResource int + deferredLookup bool +} + +func (s *typeScopedEntitlementsSyncer) Entitlements(ctx context.Context, r *v2.Resource, opts resource.SyncOpAttrs) ([]*v2.Entitlement, *resource.SyncOpResults, error) { + s.perResource++ + return s.testResourceSyncerV2Simple.Entitlements(ctx, r, opts) +} + +func (s *typeScopedEntitlementsSyncer) EntitlementsForResourceType(ctx context.Context, resourceTypeID string, opts resource.SyncOpAttrs) ([]*v2.Entitlement, *resource.SyncOpResults, error) { + s.calls++ + if s.deferredLookup { + _, _, err := opts.SourceCache.LookupPreviousSourceCache(ctx, sourcecache.RowKindEntitlements, "ents/type") + if err != nil { + return nil, nil, err + } + } + r := v2.Resource_builder{ + Id: v2.ResourceId_builder{ResourceType: resourceTypeID, Resource: "g1"}.Build(), + }.Build() + return []*v2.Entitlement{ + v2.Entitlement_builder{ + Id: "ent-type-1", + Resource: r, + }.Build(), + }, &resource.SyncOpResults{}, nil +} + +func typeScopedEntitlementsRequest(annos annotations.Annotations) *v2.EntitlementsServiceListEntitlementsRequest { + return v2.EntitlementsServiceListEntitlementsRequest_builder{ + Resource: v2.Resource_builder{ + Id: v2.ResourceId_builder{ResourceType: "group", Resource: "group"}.Build(), + }.Build(), + Annotations: annos, + }.Build() +} + +func TestListEntitlements_TypeScopedRoutes(t *testing.T) { + ctx := context.Background() + ts := &typeScopedEntitlementsSyncer{testResourceSyncerV2Simple: testResourceSyncerV2Simple{resourceType: "group"}} + conn, err := NewConnector(ctx, &testConnectorV2{resourceSyncers: []ResourceSyncerV2{ts}}) + require.NoError(t, err) + + resp, err := conn.ListEntitlements(ctx, typeScopedEntitlementsRequest(annotations.New(&v2.TypeScopedEntitlements{}))) + require.NoError(t, err) + require.Len(t, resp.GetList(), 1) + require.Equal(t, "ent-type-1", resp.GetList()[0].GetId()) + require.Equal(t, 1, ts.calls) + require.Zero(t, ts.perResource) +} + +func TestListEntitlements_TypeScopedMissingInterface(t *testing.T) { + ctx := context.Background() + ts := &testResourceSyncerV2Simple{resourceType: "group"} + conn, err := NewConnector(ctx, &testConnectorV2{resourceSyncers: []ResourceSyncerV2{ts}}) + require.NoError(t, err) + + _, err = conn.ListEntitlements(ctx, typeScopedEntitlementsRequest(annotations.New(&v2.TypeScopedEntitlements{}))) + require.Error(t, err) + require.Equal(t, codes.InvalidArgument, status.Code(err)) +} + +func TestListEntitlements_UnmarkedStillPerResource(t *testing.T) { + ctx := context.Background() + ts := &typeScopedEntitlementsSyncer{testResourceSyncerV2Simple: testResourceSyncerV2Simple{resourceType: "group"}} + conn, err := NewConnector(ctx, &testConnectorV2{resourceSyncers: []ResourceSyncerV2{ts}}) + require.NoError(t, err) + + resp, err := conn.ListEntitlements(ctx, typeScopedEntitlementsRequest(nil)) + require.NoError(t, err) + require.Len(t, resp.GetList(), 1) + require.Equal(t, "entitlement-1", resp.GetList()[0].GetId()) + require.Zero(t, ts.calls) + require.Equal(t, 1, ts.perResource) +} + +func TestListEntitlements_TypeScopedContinuationPreservesMarker(t *testing.T) { + ctx := context.Background() + ts := &typeScopedEntitlementsSyncer{ + testResourceSyncerV2Simple: testResourceSyncerV2Simple{resourceType: "group"}, + deferredLookup: true, + } + conn, err := NewConnector(ctx, &testConnectorV2{resourceSyncers: []ResourceSyncerV2{ts}}) + require.NoError(t, err) + + offer := annotations.New(&v2.SourceCacheLookupOffer{}, &v2.TypeScopedEntitlements{}) + resp, err := conn.ListEntitlements(ctx, typeScopedEntitlementsRequest(offer)) + require.NoError(t, err) + require.Empty(t, resp.GetList()) + respAnnos := annotations.Annotations(resp.GetAnnotations()) + require.True(t, respAnnos.Contains(&v2.SourceCacheLookupAsk{})) + require.Equal(t, 1, ts.calls) + + answers := annotations.New(&v2.SourceCacheLookupOffer{}, &v2.TypeScopedEntitlements{}) + answers.Update(sourcecache.AnswersProto([]sourcecache.Answer{ + {Query: sourcecache.Query{RowKind: sourcecache.RowKindEntitlements, ScopeHash: "ents/type"}, Found: false}, + })) + resp2, err := conn.ListEntitlements(ctx, typeScopedEntitlementsRequest(answers)) + require.NoError(t, err) + require.Len(t, resp2.GetList(), 1) + require.Equal(t, 2, ts.calls) + require.Zero(t, ts.perResource) +} diff --git a/pkg/dotc1z/engine/pebble/cleanup.go b/pkg/dotc1z/engine/pebble/cleanup.go index b07c98ba3..ef8e84f29 100644 --- a/pkg/dotc1z/engine/pebble/cleanup.go +++ b/pkg/dotc1z/engine/pebble/cleanup.go @@ -113,6 +113,11 @@ func (e *Engine) ResetForNewSync(ctx context.Context) error { return err } } + // Ingestion-fact markers live in engine-meta, outside the excised + // record span; the new sync must not inherit the old sync's facts. + if err := e.clearIngestFactMarkers(); err != nil { + return fmt.Errorf("ResetForNewSync: clear ingest fact markers: %w", err) + } e.noteEntitlementKeyspaceWrite() return nil }) diff --git a/pkg/dotc1z/engine/pebble/engine.go b/pkg/dotc1z/engine/pebble/engine.go index e35db586e..a75bd5b03 100644 --- a/pkg/dotc1z/engine/pebble/engine.go +++ b/pkg/dotc1z/engine/pebble/engine.go @@ -115,6 +115,12 @@ type Engine struct { testDigestBuildHook func(stage string) error testDigestNodeFlushBytes int + // externalMatchFact is the monotone existence bit for "this sync + // holds at least one ExternalResourceMatch*-annotated grant" — a + // dense ingestion fact tracked without an index (see + // ingest_facts.go). Durable twin: encodeExternalMatchFactKey. + externalMatchFact atomic.Bool + // synthLayer is the open wave-scoped layer session, if any (see // BeginSynthesizedGrantLayer). Single producer: the expansion driver // opens/adds/finishes sessions strictly sequentially. synthLayerMu @@ -272,6 +278,12 @@ func Open(ctx context.Context, dir string, opts ...Option) (*Engine, error) { _ = e.Close() return nil, err } + // Restore durable ingestion-fact markers (see ingest_facts.go) — + // same rationale as the deferred-index marker above. + if err := e.restoreIngestFactMarkers(); err != nil { + _ = e.Close() + return nil, err + } // Run secondary-index migrations before returning. Migrations // are skipped for read-only opens (the on-disk file is // immutable, so we'd error out trying to backfill). diff --git a/pkg/dotc1z/engine/pebble/grants.go b/pkg/dotc1z/engine/pebble/grants.go index f14c1f7c1..1fa3cf5d8 100644 --- a/pkg/dotc1z/engine/pebble/grants.go +++ b/pkg/dotc1z/engine/pebble/grants.go @@ -114,6 +114,11 @@ func (e *Engine) PutGrantRecords(ctx context.Context, records ...*v3.GrantRecord if r == nil { continue } + // Dense ingestion facts (one atomic load once armed; the + // annotation walk only runs until the sync's first hit). + if err := e.noteGrantRecordFacts(r); err != nil { + return err + } id, err := grantIdentityFromRecord(r) if err != nil { return err diff --git a/pkg/dotc1z/engine/pebble/ingest_facts.go b/pkg/dotc1z/engine/pebble/ingest_facts.go new file mode 100644 index 000000000..9b2cb66cc --- /dev/null +++ b/pkg/dotc1z/engine/pebble/ingest_facts.go @@ -0,0 +1,207 @@ +package pebble + +// Ingestion facts: engine-maintained evidence about rows, consumed by the +// syncer's ingestion invariants (see +// docs/tasks/source-cache-ingestion-invariants.md). +// +// Facts are chosen by density. Sparse facts get partial indexes +// (by_needs_expansion). Dense facts — annotations that the biggest +// connectors stamp on ~100% of grants — get a MONOTONE existence bit +// (this file) or no structure at all (read from row values on demand). +// A dense inline secondary index is deliberately rejected: it is the +// cost class the engine already refuses (the reason by_principal is a +// deferred build). +// +// The existence bit mirrors the deferred-index pending marker exactly: +// an atomic bool for in-process reads, a durable engine-meta key so a +// crash/resume keeps the fact, restored at Open, cleared by +// ResetForNewSync. Monotonicity (set-once, never unset within a sync) +// makes the bit order-independent under parallel workers and mixed +// stream/replay ingestion. + +import ( + "context" + "errors" + "fmt" + "strings" + + "github.com/cockroachdb/pebble/v2" + "google.golang.org/protobuf/types/known/anypb" + + v3 "github.com/conductorone/baton-sdk/pb/c1/storage/v3" + "github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/codec" +) + +// encodeExternalMatchFactKey is the durable engine-meta marker for "this +// sync wrote at least one grant carrying an ExternalResourceMatch* +// annotation". Engine-meta (typeEngineMeta) survives ResetForNewSync's +// record-span excise, so the reset path clears it explicitly. +func encodeExternalMatchFactKey() []byte { + buf := make([]byte, 0, 2+len("fact_external_match_grants")) + buf = append(buf, versionV3, typeEngineMeta) + return codec.AppendTupleStrings(buf, "fact_external_match_grants") +} + +// markExternalMatchFact records the existence fact, durably. CAS keeps +// the durable write to one per sync, off the per-record hot path: after +// the first hit, callers pay one atomic load. +func (e *Engine) markExternalMatchFact() error { + if !e.externalMatchFact.CompareAndSwap(false, true) { + return nil + } + if err := e.db.Set(encodeExternalMatchFactKey(), nil, pebble.Sync); err != nil { + // Roll back so a retried write re-attempts the durable marker; + // an armed flag without the durable key would diverge from a + // crash/resume of the same sync (same rationale as + // markDeferredIdxPending). + e.externalMatchFact.Store(false) + return fmt.Errorf("arm external-match fact marker: %w", err) + } + return nil +} + +// HasExternalMatchGrants reports the existence fact for the open sync. +func (e *Engine) HasExternalMatchGrants() bool { + return e.externalMatchFact.Load() +} + +// restoreIngestFactMarkers reloads durable fact markers at Open (a prior +// process may have set them before crashing; a finished file carries them +// for read-side consumers). +func (e *Engine) restoreIngestFactMarkers() error { + if _, closer, err := e.db.Get(encodeExternalMatchFactKey()); err == nil { + closer.Close() + e.externalMatchFact.Store(true) + } else if !errors.Is(err, pebble.ErrNotFound) { + return err + } + return nil +} + +// clearIngestFactMarkers drops the durable markers and in-memory bits. +// Called by ResetForNewSync, whose record-span excise does not reach the +// engine-meta keyspace. +func (e *Engine) clearIngestFactMarkers() error { + e.externalMatchFact.Store(false) + return e.db.Delete(encodeExternalMatchFactKey(), pebble.Sync) +} + +// grantRecordHasExternalMatch reports whether a translated GrantRecord's +// annotations carry any ExternalResourceMatch* type. Called on the put +// path only while the fact bit is still unset, so the per-record cost +// vanishes after the first hit. +func grantRecordHasExternalMatch(anns []*anypb.Any) bool { + for _, a := range anns { + url := a.GetTypeUrl() + name := url + if i := strings.LastIndexByte(url, '/'); i >= 0 { + name = url[i+1:] + } + switch name { + case anyTypeExternalResourceMatch, anyTypeExternalResourceMatchAll, anyTypeExternalResourceMatchID: + return true + } + } + return false +} + +// noteGrantRecordFacts inspects one to-be-written grant record for dense +// facts. Cheap by construction: one atomic load when the bit is already +// set; the annotation walk only runs until the sync's first hit. +func (e *Engine) noteGrantRecordFacts(r *v3.GrantRecord) error { + if e.externalMatchFact.Load() { + return nil + } + if grantRecordHasExternalMatch(r.GetAnnotations()) { + return e.markExternalMatchFact() + } + return nil +} + +// ForEachDistinctGrantEntitlementResource visits each distinct +// (entitlement resource_type_id, resource_id) pair present in the grant +// primary keyspace, in key order. The keyspace leads with +// ent_rt | ent_rid, so distinctness is a prefix-skip: one seek per +// distinct resource — O(distinct) seeks, never O(grants). Backs the +// syncer's grant→resource referential invariant. +func (e *Engine) ForEachDistinctGrantEntitlementResource(ctx context.Context, visit func(resourceTypeID, resourceID string) error) error { + prefix := encodeGrantPrefix() + iter, err := e.db.NewIter(&pebble.IterOptions{ + LowerBound: prefix, + UpperBound: upperBoundOf(prefix), + }) + if err != nil { + return err + } + defer iter.Close() + + const headerLen = 3 // versionV3, typeGrant, separator + for valid := iter.First(); valid; { + if err := ctx.Err(); err != nil { + return err + } + key := iter.Key() + if len(key) <= headerLen { + return fmt.Errorf("distinct ent-resource scan: malformed grant key %x", key) + } + tail := key[headerLen:] + rtBytes, next, ok := codec.DecodeTupleStringAlias(tail, 0) + if !ok || next >= len(tail) { + return fmt.Errorf("distinct ent-resource scan: malformed grant key tail %x", key) + } + ridBytes, _, ok := codec.DecodeTupleStringAlias(tail, next+1) + if !ok { + return fmt.Errorf("distinct ent-resource scan: malformed grant key tail %x", key) + } + rt, rid := string(rtBytes), string(ridBytes) + if err := visit(rt, rid); err != nil { + return err + } + // Skip every remaining grant of this entitlement resource. + valid = iter.SeekGE(upperBoundOf(encodeGrantPrimaryEntitlementResourcePrefix(rt, rid))) + } + return iter.Error() +} + +// GrantsForEntResourceCarryInsertFact reports whether any grant under the +// given entitlement resource carries the InsertResourceGrants annotation. +// Reads row values, so it is reserved for DANGLING referential probes — +// rare to zero on healthy syncs — never the bulk path. +func (e *Engine) GrantsForEntResourceCarryInsertFact(ctx context.Context, resourceTypeID, resourceID string) (bool, error) { + prefix := encodeGrantPrimaryEntitlementResourcePrefix(resourceTypeID, resourceID) + iter, err := e.db.NewIter(&pebble.IterOptions{ + LowerBound: prefix, + UpperBound: upperBoundOf(prefix), + }) + if err != nil { + return false, err + } + defer iter.Close() + for iter.First(); iter.Valid(); iter.Next() { + if err := ctx.Err(); err != nil { + return false, err + } + flags, err := scanGrantSourceCacheFlagsRaw(iter.Value()) + if err != nil { + return false, err + } + if flags.insertResourceGrants { + return true, nil + } + } + return false, iter.Error() +} + +// HasResourceRecord reports whether a resource row exists — the probe +// side of the referential invariant. +func (e *Engine) HasResourceRecord(ctx context.Context, resourceTypeID, resourceID string) (bool, error) { + _, closer, err := e.db.Get(encodeResourceKey(resourceTypeID, resourceID)) + if err != nil { + if errors.Is(err, pebble.ErrNotFound) { + return false, nil + } + return false, err + } + closer.Close() + return true, nil +} diff --git a/pkg/dotc1z/engine/pebble/raw_records.go b/pkg/dotc1z/engine/pebble/raw_records.go index 6b29395c9..1881a4a9f 100644 --- a/pkg/dotc1z/engine/pebble/raw_records.go +++ b/pkg/dotc1z/engine/pebble/raw_records.go @@ -773,47 +773,6 @@ func scanResourceChildTypeIDsRaw(value []byte) ([]string, error) { return out, nil } -const anyTypeEntitlementExclusionGroup = "c1.connector.v2.EntitlementExclusionGroup" - -// scanEntitlementExclusionGroupsRaw returns the exclusion-group -// annotations on a marshaled EntitlementRecord (annotations field 7), -// paired with the row's identity so the syncer can validate them exactly -// like fresh response rows. Nil for the common unannotated case, with no -// identity scan performed. -func scanEntitlementExclusionGroupsRaw(value []byte) ([]ReplayedExclusionGroup, error) { - var groups []*v2.EntitlementExclusionGroup - err := scanAnnotationAnysRaw(value, 7, func(typeName string, payload []byte) error { - if typeName != anyTypeEntitlementExclusionGroup { - return nil - } - eg := &v2.EntitlementExclusionGroup{} - if err := proto.Unmarshal(payload, eg); err != nil { - return fmt.Errorf("raw record: unmarshal EntitlementExclusionGroup annotation: %w", err) - } - groups = append(groups, eg) - return nil - }) - if err != nil { - return nil, err - } - if len(groups) == 0 { - return nil, nil - } - rt, _, externalID, err := scanEntitlementIdentityFieldsRaw(value) - if err != nil { - return nil, err - } - out := make([]ReplayedExclusionGroup, 0, len(groups)) - for _, eg := range groups { - out = append(out, ReplayedExclusionGroup{ - EntitlementID: externalID, - ResourceTypeID: rt, - Group: eg, - }) - } - return out, nil -} - // grantSourceCacheFlags reports which replay side-effect annotations a // marshaled GrantRecord carries (annotations field 8). type grantSourceCacheFlags struct { diff --git a/pkg/dotc1z/engine/pebble/source_cache.go b/pkg/dotc1z/engine/pebble/source_cache.go index 733f326b9..961562dc5 100644 --- a/pkg/dotc1z/engine/pebble/source_cache.go +++ b/pkg/dotc1z/engine/pebble/source_cache.go @@ -9,7 +9,6 @@ import ( "google.golang.org/protobuf/encoding/protowire" "google.golang.org/protobuf/types/known/timestamppb" - v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" v3 "github.com/conductorone/baton-sdk/pb/c1/storage/v3" "github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/codec" ) @@ -43,24 +42,19 @@ type ReplayedParentResource struct { ChildTypeIDs []string } -// ReplayedExclusionGroup is one exclusion-group annotation found on a -// replayed entitlement row, paired with the row's identity so the syncer -// can run the same invariant validation fresh response rows get. -type ReplayedExclusionGroup struct { - EntitlementID string - ResourceTypeID string - Group *v2.EntitlementExclusionGroup -} - // SourceCacheReplayResult reports what one scope's replay copied. +// +// NOTE on side effects: replayed rows contribute to the syncer's +// side-effect state through STORE-DERIVED mechanisms, not through fields +// here — the needs_expansion index and the external-match existence bit +// are maintained for replayed rows exactly as for fresh ones, and the +// syncer's ingestion invariants read the store (see +// docs/tasks/source-cache-ingestion-invariants.md). The two fields that +// remain (ChildResources, RelatedResourceRows) are MECHANISMS, not +// evidence: child scheduling and related-resource recreation cannot be +// derived from the store after the fact, so replay must carry them. type SourceCacheReplayResult struct { Rows int64 - // NeedsExpansion is true when at least one copied grant row carried - // needs_expansion. The syncer must arm grant expansion in this case: - // replayed pages never pass GrantExpandable-annotated rows through - // the syncer's connector-response path, which is otherwise the only - // thing that enables the expansion phase. - NeedsExpansion bool // StaleSkipped counts index entries under the scope that did NOT // yield a copied row: the primary was missing, or its value stamp // named a different scope. This is the discriminator between "scope @@ -72,12 +66,6 @@ type SourceCacheReplayResult struct { // ChildResources lists replayed parents carrying ChildResourceType // annotations (resources replay only). See ReplayedParentResource. ChildResources []ReplayedParentResource - // HasExternalResourceGrants is true when at least one copied grant - // row carries an ExternalResourceMatch* annotation (grants replay - // only). The syncer must arm external-resource-grant processing — - // same rationale as NeedsExpansion: response rows are the only - // other place the flag is set, and a replayed page has none. - HasExternalResourceGrants bool // RelatedResourceRows counts resource rows copied from prev because // a replayed grant carried InsertResourceGrants (grants replay // only). Those resources exist ONLY as grant-driven insertions — @@ -85,12 +73,6 @@ type SourceCacheReplayResult struct { // recreate them or the current sync holds grants whose resources // do not exist. RelatedResourceRows int64 - // ExclusionGroups lists exclusion-group annotations found on - // replayed entitlement rows (entitlements replay only). The syncer - // must run the same invariant validation fresh rows get (one - // default per group, single resource type, size cap) — validation - // runs on response rows, which a replayed page has none of. - ExclusionGroups []ReplayedExclusionGroup } // SourceCacheManifestSnapshot returns every manifest entry as @@ -749,16 +731,19 @@ func (e *Engine) ReplaySourceCacheGrants(ctx context.Context, prev *Engine, scop writeVal = stripped } } - // Replay side-effect annotations: response rows are the only - // other place these are detected, and a replayed page has - // none — see SourceCacheReplayResult field docs. + // Dense ingestion facts for replayed rows: same evidence the + // put path maintains, so the syncer's store-derived seams + // (I2) see replayed and fresh rows identically. flags, flagsErr := scanGrantSourceCacheFlagsRaw(val) if flagsErr != nil { closer.Close() return flagsErr } if flags.externalResourceMatch { - res.HasExternalResourceGrants = true + if err := e.markExternalMatchFact(); err != nil { + closer.Close() + return err + } } if flags.insertResourceGrants && entRT != "" && entRID != "" { resKey := encodeResourceKey(entRT, entRID) @@ -795,9 +780,6 @@ func (e *Engine) ReplaySourceCacheGrants(ctx context.Context, prev *Engine, scop return err } } - if needsExpansion { - res.NeedsExpansion = true - } res.Rows++ rowsInBatch++ if rowsInBatch >= replayBatchRows { @@ -946,14 +928,6 @@ func (e *Engine) ReplaySourceCacheEntitlements(ctx context.Context, prev *Engine res.StaleSkipped++ continue } - // Exclusion-group annotations must reach the syncer's - // validator — see SourceCacheReplayResult.ExclusionGroups. - egs, egErr := scanEntitlementExclusionGroupsRaw(val) - if egErr != nil { - closer.Close() - return egErr - } - res.ExclusionGroups = append(res.ExclusionGroups, egs...) // The replayed row's source-scope index key is byte-identical // to prev's — the only entitlement secondary index. Clean up a // differing stamp on any record the current sync already wrote diff --git a/pkg/dotc1z/engine/pebble/source_cache_engine_test.go b/pkg/dotc1z/engine/pebble/source_cache_engine_test.go index d275b6464..7e829e170 100644 --- a/pkg/dotc1z/engine/pebble/source_cache_engine_test.go +++ b/pkg/dotc1z/engine/pebble/source_cache_engine_test.go @@ -101,7 +101,8 @@ func TestSourceCacheReplayGrantsAcrossEngines(t *testing.T) { res, err := cur.PebbleEngine().ReplaySourceCacheGrants(ctx, prev.PebbleEngine(), scopeA) require.NoError(t, err) require.Equal(t, int64(2), res.Rows) - require.True(t, res.NeedsExpansion, "replay must report the expandable grant so the syncer can arm expansion") + // Expansion arming is store-derived (the by_needs_expansion index the + // replay synthesizes below), no longer reported on the result. // Replayed rows are readable by ID and carry their expansion flag. got, err := cur.PebbleEngine().GetGrantRecord(ctx, gExpandable.GetId()) @@ -153,7 +154,7 @@ func TestSourceCacheReplayIsIdempotent(t *testing.T) { res2, err := cur.PebbleEngine().ReplaySourceCacheGrants(ctx, prev.PebbleEngine(), scopeA) require.NoError(t, err) require.Equal(t, int64(2), res2.Rows, "second replay converges, not errors") - require.Equal(t, res1.NeedsExpansion, res2.NeedsExpansion) + _ = res1 // State after double replay is exactly the two rows, once each. var seen int diff --git a/pkg/dotc1z/source_cache.go b/pkg/dotc1z/source_cache.go index 80171981f..f89d38f7f 100644 --- a/pkg/dotc1z/source_cache.go +++ b/pkg/dotc1z/source_cache.go @@ -67,6 +67,49 @@ type SourceCacheStore interface { var _ SourceCacheStore = (*pebbleStore)(nil) +// IngestFactStore is the optional store capability backing the syncer's +// ingestion invariants (see docs/tasks/source-cache-ingestion-invariants.md): +// dense row facts tracked as existence bits, and the ordered-key +// inspection reads the referential invariant uses. Pebble-only; the +// syncer degrades per-invariant when the store lacks it. +type IngestFactStore interface { + // HasExternalMatchGrants reports whether the open sync holds at + // least one grant carrying an ExternalResourceMatch* annotation — + // the store-derived truth behind HasExternalResourcesGrants. + HasExternalMatchGrants(ctx context.Context) (bool, error) + + // ForEachDistinctGrantEntitlementResource visits each distinct + // entitlement resource referenced by any grant: one seek per + // distinct resource, never O(grants). + ForEachDistinctGrantEntitlementResource(ctx context.Context, visit func(resourceTypeID, resourceID string) error) error + + // GrantsForEntResourceCarryInsertFact reports whether any grant + // under the entitlement resource carries InsertResourceGrants. + // Value-reading; reserved for dangling-reference probes. + GrantsForEntResourceCarryInsertFact(ctx context.Context, resourceTypeID, resourceID string) (bool, error) + + // HasResourceRecord reports whether a resource row exists. + HasResourceRecord(ctx context.Context, resourceTypeID, resourceID string) (bool, error) +} + +var _ IngestFactStore = (*pebbleStore)(nil) + +func (s *pebbleStore) HasExternalMatchGrants(ctx context.Context) (bool, error) { + return s.engine.HasExternalMatchGrants(), nil +} + +func (s *pebbleStore) ForEachDistinctGrantEntitlementResource(ctx context.Context, visit func(resourceTypeID, resourceID string) error) error { + return s.engine.ForEachDistinctGrantEntitlementResource(ctx, visit) +} + +func (s *pebbleStore) GrantsForEntResourceCarryInsertFact(ctx context.Context, resourceTypeID, resourceID string) (bool, error) { + return s.engine.GrantsForEntResourceCarryInsertFact(ctx, resourceTypeID, resourceID) +} + +func (s *pebbleStore) HasResourceRecord(ctx context.Context, resourceTypeID, resourceID string) (bool, error) { + return s.engine.HasResourceRecord(ctx, resourceTypeID, resourceID) +} + // sourceCacheEngine recovers the Pebble engine from an arbitrary store, // nil-safe. Mirrors pebble.AsEngine but accepts any value so the syncer // can probe its previous-sync reader without caring about its static type. diff --git a/pkg/sync/ingest_invariants.go b/pkg/sync/ingest_invariants.go new file mode 100644 index 000000000..83e9d70f5 --- /dev/null +++ b/pkg/sync/ingest_invariants.go @@ -0,0 +1,338 @@ +package sync //nolint:revive,nolintlint // we can't change the package name for backwards compatibility + +// Ingestion invariants: store-derived definitions of the syncer's +// side effects, evaluated at consuming seams. Design: +// docs/tasks/source-cache-ingestion-invariants.md. +// +// Background. The syncer historically attached side effects (flags, +// scheduled actions, derived rows, validation) to the connector-response +// ingest loop. That was correct while response pages were the only way +// rows entered the store; source-cache replay added an engine-side +// ingestion path and turned every stream-coupled side effect into a +// latent bug. Rather than mirroring each side effect once per ingestion +// path (unmarked coupling, rediscovered by review), each side effect is +// defined as a function of the store and evaluated here: +// +// - I1 expansion arming: PendingExpansion probe at SyncGrantExpansionOp +// (lives in parallel_syncer.go — the pattern's prototype). +// - I2 external-match arming: engine existence-bit probe at +// SyncExternalResourcesOp start; REPAIRS the flag. Pebble maintains +// the bit for fresh and replayed rows alike; the response-loop arm +// remains as the fast path and as SQLite's only mechanism. +// - I3 grant→resource referential integrity: post-collection prefix +// skip-scan over the grant keyspace. Dangling ref + an +// InsertResourceGrants-annotated grant = hard violation (the +// related-resource machinery lost a row); dangling without the +// annotation = warning (tolerated connector behavior). Pebble-only +// (the scan rides the pebble key order); full syncs only. +// - I4 child-scheduling completeness: every stored resource carrying +// ChildResourceType must have had its (childType, parent) action +// scheduled. Check-only; skipped when the resources phase did not +// run in this process (a zero-row child listing leaves no store +// evidence that its action ran). Full syncs only. +// - I5 exclusion-group validation: the stored entitlement keyspace is +// validated post-collection (one default per group, one resource +// type per group, size cap). This REPLACES the former streaming +// validation entirely — it is engine-agnostic and covers replayed, +// fresh, and generated (static) entitlements uniformly. +// +// Evidence is monotone (existence bits, set-union scheduling records), +// so parallel workers and mixed stream/replay arrival orders yield +// identical verdicts; checks run at phase-quiesced points, so they are +// deterministic and idempotent under crash/resume. + +import ( + "context" + "fmt" + stdsync "sync" + + "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" + "go.uber.org/zap" + + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + "github.com/conductorone/baton-sdk/pkg/annotations" + "github.com/conductorone/baton-sdk/pkg/connectorstore" + "github.com/conductorone/baton-sdk/pkg/dotc1z" +) + +// sideEffectAnnotationCoverage is the enumerated coupling between +// connector annotations that imply syncer side effects and the mechanism +// that guarantees them independent of ingestion path. This map IS the +// audit that review rounds used to re-derive from memory; the +// completeness meta-test (ingest_invariants_test.go) fails when a listed +// annotation loses its mapping, and NEW side-effect-implying annotations +// must be added here with an invariant (or a documented exclusion) +// before they ship. +var sideEffectAnnotationCoverage = map[string]string{ + "c1.connector.v2.GrantExpandable": "I1: needs_expansion index probe at SyncGrantExpansionOp (parallel_syncer.go)", + "c1.connector.v2.ExternalResourceMatch": "I2: engine existence-bit repair at SyncExternalResourcesOp", + "c1.connector.v2.ExternalResourceMatchAll": "I2: engine existence-bit repair at SyncExternalResourcesOp", + "c1.connector.v2.ExternalResourceMatchID": "I2: engine existence-bit repair at SyncExternalResourcesOp", + "c1.connector.v2.InsertResourceGrants": "I3: grant→resource referential check post-collection; replay carries the row copy", + "c1.connector.v2.ChildResourceType": "I4: scheduled-set completeness check post-collection; replay carries the scheduling", + "c1.connector.v2.EntitlementExclusionGroup": "I5: stored-keyspace validation post-collection", +} + +// childScheduleSet is the monotone record of scheduled child-resource +// actions, keyed (childTypeID, parentTypeID, parentID). Guarded for +// parallel workers; in-memory only (see the I4 resume caveat). +type childScheduleSet struct { + mu stdsync.Mutex + m map[string]struct{} +} + +func childScheduleKey(childTypeID, parentTypeID, parentID string) string { + return childTypeID + "\x00" + parentTypeID + "\x00" + parentID +} + +func (c *childScheduleSet) record(childTypeID, parentTypeID, parentID string) { + c.mu.Lock() + defer c.mu.Unlock() + if c.m == nil { + c.m = make(map[string]struct{}) + } + c.m[childScheduleKey(childTypeID, parentTypeID, parentID)] = struct{}{} +} + +func (c *childScheduleSet) has(childTypeID, parentTypeID, parentID string) bool { + c.mu.Lock() + defer c.mu.Unlock() + _, ok := c.m[childScheduleKey(childTypeID, parentTypeID, parentID)] + return ok +} + +// repairExternalMatchFlag is invariant I2's repair: at the consuming seam +// (SyncExternalResourcesOp start), reconcile the in-memory flag with the +// engine's existence bit. Sound because the gated computation is a pure +// function of the store: running match processing when the store holds +// match-annotated grants is required; the flag is only an optimization +// that lets other syncs skip the scan. +func (s *syncer) repairExternalMatchFlag(ctx context.Context) { + if s.state.HasExternalResourcesGrants() { + return + } + facts, ok := s.store.(dotc1z.IngestFactStore) + if !ok { + return // SQLite: the response-loop arm is the mechanism. + } + has, err := facts.HasExternalMatchGrants(ctx) + if err != nil { + ctxzap.Extract(ctx).Warn("ingest invariant I2: could not read external-match fact; relying on stream-armed flag", zap.Error(err)) + return + } + if has { + ctxzap.Extract(ctx).Debug("ingest invariant I2: arming external-match processing from store-derived fact") + s.state.SetHasExternalResourcesGrants() + } +} + +// checkGrantResourceReferences is invariant I3: every distinct +// entitlement resource referenced by a grant must exist as a resource +// row. One seek per distinct resource (the grant keyspace leads with the +// entitlement resource), value reads only for dangling refs. +func (s *syncer) checkGrantResourceReferences(ctx context.Context) error { + if s.syncType != connectorstore.SyncTypeFull { + return nil + } + facts, ok := s.store.(dotc1z.IngestFactStore) + if !ok { + return nil // engine without the inspection surface + } + l := ctxzap.Extract(ctx) + return facts.ForEachDistinctGrantEntitlementResource(ctx, func(rt, rid string) error { + exists, err := facts.HasResourceRecord(ctx, rt, rid) + if err != nil { + return fmt.Errorf("ingest invariant I3: probing resource %s/%s: %w", rt, rid, err) + } + if exists { + return nil + } + annotated, err := facts.GrantsForEntResourceCarryInsertFact(ctx, rt, rid) + if err != nil { + return fmt.Errorf("ingest invariant I3: probing grant annotations for %s/%s: %w", rt, rid, err) + } + if annotated { + return fmt.Errorf( + "ingest invariant I3 violated: grants reference resource %s/%s via InsertResourceGrants but no resource row exists (related-resource insertion was lost)", + rt, rid) + } + // Tolerated today: connectors emitting grants for unlisted + // resources without InsertResourceGrants. Visible, not fatal. + l.Warn("ingest invariant I3: grants reference a resource that was never synced", + zap.String("resource_type_id", rt), + zap.String("resource_id", rid), + ) + return nil + }) +} + +// checkChildScheduling is invariant I4: every stored resource carrying a +// ChildResourceType annotation (and passing the resource-type filter) +// must have had its child action scheduled. Check-only — scheduling +// cannot be derived after the fact (an executed child action that +// returned zero rows leaves no store evidence). +func (s *syncer) checkChildScheduling(ctx context.Context) error { + if s.syncType != connectorstore.SyncTypeFull || !s.resourcesPhaseRanHere { + // Resumed past the resources phase: the scheduled set from the + // prior process is gone; the predicate is unverifiable. + return nil + } + syncResourceTypeMap := make(map[string]bool, len(s.syncResourceTypes)) + for _, rt := range s.syncResourceTypes { + syncResourceTypeMap[rt] = true + } + + var violations []string + pageToken := "" + for { + resp, err := s.store.ListResources(ctx, v2.ResourcesServiceListResourcesRequest_builder{ + PageToken: pageToken, + ActiveSyncId: s.getActiveSyncID(), + }.Build()) + if err != nil { + return fmt.Errorf("ingest invariant I4: listing resources: %w", err) + } + for _, r := range resp.GetList() { + for _, a := range r.GetAnnotations() { + if !a.MessageIs((*v2.ChildResourceType)(nil)) { + continue + } + crt := &v2.ChildResourceType{} + if err := a.UnmarshalTo(crt); err != nil { + return fmt.Errorf("ingest invariant I4: parsing ChildResourceType on %s/%s: %w", + r.GetId().GetResourceType(), r.GetId().GetResource(), err) + } + childType := crt.GetResourceTypeId() + if childType == "" { + continue + } + if len(s.syncResourceTypes) > 0 && !syncResourceTypeMap[childType] { + continue + } + if !s.childSchedule.has(childType, r.GetId().GetResourceType(), r.GetId().GetResource()) { + violations = append(violations, fmt.Sprintf("%s under %s/%s", + childType, r.GetId().GetResourceType(), r.GetId().GetResource())) + } + } + } + if pageToken = resp.GetNextPageToken(); pageToken == "" { + break + } + } + if len(violations) > 0 { + err := fmt.Errorf( + "ingest invariant I4 violated: %d child resource sync(s) were never scheduled: %v", + len(violations), violations) + if s.strictIngestionInvariants { + return err + } + ctxzap.Extract(ctx).Warn("ingest invariant I4 violated (lenient mode)", zap.Error(err)) + } + return nil +} + +// exclusionGroupTracker is the pure validation core behind invariant I5: +// one default per group, one resource type per group, at most +// maxEntitlementsPerExclusionGroup members. Error text intentionally +// matches the retired streaming validator. Recording the same +// entitlement twice is idempotent for the default check but counts twice +// toward the cap — callers feed each stored entitlement exactly once +// (the store keyspace holds one row per entitlement id). +type exclusionGroupTracker struct { + groups map[string]*exclusionGroupState +} + +type exclusionGroupState struct { + resourceTypeID string + defaultEntID string + count uint32 +} + +func (t *exclusionGroupTracker) record(ent *v2.Entitlement) error { + eg := &v2.EntitlementExclusionGroup{} + annos := annotations.Annotations(ent.GetAnnotations()) + ok, err := annos.Pick(eg) + if err != nil { + return fmt.Errorf("parsing exclusion group on %q: %w", ent.GetId(), err) + } + if !ok || eg.GetExclusionGroupId() == "" { + return nil + } + groupID := eg.GetExclusionGroupId() + if t.groups == nil { + t.groups = map[string]*exclusionGroupState{} + } + st := t.groups[groupID] + if st == nil { + st = &exclusionGroupState{} + t.groups[groupID] = st + } + rt := ent.GetResource().GetId().GetResourceType() + if st.resourceTypeID == "" { + st.resourceTypeID = rt + } else if st.resourceTypeID != rt { + return fmt.Errorf("exclusion group %q is used on multiple resource types (%q and %q); "+ + "exclusion groups may span resources but must be scoped to a single resource type", + groupID, st.resourceTypeID, rt) + } + if eg.GetIsDefault() { + if st.defaultEntID != "" && st.defaultEntID != ent.GetId() { + return fmt.Errorf("exclusion group %q has multiple default entitlements (%q and %q); "+ + "at most one entitlement per exclusion group may set is_default=true", + groupID, st.defaultEntID, ent.GetId()) + } + st.defaultEntID = ent.GetId() + } + st.count++ + if st.count > maxEntitlementsPerExclusionGroup { + return fmt.Errorf("exclusion group %q has too many entitlements (%d); "+ + "at most %d entitlements are allowed per exclusion group", + groupID, st.count, maxEntitlementsPerExclusionGroup) + } + return nil +} + +// validateStoredExclusionGroups is invariant I5: exclusion-group +// invariants validated over the STORED entitlement keyspace. This is the +// sole exclusion-group validation: it covers fresh, replayed, and +// generated (static) entitlements uniformly, on every engine, regardless +// of how rows arrived. +func (s *syncer) validateStoredExclusionGroups(ctx context.Context) error { + tracker := &exclusionGroupTracker{} + pageToken := "" + for { + resp, err := s.store.ListEntitlements(ctx, v2.EntitlementsServiceListEntitlementsRequest_builder{ + PageToken: pageToken, + ActiveSyncId: s.getActiveSyncID(), + }.Build()) + if err != nil { + return fmt.Errorf("ingest invariant I5: listing entitlements: %w", err) + } + for _, ent := range resp.GetList() { + if err := tracker.record(ent); err != nil { + return err + } + } + if pageToken = resp.GetNextPageToken(); pageToken == "" { + break + } + } + return nil +} + +// runIngestionInvariants evaluates the post-collection (check-flavored) +// invariants: I5, I4, I3, in that order (cheapest and most +// behavior-critical first). Runs after the action loop drains and before +// EndSync, so every ingestion path — stream, replay, resume re-runs, +// expansion, external-match processing — has finished writing. +// Idempotent: a resumed sync that re-reaches this point re-evaluates +// with the same verdict. +func (s *syncer) runIngestionInvariants(ctx context.Context) error { + if err := s.validateStoredExclusionGroups(ctx); err != nil { + return err + } + if err := s.checkChildScheduling(ctx); err != nil { + return err + } + return s.checkGrantResourceReferences(ctx) +} diff --git a/pkg/sync/ingest_invariants_test.go b/pkg/sync/ingest_invariants_test.go new file mode 100644 index 000000000..a817bce22 --- /dev/null +++ b/pkg/sync/ingest_invariants_test.go @@ -0,0 +1,37 @@ +package sync //nolint:revive,nolintlint // we can't change the package name for backwards compatibility + +import ( + "testing" + + "github.com/stretchr/testify/require" + "google.golang.org/protobuf/proto" + + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" +) + +// TestIngestInvariantAnnotationCoverage is the completeness meta-test for +// the ingestion-invariant registry: every connector annotation known to +// imply a syncer side effect must map to the mechanism that guarantees it +// regardless of ingestion path (stream, replay, or whatever comes next). +// Adding a new side-effect-implying annotation without extending +// sideEffectAnnotationCoverage — and the invariant behind it — fails +// here instead of in review round eleven. +func TestIngestInvariantAnnotationCoverage(t *testing.T) { + sideEffectAnnotations := []proto.Message{ + &v2.GrantExpandable{}, + &v2.ExternalResourceMatch{}, + &v2.ExternalResourceMatchAll{}, + &v2.ExternalResourceMatchID{}, + &v2.InsertResourceGrants{}, + &v2.ChildResourceType{}, + &v2.EntitlementExclusionGroup{}, + } + for _, msg := range sideEffectAnnotations { + name := string(msg.ProtoReflect().Descriptor().FullName()) + require.Contains(t, sideEffectAnnotationCoverage, name, + "side-effect-implying annotation %s has no registered ingestion invariant; "+ + "see docs/tasks/source-cache-ingestion-invariants.md", name) + } + require.Len(t, sideEffectAnnotationCoverage, len(sideEffectAnnotations), + "coverage map and the enumerated annotation set drifted; update both together") +} diff --git a/pkg/sync/progresslog/progresslog.go b/pkg/sync/progresslog/progresslog.go index 51be5a361..9286ddc98 100644 --- a/pkg/sync/progresslog/progresslog.go +++ b/pkg/sync/progresslog/progresslog.go @@ -32,16 +32,17 @@ const ( ) type ProgressLog struct { - resourceTypes int - resources map[string]int - entitlementsProgress map[string]int - lastEntitlementLog map[string]time.Time - grantsProgress map[string]int - lastGrantLog map[string]time.Time - grantsCountOnly map[string]bool - mu sync.RWMutex - l *zap.Logger - maxLogFrequency time.Duration + resourceTypes int + resources map[string]int + entitlementsProgress map[string]int + lastEntitlementLog map[string]time.Time + entitlementsCountOnly map[string]bool + grantsProgress map[string]int + lastGrantLog map[string]time.Time + grantsCountOnly map[string]bool + mu sync.RWMutex + l *zap.Logger + maxLogFrequency time.Duration // Optional cross-step db-size tracking for LogExpandProgress. Populated // via WithDBSizeProvider at construction time or SetDBSizeProvider after @@ -146,16 +147,17 @@ func (p *ProgressLog) SetDBSizeProvider(provider connectorstore.DBSizeProvider) func NewProgressCounts(ctx context.Context, opts ...Option) *ProgressLog { p := &ProgressLog{ - resources: make(map[string]int), - entitlementsProgress: make(map[string]int), - lastEntitlementLog: make(map[string]time.Time), - grantsProgress: make(map[string]int), - lastGrantLog: make(map[string]time.Time), - grantsCountOnly: make(map[string]bool), - l: ctxzap.Extract(ctx), - maxLogFrequency: defaultMaxLogFrequency, - mu: sync.RWMutex{}, - metricsHandler: metrics.NewNoOpHandler(ctx), + resources: make(map[string]int), + entitlementsProgress: make(map[string]int), + lastEntitlementLog: make(map[string]time.Time), + entitlementsCountOnly: make(map[string]bool), + grantsProgress: make(map[string]int), + lastGrantLog: make(map[string]time.Time), + grantsCountOnly: make(map[string]bool), + l: ctxzap.Extract(ctx), + maxLogFrequency: defaultMaxLogFrequency, + mu: sync.RWMutex{}, + metricsHandler: metrics.NewNoOpHandler(ctx), } for _, o := range opts { o(p) @@ -204,15 +206,19 @@ func (p *ProgressLog) LogResourcesProgress(ctx context.Context, resourceType str func (p *ProgressLog) LogEntitlementsProgress(ctx context.Context, resourceType string) { var entitlementsProgress, resources int var lastLogTime time.Time + var countOnly bool p.mu.RLock() entitlementsProgress = p.entitlementsProgress[resourceType] resources = p.resources[resourceType] lastLogTime = p.lastEntitlementLog[resourceType] + countOnly = p.entitlementsCountOnly[resourceType] p.mu.RUnlock() - if resources == 0 { - // if resuming sync, resource counts will be zero, so don't calculate percentage. just log every 10 seconds. + if resources == 0 || countOnly { + // Count-only: either a resumed sync (resource counts are zero) or a + // type-scoped entitlements type (no meaningful denominator). Log the + // raw synced count every log window; never compute a percentage. if time.Since(lastLogTime) > p.maxLogFrequency { p.l.Info("Syncing entitlements", zap.String("resource_type_id", resourceType), @@ -258,6 +264,26 @@ func (p *ProgressLog) LogEntitlementsProgress(ctx context.Context, resourceType } } +// SetEntitlementsCountOnly marks a resource type's entitlement progress as a +// plain count with no resources-covered denominator. Used for type-scoped +// entitlement enumeration (v2.TypeScopedEntitlements), where cursors don't +// map 1:1 to resources. +func (p *ProgressLog) SetEntitlementsCountOnly(resourceType string) { + p.mu.Lock() + defer p.mu.Unlock() + p.entitlementsCountOnly[resourceType] = true +} + +// EntitlementsProgress returns the current entitlement-coverage counter for a +// resource type. For per-resource types this is "resources covered"; for +// count-only (type-scoped) types it is the raw entitlement-row count. +// Exposed for tests pinning that accounting. +func (p *ProgressLog) EntitlementsProgress(resourceType string) int { + p.mu.RLock() + defer p.mu.RUnlock() + return p.entitlementsProgress[resourceType] +} + // SetGrantsCountOnly marks a resource type's grant progress as a plain // count with no resources-covered denominator. Used for type-scoped grant // enumeration (v2.TypeScopedGrants), where cursors don't map 1:1 to diff --git a/pkg/sync/source_cache.go b/pkg/sync/source_cache.go index 27c44ab44..375e3eb5f 100644 --- a/pkg/sync/source_cache.go +++ b/pkg/sync/source_cache.go @@ -415,46 +415,27 @@ func (s *syncer) beginSourceCachePage( zap.Int64("stale_skipped", res.StaleSkipped), ) } - // Replay bypasses the connector-response path that normally arms - // grant expansion (seeing GrantExpandable on returned rows), so a - // sync whose expandable pages all replay would silently skip the - // expansion phase without this. - if kind == sourcecache.RowKindGrants && res.NeedsExpansion && !s.dontExpandGrants { - s.state.SetNeedsExpansion() - } - // Same rationale for external-resource-match processing: the - // flag is otherwise set only from response rows, so a sync whose - // match-annotated pages all replay would skip the transformation - // step and silently drop every matched grant. - if kind == sourcecache.RowKindGrants && res.HasExternalResourceGrants { - s.state.SetHasExternalResourcesGrants() - } - // Child scheduling is response-row-driven (getSubResources runs - // over resp.GetList()), so replayed parents never reach it — - // schedule their child resource syncs here or a warm sync - // silently drops whole subtrees. + // Side effects for replayed rows are STORE-DERIVED, not armed + // here — the engine maintains the needs_expansion index and the + // external-match existence bit for replayed rows exactly as for + // fresh ones, and the ingestion invariants read the store at + // their consuming seams (expansion: the PendingExpansion probe at + // SyncGrantExpansionOp; external match: the fact probe at + // SyncExternalResourcesOp; exclusion groups: the post-collection + // keyspace validation). See + // docs/tasks/source-cache-ingestion-invariants.md. The one + // replay-carried side effect is child scheduling, which cannot be + // derived after the fact (a zero-row child listing leaves no + // store evidence), so replay carries the parents' child types: if kind == sourcecache.RowKindResources { for _, parent := range res.ChildResources { s.pushChildResourceActions(ctx, parent.ChildTypeIDs, parent.ResourceTypeID, parent.ResourceID) } } - // Exclusion-group validation runs on response rows, so replayed - // entitlements would silently stop participating in the - // one-default / single-resource-type / size-cap invariants — a - // violation split across a replayed and a fresh scope would pass - // a warm sync that a full resync hard-fails. - if kind == sourcecache.RowKindEntitlements { - for _, eg := range res.ExclusionGroups { - if err := s.recordEntitlementExclusionGroup(eg.Group, eg.EntitlementID, eg.ResourceTypeID); err != nil { - return ctx, nil, fmt.Errorf("source cache: replayed entitlement failed exclusion-group validation: %w", err) - } - } - } ctxzap.Extract(ctx).Debug("source cache replayed scope", zap.String("row_kind", string(kind)), zap.String("scope_hash", page.scopeHash), zap.Int64("rows", res.Rows), - zap.Bool("needs_expansion", res.NeedsExpansion), zap.Int("deleted_ids", len(page.deletedIDs)), zap.Int("deleted_principal_ids", len(page.deletedPrincipalIDs)), ) diff --git a/pkg/sync/source_cache_continuation_test.go b/pkg/sync/source_cache_continuation_test.go index 25d7454ae..4777e088d 100644 --- a/pkg/sync/source_cache_continuation_test.go +++ b/pkg/sync/source_cache_continuation_test.go @@ -24,6 +24,7 @@ import ( "github.com/conductorone/baton-sdk/pkg/logging" "github.com/conductorone/baton-sdk/pkg/sourcecache" "github.com/conductorone/baton-sdk/pkg/types" + et "github.com/conductorone/baton-sdk/pkg/types/entitlement" gt "github.com/conductorone/baton-sdk/pkg/types/grant" rs "github.com/conductorone/baton-sdk/pkg/types/resource" @@ -778,3 +779,144 @@ func TestSourceCacheContinuation_AskWithoutOfferFails(t *testing.T) { require.Error(t, err) require.Contains(t, err.Error(), "no offer") } + +// multiPagePlannerEntitlementsMock mirrors multiPagePlannerMock for +// TypeScopedEntitlements: a multi-page planner that asks once per page +// before spawning a real cursor, asserting the routing marker survives +// every re-invocation. +type multiPagePlannerEntitlementsMock struct { + *mockConnector + + mu sync.Mutex + groupIDs []string + resByID map[string]*v2.Resource + asks int + cursorServes int + spawns int + markerMissing bool + answersScope string + answersPresent int +} + +func (mc *multiPagePlannerEntitlementsMock) Validate(context.Context, *v2.ConnectorServiceValidateRequest, ...grpc.CallOption) (*v2.ConnectorServiceValidateResponse, error) { + return v2.ConnectorServiceValidateResponse_builder{ + Annotations: annotations.New(v2.SourceCacheCapability_builder{ + Mode: v2.SourceCacheCapability_MODE_READ_WRITE, + }.Build()), + }.Build(), nil +} + +func (mc *multiPagePlannerEntitlementsMock) ListEntitlements( + _ context.Context, in *v2.EntitlementsServiceListEntitlementsRequest, _ ...grpc.CallOption, +) (*v2.EntitlementsServiceListEntitlementsResponse, error) { + if in.GetResource().GetId().GetResourceType() != typeScopedEntGroupRT.GetId() { + return v2.EntitlementsServiceListEntitlementsResponse_builder{}.Build(), nil + } + reqAnnos := annotations.Annotations(in.GetAnnotations()) + mc.mu.Lock() + defer mc.mu.Unlock() + if !reqAnnos.Contains(&v2.TypeScopedEntitlements{}) { + mc.markerMissing = true + } + + tok := in.GetPageToken() + if tok == "cursor" { + mc.cursorServes++ + ents := make([]*v2.Entitlement, 0, len(mc.groupIDs)) + for _, gid := range mc.groupIDs { + ents = append(ents, et.NewAssignmentEntitlement(mc.resByID[gid], "member", et.WithGrantableTo(userResourceType))) + } + return v2.EntitlementsServiceListEntitlementsResponse_builder{List: ents}.Build(), nil + } + + page := 0 + if tok != "" { + if _, err := fmt.Sscanf(tok, "plan=%d", &page); err != nil { + return nil, fmt.Errorf("bad token %q", tok) + } + } + scope := fmt.Sprintf("ent-chunks/page-%d", page) + + answersMsg := &v2.SourceCacheLookupAnswers{} + hasAnswers, err := reqAnnos.Pick(answersMsg) + if err != nil { + return nil, err + } + if !hasAnswers && reqAnnos.Contains(&v2.SourceCacheLookupOffer{}) { + mc.asks++ + return v2.EntitlementsServiceListEntitlementsResponse_builder{ + Annotations: annotations.New(sourcecache.AskProto([]sourcecache.Query{ + {RowKind: sourcecache.RowKindEntitlements, ScopeHash: scope}, + })), + }.Build(), nil + } + if hasAnswers { + mc.answersPresent++ + answers, answersErr := sourcecache.AnswersFromProto(answersMsg) + if answersErr != nil { + return nil, answersErr + } + for _, a := range answers { + if a.ScopeHash == scope { + mc.answersScope = scope + } + } + } + + if page == plannerPages-1 { + mc.spawns++ + return v2.EntitlementsServiceListEntitlementsResponse_builder{ + Annotations: annotations.New(v2.SpawnCursors_builder{PageTokens: []string{"cursor"}}.Build()), + }.Build(), nil + } + return v2.EntitlementsServiceListEntitlementsResponse_builder{ + NextPageToken: fmt.Sprintf("plan=%d", page+1), + }.Build(), nil +} + +func (mc *multiPagePlannerEntitlementsMock) ListGrants(context.Context, *v2.GrantsServiceListGrantsRequest, ...grpc.CallOption) (*v2.GrantsServiceListGrantsResponse, error) { + return v2.GrantsServiceListGrantsResponse_builder{}.Build(), nil +} + +func TestSourceCacheContinuation_MultiPagePlannerEntitlementsBouncesPerRequest(t *testing.T) { + ctx, err := logging.Init(context.Background()) + require.NoError(t, err) + tmpDir := t.TempDir() + + mc := &multiPagePlannerEntitlementsMock{ + mockConnector: newMockConnector(), + resByID: map[string]*v2.Resource{}, + } + mc.rtDB = append(mc.rtDB, typeScopedEntGroupRT, userResourceType) + u, err := rs.NewUserResource("user-00", userResourceType, "user-00", nil, rs.WithAnnotation(&v2.SkipEntitlementsAndGrants{})) + require.NoError(t, err) + mc.AddResource(ctx, u) + mc.resByID["user-00"] = u + for i := 0; i < 2; i++ { + gid := fmt.Sprintf("group-%02d", i) + g, err := rs.NewGroupResource(gid, typeScopedEntGroupRT, gid, nil) + require.NoError(t, err) + mc.AddResource(ctx, g) + mc.resByID[gid] = g + mc.groupIDs = append(mc.groupIDs, gid) + } + + sync1 := filepath.Join(tmpDir, "sync1.c1z") + require.NoError(t, runContinuationSync(ctx, t, mc, sync1, "", tmpDir, 0)) + require.Zero(t, mc.asks) + require.Equal(t, 1, mc.spawns) + require.Equal(t, 1, mc.cursorServes) + ents1 := listEntitlementsInFile(ctx, t, sync1) + require.Len(t, ents1, 2) + + sync2 := filepath.Join(tmpDir, "sync2.c1z") + require.NoError(t, runContinuationSync(ctx, t, mc, sync2, sync1, tmpDir, 0), + "a multi-page entitlements planner asking once per page must not trip the per-request bounce cap") + require.Equal(t, plannerPages, mc.asks, "one ask per planning page") + require.Equal(t, plannerPages, mc.answersPresent, "every re-invoked planning page carries answers") + require.NotEmpty(t, mc.answersScope, "answers must cover the asked scope") + require.False(t, mc.markerMissing, "TypeScopedEntitlements routing marker must survive on every request, including re-invokes") + require.Equal(t, 2, mc.spawns, "the asking planner page spawns exactly once per sync") + require.Equal(t, 2, mc.cursorServes, "spawned cursor executes exactly once per sync") + requireSameEntitlementIDs(t, ents1, listEntitlementsInFile(ctx, t, sync2), "warm planner sync vs cold") +} diff --git a/pkg/sync/source_cache_equivalence_test.go b/pkg/sync/source_cache_equivalence_test.go index 41afd3dd5..fd83bf085 100644 --- a/pkg/sync/source_cache_equivalence_test.go +++ b/pkg/sync/source_cache_equivalence_test.go @@ -59,8 +59,10 @@ package sync //nolint:revive,nolintlint // we can't change the package name for // parity test in source_cache_sideeffects_test.go); // - the ask/answer lookup continuation topology (covered by // source_cache_continuation_test.go / _e2e_test.go); -// - SpawnCursors / type-scoped grants (spawn_cursors_test.go, -// type_scoped_grants_test.go); +// - SpawnCursors / type-scoped grants and entitlements +// (spawn_cursors_test.go, type_scoped_grants_test.go, +// type_scoped_entitlements_test.go); the team topology below also +// exercises both type-scoped shapes in the differential chain; // - compaction in the warm chain (covered separately by // pkg/synccompactor's TestCompactedFileIsColdReplaySource — an import // cycle keeps the compactor out of this package; the contract is that @@ -118,13 +120,14 @@ var ( equivOrgRT = v2.ResourceType_builder{Id: "eq_org", DisplayName: "Org"}.Build() equivProjectRT = v2.ResourceType_builder{Id: "eq_project", DisplayName: "Project"}.Build() equivRepoRT = v2.ResourceType_builder{Id: "eq_repo", DisplayName: "Repo"}.Build() - // equivTeamRT is TYPE-SCOPED: no per-resource grants pages; one - // planner call spawns chunk cursors (SpawnCursors), each chunk a - // scope covering two teams — the Entra/Okta planner shape. + // equivTeamRT is TYPE-SCOPED for both grants and entitlements: no + // per-resource pages; one planner call per phase spawns chunk + // cursors (SpawnCursors), each chunk a scope covering two teams — + // the Entra/Okta planner shape. equivTeamRT = v2.ResourceType_builder{ Id: "eq_team", DisplayName: "Team", - Annotations: annotations.New(&v2.TypeScopedGrants{}), + Annotations: annotations.New(&v2.TypeScopedGrants{}, &v2.TypeScopedEntitlements{}), }.Build() ) @@ -886,16 +889,7 @@ func (c *equivConnector) listEntitlementsInner( ) (*v2.EntitlementsServiceListEntitlementsResponse, error) { rid := in.GetResource().GetId() if rid.GetResourceType() == equivTeamRT.GetId() { - // Team entitlements are unscoped: served fresh every sync. - c.mu.Lock() - _, ok := c.model.teams[rid.GetResource()] - c.mu.Unlock() - if !ok { - return v2.EntitlementsServiceListEntitlementsResponse_builder{}.Build(), nil - } - return v2.EntitlementsServiceListEntitlementsResponse_builder{ - List: []*v2.Entitlement{equivMemberEnt(equivTeamResource(rid.GetResource()), false)}, - }.Build(), nil + return c.teamEntitlementsPage(ctx, lookup, in.GetPageToken()) } if rid.GetResourceType() != groupResourceType.GetId() { return v2.EntitlementsServiceListEntitlementsResponse_builder{}.Build(), nil @@ -963,6 +957,56 @@ func (c *equivConnector) ListGrants(ctx context.Context, in *v2.GrantsServiceLis return resp, err } +// teamEntitlementsPage mirrors teamGrantsPage for type-scoped +// entitlements: planning SpawnCursors, then one fresh-or-304 scope per +// chunk of two teams. Entitlement rows are stable (one "member" per +// team); version stays 0 so warm rounds are pure 304s after the cold +// fetch — exercising type-scoped entitlement replay in the differential +// chain without coupling to membership churn. +func (c *equivConnector) teamEntitlementsPage(ctx context.Context, lookup sourcecache.Lookup, pageToken string) (*v2.EntitlementsServiceListEntitlementsResponse, error) { + if pageToken == "" { + c.mu.Lock() + chunks := len(c.model.teamChunkVersions) + c.mu.Unlock() + c.count(func(ct *equivCounters) { ct.plannerPages++ }) + tokens := make([]string, 0, chunks) + for i := 0; i < chunks; i++ { + tokens = append(tokens, fmt.Sprintf("ent-chunk:%d", i)) + } + return v2.EntitlementsServiceListEntitlementsResponse_builder{ + Annotations: annotations.New(v2.SpawnCursors_builder{PageTokens: tokens}.Build()), + }.Build(), nil + } + var chunk int + if _, err := fmt.Sscanf(pageToken, "ent-chunk:%d", &chunk); err != nil { + return nil, fmt.Errorf("teamEntitlementsPage: bad page token %q: %w", pageToken, err) + } + c.mu.Lock() + teamIDs := c.model.teamsInChunk(chunk) + c.mu.Unlock() + + scope := fmt.Sprintf("ents:team-chunk:%d", chunk) + verdict, err := c.revalidate(ctx, lookup, sourcecache.RowKindEntitlements, sourcecache.HashScope(scope), 0) + if err != nil { + return nil, err + } + if verdict == equivCurrent { + c.count(func(ct *equivCounters) { ct.chunkPages304++ }) + return v2.EntitlementsServiceListEntitlementsResponse_builder{ + Annotations: equivReplayAnnos(scope, 0), + }.Build(), nil + } + c.count(func(ct *equivCounters) { ct.chunkPagesFresh++ }) + rows := make([]*v2.Entitlement, 0, len(teamIDs)) + for _, tid := range teamIDs { + rows = append(rows, equivMemberEnt(equivTeamResource(tid), false)) + } + return v2.EntitlementsServiceListEntitlementsResponse_builder{ + List: rows, + Annotations: equivScopeAnnos(scope, 0), + }.Build(), nil +} + // teamGrantsPage is the type-scoped planner shape: the planning call // (empty page token) emits SpawnCursors — one token per chunk of two // teams — and each spawned cursor serves its chunk as a fresh-or-304 @@ -1676,6 +1720,10 @@ func runEquivSync(ctx context.Context, t *testing.T, cc types.ConnectorClient, p WithC1ZPath(path), WithStorageEngine(c1zstore.EnginePebble), WithTmpDir(tmpDir), + // Check-only ingestion invariants hard-fail in the harness: a + // violation must name itself here before the differential + // comparison would catch its downstream divergence. + WithStrictIngestionInvariants(), } if prevPath != "" { opts = append(opts, WithPreviousSyncC1ZPath(prevPath)) diff --git a/pkg/sync/source_cache_sideeffects_test.go b/pkg/sync/source_cache_sideeffects_test.go index 811befa6f..6e11f5da7 100644 --- a/pkg/sync/source_cache_sideeffects_test.go +++ b/pkg/sync/source_cache_sideeffects_test.go @@ -38,7 +38,7 @@ func runSourceCacheSyncClient(ctx context.Context, t *testing.T, cc types.Connec dotc1z.WithTmpDir(tmpDir), ) require.NoError(t, err) - opts := []SyncOpt{WithConnectorStore(store), WithTmpDir(tmpDir)} + opts := []SyncOpt{WithConnectorStore(store), WithTmpDir(tmpDir), WithStrictIngestionInvariants()} if prevPath != "" { opts = append(opts, WithPreviousSyncC1ZPath(prevPath)) } diff --git a/pkg/sync/source_cache_test.go b/pkg/sync/source_cache_test.go index 1f2714452..54db60409 100644 --- a/pkg/sync/source_cache_test.go +++ b/pkg/sync/source_cache_test.go @@ -169,7 +169,7 @@ func runSourceCacheSync(ctx context.Context, t *testing.T, mc *sourceCacheMockCo dotc1z.WithTmpDir(tmpDir), ) require.NoError(t, err) - opts := []SyncOpt{WithConnectorStore(store), WithTmpDir(tmpDir)} + opts := []SyncOpt{WithConnectorStore(store), WithTmpDir(tmpDir), WithStrictIngestionInvariants()} if prevPath != "" { opts = append(opts, WithPreviousSyncC1ZPath(prevPath)) } diff --git a/pkg/sync/state.go b/pkg/sync/state.go index 34ba47a85..70c4e38f0 100644 --- a/pkg/sync/state.go +++ b/pkg/sync/state.go @@ -220,10 +220,11 @@ type Action struct { // Spawned marks an action enqueued by a connector's SpawnCursors // annotation (a sibling cursor) rather than by the syncer's own - // planners. Progress accounting uses it: per-resource grant coverage - // counts a resource once, when its ORIGIN action's chain ends — - // spawned siblings (and their NextPage continuations, which inherit - // the action) don't count, or one resource would count N times. + // planners. Progress accounting uses it: per-resource grant / + // entitlement coverage counts a resource once, when its ORIGIN + // action's chain ends — spawned siblings (and their NextPage + // continuations, which inherit the action) don't count, or one + // resource would count N times. // omitempty keeps old checkpointed sync tokens decoding unchanged. Spawned bool `json:"spawned,omitempty"` } diff --git a/pkg/sync/syncer.go b/pkg/sync/syncer.go index 548d05d31..e1541d7f8 100644 --- a/pkg/sync/syncer.go +++ b/pkg/sync/syncer.go @@ -143,35 +143,44 @@ type syncer struct { store c1zstore.Store externalResourceReader connectorstore.Reader previousSyncReader connectorstore.Reader - connector types.ConnectorClient - state State - runDuration time.Duration - transitionHandler func(s Action) - progressHandler func(p *Progress) - tmpDir string - storageEngine c1zstore.Engine - skipFullSync bool - lastCheckPointTime time.Time - counts *progresslog.ProgressLog - targetedSyncResources []*v2.Resource - onlyExpandGrants bool - dontExpandGrants bool - syncID string - skipEGForResourceType syncMap[string, bool] - skipEntitlementsForResourceType syncMap[string, bool] - typeScopedGrantsForResourceType syncMap[string, bool] - skipEntitlementsAndGrants bool - skipGrants bool - resourceTypeTraits syncMap[string, []v2.ResourceType_Trait] - syncType connectorstore.SyncType - injectSyncIDAnnotation bool - setSessionStore sessions.SetSessionStore - syncResourceTypes []string - workerCount int // If 1, sync is sequential (default). If > 1, sync operations are done in parallel. - metricsHandler metrics.Handler - syncIdentity uotel.SyncIdentity - recordStats bool - sourceCache syncerSourceCache + // Ingestion-invariant state (see ingest_invariants.go): + // childSchedule is the monotone record backing invariant I4; + // resourcesPhaseRanHere gates I4 to processes that actually ran the + // resources phase; strictIngestionInvariants promotes I4 from + // warn to hard-fail (tests and the equivalence harness set it). + childSchedule childScheduleSet + resourcesPhaseRanHere bool + strictIngestionInvariants bool + connector types.ConnectorClient + state State + runDuration time.Duration + transitionHandler func(s Action) + progressHandler func(p *Progress) + tmpDir string + storageEngine c1zstore.Engine + skipFullSync bool + lastCheckPointTime time.Time + counts *progresslog.ProgressLog + targetedSyncResources []*v2.Resource + onlyExpandGrants bool + dontExpandGrants bool + syncID string + skipEGForResourceType syncMap[string, bool] + skipEntitlementsForResourceType syncMap[string, bool] + typeScopedGrantsForResourceType syncMap[string, bool] + typeScopedEntitlementsForResourceType syncMap[string, bool] + skipEntitlementsAndGrants bool + skipGrants bool + resourceTypeTraits syncMap[string, []v2.ResourceType_Trait] + syncType connectorstore.SyncType + injectSyncIDAnnotation bool + setSessionStore sessions.SetSessionStore + syncResourceTypes []string + workerCount int // If 1, sync is sequential (default). If > 1, sync operations are done in parallel. + metricsHandler metrics.Handler + syncIdentity uotel.SyncIdentity + recordStats bool + sourceCache syncerSourceCache } var _ Syncer = (*syncer)(nil) @@ -579,60 +588,6 @@ func (s *syncer) handleProgress(ctx context.Context, a *Action, c int) { // single exclusion_group_id. Phase 1 limit. const maxEntitlementsPerExclusionGroup = 50 -// recordEntitlementExclusionGroup enforces the invariants on an exclusion -// group membership: a given exclusion_group_id must stay within one resource -// type, a group may have at most one entitlement marked is_default, and a group -// may contain at most maxEntitlementsPerExclusionGroup entitlements. Empty -// group ids are treated as "no exclusion group" and skipped. -func (s *syncer) recordEntitlementExclusionGroup(eg *v2.EntitlementExclusionGroup, entitlementID, resourceTypeID string) error { - groupID := eg.GetExclusionGroupId() - if groupID == "" { - return nil - } - if existing, conflict := s.state.CheckAndSetExclusionGroupResourceType(groupID, resourceTypeID); conflict { - return fmt.Errorf("exclusion group %q is used on multiple resource types (%q and %q); "+ - "exclusion groups may span resources but must be scoped to a single resource type", - groupID, existing, resourceTypeID) - } - if eg.GetIsDefault() { - if existing, conflict := s.state.CheckAndSetExclusionGroupDefault(groupID, entitlementID); conflict { - return fmt.Errorf("exclusion group %q has multiple default entitlements (%q and %q); "+ - "at most one entitlement per exclusion group may set is_default=true", - groupID, existing, entitlementID) - } - } - if count := s.state.IncrementExclusionGroupCount(groupID); count > maxEntitlementsPerExclusionGroup { - return fmt.Errorf("exclusion group %q has too many entitlements (%d); "+ - "at most %d entitlements are allowed per exclusion group", - groupID, count, maxEntitlementsPerExclusionGroup) - } - return nil -} - -// validateEntitlementExclusionGroups picks the exclusion group annotation off -// each entitlement (if present) and forwards to recordEntitlementExclusionGroup. -// Use this on lists of entitlements that may independently carry exclusion -// group annotations (e.g., the dynamic ListEntitlements path); callers that -// already have the annotation in hand should call recordEntitlementExclusionGroup -// directly to avoid the per-entitlement Pick. -func (s *syncer) validateEntitlementExclusionGroups(ents []*v2.Entitlement) error { - for _, ent := range ents { - eg := &v2.EntitlementExclusionGroup{} - entAnnos := annotations.Annotations(ent.GetAnnotations()) - ok, err := entAnnos.Pick(eg) - if err != nil { - return err - } - if !ok { - continue - } - if err := s.recordEntitlementExclusionGroup(eg, ent.GetId(), ent.GetResource().GetId().GetResourceType()); err != nil { - return err - } - } - return nil -} - // nextPageOrFinishAction updates the action with the next page token, or if there is no next page, finishes the action. // It also pushes any child actions before updating/finishing the action. // This is useful for pagination, and for actions that create other actions. @@ -882,6 +837,14 @@ func (s *syncer) Sync(ctx context.Context) error { return s.returnSyncError(l, span, err) } + // Post-collection ingestion invariants (ingest_invariants.go): every + // ingestion path has finished writing, so the store-derived + // definitions of the sync's side effects are checkable. Runs before + // EndSync so a violating sync is never sealed as complete. + if err := s.runIngestionInvariants(ctx); err != nil { + return s.returnSyncError(l, span, err) + } + // Force a checkpoint to clear completed actions & entitlement graph in sync_token. s.state.ClearEntitlementGraph(ctx) s.state.ClearExclusionGroupTracking(ctx) @@ -1124,6 +1087,10 @@ func (s *syncer) pushChildResourceActions(ctx context.Context, childTypeIDs []st if len(s.syncResourceTypes) > 0 && !slices.Contains(s.syncResourceTypes, childTypeID) { continue } + // Monotone evidence for ingestion invariant I4 (see + // ingest_invariants.go): record the pair regardless of which + // ingestion path discovered the parent. + s.childSchedule.record(childTypeID, parentTypeID, parentID) s.state.PushAction(ctx, Action{ Op: SyncResourcesOp, ResourceTypeID: childTypeID, @@ -1205,18 +1172,29 @@ func (s *syncer) SyncTargetedResource(ctx context.Context, action *Action) error s.state.FinishAction(ctx, action) - // Actions happen in reverse order. We want to sync child resources, then entitlements, then grants + // Actions happen in reverse order. We want to sync child resources, then entitlements, then grants. + // Type-scoped types are excluded from per-resource fan-out here as in + // the full-sync planners: SyncTargetedResource must not enqueue + // ListGrants/ListEntitlements for a resource whose type is annotated + // TypeScopedGrants / TypeScopedEntitlements (those paths use a single + // type-scoped action owned by the full phase planners). shouldSkipGrants, err := s.shouldSkipGrants(ctx, resource) if err != nil { return err } if !shouldSkipGrants { - s.state.PushAction(ctx, Action{ - Op: SyncGrantsOp, - ResourceTypeID: resourceTypeID, - ResourceID: resourceID, - }) + typeScopedGrants, err := s.resourceTypeHasTypeScopedGrants(ctx, resourceTypeID) + if err != nil { + return err + } + if !typeScopedGrants { + s.state.PushAction(ctx, Action{ + Op: SyncGrantsOp, + ResourceTypeID: resourceTypeID, + ResourceID: resourceID, + }) + } } shouldSkipEnts, err := s.shouldSkipEntitlements(ctx, resource) @@ -1225,11 +1203,17 @@ func (s *syncer) SyncTargetedResource(ctx context.Context, action *Action) error } if !shouldSkipEnts { - s.state.PushAction(ctx, Action{ - Op: SyncEntitlementsOp, - ResourceTypeID: resourceTypeID, - ResourceID: resourceID, - }) + typeScopedEnts, err := s.resourceTypeHasTypeScopedEntitlements(ctx, resourceTypeID) + if err != nil { + return err + } + if !typeScopedEnts { + s.state.PushAction(ctx, Action{ + Op: SyncEntitlementsOp, + ResourceTypeID: resourceTypeID, + ResourceID: resourceID, + }) + } } err = s.getSubResources(ctx, resource) @@ -1252,6 +1236,10 @@ func (s *syncer) SyncResources(ctx context.Context, action *Action) error { if action.PageToken == "" { ctxzap.Extract(ctx).Info("Syncing resources...") s.handleInitialActionForStep(ctx, *action) + // Ingestion invariant I4 is only verifiable when the + // resources phase started in THIS process (the scheduled + // set is in-memory); see ingest_invariants.go. + s.resourcesPhaseRanHere = true } resp, err := s.store.ListResourceTypes(ctx, v2.ResourceTypesServiceListResourceTypesRequest_builder{ @@ -1537,6 +1525,9 @@ func (s *syncer) shouldSkipEntitlements(ctx context.Context, r *v2.Resource) (bo // SyncEntitlements fetches the entitlements from the connector. It first lists each resource from the datastore, // and pushes an action to fetch the entitlements for each resource. +// Resource types annotated with TypeScopedEntitlements are excluded from the per-resource fan-out and +// get a single type-scoped action instead (empty ResourceID); the connector enumerates the whole +// type, optionally spawning additional cursors via the SpawnCursors annotation. func (s *syncer) SyncEntitlements(ctx context.Context, action *Action) error { ctx, span := uotel.StartWithLink(ctx, tracer, "syncer.SyncEntitlements") uotel.SetSyncIdentityAttrs(ctx, span) @@ -1544,11 +1535,22 @@ func (s *syncer) SyncEntitlements(ctx context.Context, action *Action) error { defer func() { uotel.EndSpanWithError(span, err) }() if action.ResourceTypeID == "" && action.ResourceID == "" { + actions := make([]Action, 0) pageToken := action.PageToken if pageToken == "" { ctxzap.Extract(ctx).Info("Syncing entitlements...") s.handleInitialActionForStep(ctx, *action) + + // One type-scoped action per annotated resource type, enqueued + // exactly once (the planner's first page). + typeScoped, err := s.typeScopedEntitlementsResourceTypes(ctx) + if err != nil { + return fmt.Errorf("sync-entitlements: error listing type-scoped resource types: %w", err) + } + for _, rtID := range typeScoped { + actions = append(actions, Action{Op: SyncEntitlementsOp, ResourceTypeID: rtID}) + } } resp, err := s.store.ListResources(ctx, v2.ResourcesServiceListResourcesRequest_builder{ @@ -1559,7 +1561,6 @@ func (s *syncer) SyncEntitlements(ctx context.Context, action *Action) error { return err } - actions := make([]Action, 0) for _, r := range resp.GetList() { shouldSkipEntitlements, err := s.shouldSkipEntitlements(ctx, r) if err != nil { @@ -1568,6 +1569,17 @@ func (s *syncer) SyncEntitlements(ctx context.Context, action *Action) error { if shouldSkipEntitlements { continue } + + // Types with type-scoped entitlements are excluded from the + // per-resource fan-out; their single action is enqueued above. + typeScoped, err := s.resourceTypeHasTypeScopedEntitlements(ctx, r.GetId().GetResourceType()) + if err != nil { + return err + } + if typeScoped { + continue + } + actions = append(actions, Action{Op: SyncEntitlementsOp, ResourceID: r.GetId().GetResource(), ResourceTypeID: r.GetId().GetResourceType()}) } @@ -1582,30 +1594,97 @@ func (s *syncer) SyncEntitlements(ctx context.Context, action *Action) error { return nil } +// resourceTypeHasTypeScopedEntitlements reports (cached per sync) whether the +// resource type carries the TypeScopedEntitlements annotation. +func (s *syncer) resourceTypeHasTypeScopedEntitlements(ctx context.Context, resourceTypeID string) (bool, error) { + if v, ok := s.typeScopedEntitlementsForResourceType.Load(resourceTypeID); ok { + return v, nil + } + rt, err := s.store.GetResourceType(ctx, reader_v2.ResourceTypesReaderServiceGetResourceTypeRequest_builder{ + ResourceTypeId: resourceTypeID, + }.Build()) + if err != nil { + return false, err + } + rtAnnos := annotations.Annotations(rt.GetResourceType().GetAnnotations()) + typeScoped := rtAnnos.Contains(&v2.TypeScopedEntitlements{}) + s.typeScopedEntitlementsForResourceType.Store(resourceTypeID, typeScoped) + return typeScoped, nil +} + +// typeScopedEntitlementsResourceTypes lists every synced resource type annotated +// with TypeScopedEntitlements. +func (s *syncer) typeScopedEntitlementsResourceTypes(ctx context.Context) ([]string, error) { + var out []string + pageToken := "" + for { + resp, err := s.store.ListResourceTypes(ctx, v2.ResourceTypesServiceListResourceTypesRequest_builder{ + PageToken: pageToken, + }.Build()) + if err != nil { + return nil, err + } + for _, rt := range resp.GetList() { + rtAnnos := annotations.Annotations(rt.GetAnnotations()) + typeScoped := rtAnnos.Contains(&v2.TypeScopedEntitlements{}) + s.typeScopedEntitlementsForResourceType.Store(rt.GetId(), typeScoped) + if typeScoped { + out = append(out, rt.GetId()) + } + } + pageToken = resp.GetNextPageToken() + if pageToken == "" { + return out, nil + } + } +} + // syncEntitlementsForResource fetches the entitlements for a specific resource from the connector. +// An action with an empty ResourceID is a TYPE-SCOPED entitlements cursor: the connector +// enumerates entitlements for the whole resource type (no single resource backs the call), +// and may spawn sibling cursors via the SpawnCursors response annotation. // No span here: only call site is SyncEntitlements, which already owns a span. func (s *syncer) syncEntitlementsForResource(ctx context.Context, action *Action) error { + typeScoped := action.ResourceID == "" resourceID := v2.ResourceId_builder{ ResourceType: action.ResourceTypeID, Resource: action.ResourceID, }.Build() - resourceResponse, err := s.store.GetResource(ctx, reader_v2.ResourcesReaderServiceGetResourceRequest_builder{ - ResourceId: resourceID, - }.Build()) - if err != nil { - return err - } - resource := resourceResponse.GetResource() + var resource *v2.Resource + var reqAnnos annotations.Annotations + if typeScoped { + // Wire validation requires a non-empty resource id, so the stub is + // self-referential ({type, type}) and the request carries the + // TypeScopedEntitlements annotation as the routing marker. + resource = v2.Resource_builder{ + Id: v2.ResourceId_builder{ + ResourceType: action.ResourceTypeID, + Resource: action.ResourceTypeID, + }.Build(), + }.Build() + reqAnnos.Update(&v2.TypeScopedEntitlements{}) + } else { + resourceResponse, err := s.store.GetResource(ctx, reader_v2.ResourcesReaderServiceGetResourceRequest_builder{ + ResourceId: resourceID, + }.Build()) + if err != nil { + return err + } + resource = resourceResponse.GetResource() + } var resp *v2.EntitlementsServiceListEntitlementsResponse - err = s.withSourceCacheContinuation(ctx, "sync-entitlements", func(extra annotations.Annotations, attempt int) (listAttempt, error) { + err := s.withSourceCacheContinuation(ctx, "sync-entitlements", func(extra annotations.Annotations, attempt int) (listAttempt, error) { + annos := make(annotations.Annotations, 0, len(reqAnnos)+len(extra)) + annos = append(annos, reqAnnos...) + annos = append(annos, extra...) start := time.Now() r, err := s.connector.ListEntitlements(ctx, v2.EntitlementsServiceListEntitlementsRequest_builder{ Resource: resource, PageToken: action.PageToken, ActiveSyncId: s.getActiveSyncID(), - Annotations: extra, + Annotations: annos, }.Build()) s.observeConnectorCall(ctx, continuationCallMethod("list-entitlements", attempt), start, action.ResourceTypeID, action.ResourceID) if err != nil { @@ -1622,11 +1701,8 @@ func (s *syncer) syncEntitlementsForResource(ctx context.Context, action *Action if err != nil { return err } - if err := s.validateEntitlementExclusionGroups(resp.GetList()); err != nil { - return err - } - - putCtx, scPage, err := s.beginSourceCachePage(ctx, sourcecache.RowKindEntitlements, annotations.Annotations(resp.GetAnnotations()), len(resp.GetList())) + respAnnos := annotations.Annotations(resp.GetAnnotations()) + putCtx, scPage, err := s.beginSourceCachePage(ctx, sourcecache.RowKindEntitlements, respAnnos, len(resp.GetList())) if err != nil { return err } @@ -1641,12 +1717,63 @@ func (s *syncer) syncEntitlementsForResource(ctx context.Context, action *Action } s.handleProgress(ctx, action, len(resp.GetList())) - if resp.GetNextPageToken() == "" { - s.counts.AddEntitlementsProgress(resourceID.ResourceType, 1) + + if typeScoped { + // Cursors don't map 1:1 to resources, so the per-resource + // "N of M resources covered" accounting is meaningless here. + s.counts.SetEntitlementsCountOnly(resourceID.GetResourceType()) + s.counts.AddEntitlementsProgress(resourceID.GetResourceType(), len(resp.GetList())) + s.counts.LogEntitlementsProgress(ctx, resourceID.GetResourceType()) + } else if resp.GetNextPageToken() == "" && !action.Spawned { + // A resource counts as covered exactly once: when its ORIGIN + // action's page chain ends. Spawned sibling cursors must not count. + s.counts.AddEntitlementsProgress(resourceID.GetResourceType(), 1) s.counts.LogEntitlementsProgress(ctx, resourceID.GetResourceType()) } - return s.nextPageOrFinishAction(ctx, action, resp.GetNextPageToken()) + // SpawnCursors: enqueue sibling cursors (type-scoped shards or + // per-resource parallel pages). Same contracts as syncGrantsForResource. + spawn := &v2.SpawnCursors{} + hasSpawn, err := respAnnos.Pick(spawn) + if err != nil { + return fmt.Errorf("sync-entitlements: error parsing spawn-cursors annotation: %w", err) + } + var spawned []Action + if hasSpawn { + if len(spawn.GetPageTokens()) > maxSpawnCursorsPerResponse { + return fmt.Errorf( + "sync-entitlements: SpawnCursors carried %d page tokens (max %d per response); chain additional spawns across pages instead", + len(spawn.GetPageTokens()), maxSpawnCursorsPerResponse) + } + for _, tok := range spawn.GetPageTokens() { + if tok == "" { + continue + } + if len(tok) > maxSpawnCursorTokenBytes { + return fmt.Errorf( + "sync-entitlements: SpawnCursors page token is %d bytes (max %d)", + len(tok), maxSpawnCursorTokenBytes) + } + spawned = append(spawned, Action{ + Op: SyncEntitlementsOp, + ResourceTypeID: action.ResourceTypeID, + ResourceID: action.ResourceID, // empty on type-scoped actions + PageToken: tok, + Spawned: true, + }) + } + if len(spawned) > 0 { + s.state.AddSourceCacheStats(SourceCacheStats{SpawnedCursors: int64(len(spawned))}) + } + ctxzap.Extract(ctx).Debug("sync-entitlements: spawned sibling entitlement cursors", + zap.String("resource_type_id", action.ResourceTypeID), + zap.String("resource_id", action.ResourceID), + zap.Bool("type_scoped", typeScoped), + zap.Int("cursors", len(spawned)), + zap.Int64("estimated_total", spawn.GetEstimatedTotal())) + } + + return s.nextPageOrFinishAction(ctx, action, resp.GetNextPageToken(), spawned...) } func (s *syncer) SyncStaticEntitlements(ctx context.Context, action *Action) error { @@ -1739,11 +1866,6 @@ func (s *syncer) syncStaticEntitlementsForResourceType(ctx context.Context, acti } entID := entitlement.NewEntitlementID(resource, ent.GetSlug()) - if hasExclusionGroup { - if err := s.recordEntitlementExclusionGroup(exclusionGroup, entID, resource.GetId().GetResourceType()); err != nil { - return err - } - } entitlements = append(entitlements, &v2.Entitlement{ Resource: resource, @@ -2449,6 +2571,11 @@ func (s *syncer) SyncExternalResources(ctx context.Context, action *Action) erro l := ctxzap.Extract(ctx) l.Info("Syncing external resources") + // Ingestion invariant I2 (repair): reconcile the match-processing + // flag with the store-derived fact before anything gates on it — + // replayed rows arm the engine's existence bit, never the stream arm. + s.repairExternalMatchFlag(ctx) + if s.externalResourceEntitlementIdFilter != "" { err := s.SyncExternalResourcesWithGrantToEntitlement(ctx, s.externalResourceEntitlementIdFilter) if err != nil { @@ -3420,6 +3547,16 @@ func WithStorageEngine(engine c1zstore.Engine) SyncOpt { } } +// WithStrictIngestionInvariants promotes check-only ingestion invariants +// (see ingest_invariants.go) from warn-with-log to hard sync failure. +// Tests and the replay-equivalence harness enable it; production default +// is lenient until the predicates have a clean release behind them. +func WithStrictIngestionInvariants() SyncOpt { + return func(s *syncer) { + s.strictIngestionInvariants = true + } +} + // WithSkipFullSync skips syncing entirely. func WithSkipFullSync() SyncOpt { return func(s *syncer) { diff --git a/pkg/sync/syncer_test.go b/pkg/sync/syncer_test.go index c3d0cc556..704847891 100644 --- a/pkg/sync/syncer_test.go +++ b/pkg/sync/syncer_test.go @@ -2022,6 +2022,18 @@ func TestSyncGrants_DoesNotPropagateAnnotationWhenAbsent(t *testing.T) { } func TestValidateEntitlementExclusionGroups(t *testing.T) { + // Drives the exclusionGroupTracker core (ingest invariant I5) over a + // slice, standing in for the store scan validateStoredExclusionGroups + // performs post-collection. + validate := func(ents []*v2.Entitlement) error { + tracker := &exclusionGroupTracker{} + for _, ent := range ents { + if err := tracker.record(ent); err != nil { + return err + } + } + return nil + } newEntitlement := func(t *testing.T, rt *v2.ResourceType, resourceID, slug string, opts ...et.EntitlementOption) *v2.Entitlement { t.Helper() r, err := rs.NewResource(resourceID, rt, resourceID) @@ -2030,78 +2042,67 @@ func TestValidateEntitlementExclusionGroups(t *testing.T) { } t.Run("empty input is a no-op", func(t *testing.T) { - s := &syncer{state: newState()} - require.NoError(t, s.validateEntitlementExclusionGroups(nil)) + require.NoError(t, validate(nil)) }) t.Run("entitlements without exclusion group annotation pass", func(t *testing.T) { - s := &syncer{state: newState()} ents := []*v2.Entitlement{ newEntitlement(t, userResourceType, "user1", "member"), newEntitlement(t, groupResourceType, "group1", "admin"), } - require.NoError(t, s.validateEntitlementExclusionGroups(ents)) + require.NoError(t, validate(ents)) }) t.Run("shared group id within one resource type passes", func(t *testing.T) { - s := &syncer{state: newState()} ents := []*v2.Entitlement{ newEntitlement(t, userResourceType, "user1", "viewer", et.WithExclusionGroup("role")), newEntitlement(t, userResourceType, "user2", "editor", et.WithExclusionGroup("role")), newEntitlement(t, userResourceType, "user3", "owner", et.WithExclusionGroup("role")), } - require.NoError(t, s.validateEntitlementExclusionGroups(ents)) + require.NoError(t, validate(ents)) }) t.Run("distinct group ids across resource types are independent", func(t *testing.T) { - s := &syncer{state: newState()} ents := []*v2.Entitlement{ newEntitlement(t, userResourceType, "user1", "viewer", et.WithExclusionGroup("user-role")), newEntitlement(t, groupResourceType, "group1", "admin", et.WithExclusionGroup("group-role")), } - require.NoError(t, s.validateEntitlementExclusionGroups(ents)) + require.NoError(t, validate(ents)) }) t.Run("same group id on two resource types fails", func(t *testing.T) { - s := &syncer{state: newState()} ents := []*v2.Entitlement{ newEntitlement(t, userResourceType, "user1", "viewer", et.WithExclusionGroup("role")), newEntitlement(t, groupResourceType, "group1", "admin", et.WithExclusionGroup("role")), } - err := s.validateEntitlementExclusionGroups(ents) + err := validate(ents) require.Error(t, err) require.Contains(t, err.Error(), `"role"`) require.Contains(t, err.Error(), `"user"`) require.Contains(t, err.Error(), `"group"`) }) - t.Run("cross-resource-type conflict is detected across separate batches via persisted state", func(t *testing.T) { - s := &syncer{state: newState()} - require.NoError(t, s.validateEntitlementExclusionGroups([]*v2.Entitlement{ - newEntitlement(t, userResourceType, "user1", "viewer", et.WithExclusionGroup("role")), - })) - err := s.validateEntitlementExclusionGroups([]*v2.Entitlement{ - newEntitlement(t, groupResourceType, "group1", "admin", et.WithExclusionGroup("role")), - }) + t.Run("cross-resource-type conflict is detected across pages of one scan", func(t *testing.T) { + tracker := &exclusionGroupTracker{} + require.NoError(t, tracker.record(newEntitlement(t, userResourceType, "user1", "viewer", et.WithExclusionGroup("role")))) + err := tracker.record(newEntitlement(t, groupResourceType, "group1", "admin", et.WithExclusionGroup("role"))) require.Error(t, err) require.Contains(t, err.Error(), `"role"`) }) t.Run("single default per group passes", func(t *testing.T) { - s := &syncer{state: newState()} ents := []*v2.Entitlement{ newEntitlement(t, userResourceType, "user1", "viewer", et.WithExclusionGroupDefault("role", 1)), newEntitlement(t, userResourceType, "user2", "editor", et.WithExclusionGroupOrder("role", 2)), newEntitlement(t, userResourceType, "user3", "owner", et.WithExclusionGroupOrder("role", 3)), } - require.NoError(t, s.validateEntitlementExclusionGroups(ents)) + require.NoError(t, validate(ents)) }) t.Run("two defaults in the same group fail", func(t *testing.T) { - s := &syncer{state: newState()} viewer := newEntitlement(t, userResourceType, "user1", "viewer", et.WithExclusionGroupDefault("role", 1)) editor := newEntitlement(t, userResourceType, "user2", "editor", et.WithExclusionGroupDefault("role", 2)) - err := s.validateEntitlementExclusionGroups([]*v2.Entitlement{viewer, editor}) + err := validate([]*v2.Entitlement{viewer, editor}) require.Error(t, err) require.Contains(t, err.Error(), `"role"`) require.Contains(t, err.Error(), viewer.GetId()) @@ -2109,60 +2110,55 @@ func TestValidateEntitlementExclusionGroups(t *testing.T) { require.Contains(t, err.Error(), "is_default") }) - t.Run("default conflict is detected across separate batches via persisted state", func(t *testing.T) { - s := &syncer{state: newState()} + t.Run("default conflict is detected across pages of one scan", func(t *testing.T) { + tracker := &exclusionGroupTracker{} viewer := newEntitlement(t, userResourceType, "user1", "viewer", et.WithExclusionGroupDefault("role", 1)) - require.NoError(t, s.validateEntitlementExclusionGroups([]*v2.Entitlement{viewer})) + require.NoError(t, tracker.record(viewer)) editor := newEntitlement(t, userResourceType, "user2", "editor", et.WithExclusionGroupDefault("role", 2)) - err := s.validateEntitlementExclusionGroups([]*v2.Entitlement{editor}) + err := tracker.record(editor) require.Error(t, err) require.Contains(t, err.Error(), viewer.GetId()) require.Contains(t, err.Error(), editor.GetId()) }) t.Run("defaults in different groups are independent", func(t *testing.T) { - s := &syncer{state: newState()} ents := []*v2.Entitlement{ newEntitlement(t, userResourceType, "user1", "viewer", et.WithExclusionGroupDefault("role-a", 1)), newEntitlement(t, userResourceType, "user2", "admin", et.WithExclusionGroupDefault("role-b", 1)), } - require.NoError(t, s.validateEntitlementExclusionGroups(ents)) + require.NoError(t, validate(ents)) }) t.Run("group at the cap passes", func(t *testing.T) { - s := &syncer{state: newState()} ents := make([]*v2.Entitlement, 0, maxEntitlementsPerExclusionGroup) for i := 0; i < maxEntitlementsPerExclusionGroup; i++ { ents = append(ents, newEntitlement(t, userResourceType, fmt.Sprintf("user%d", i), "member", et.WithExclusionGroup("role"))) } - require.NoError(t, s.validateEntitlementExclusionGroups(ents)) + require.NoError(t, validate(ents)) }) t.Run("group over the cap fails", func(t *testing.T) { - s := &syncer{state: newState()} ents := make([]*v2.Entitlement, 0, maxEntitlementsPerExclusionGroup+1) for i := 0; i < maxEntitlementsPerExclusionGroup+1; i++ { ents = append(ents, newEntitlement(t, userResourceType, fmt.Sprintf("user%d", i), "member", et.WithExclusionGroup("role"))) } - err := s.validateEntitlementExclusionGroups(ents) + err := validate(ents) require.Error(t, err) require.Contains(t, err.Error(), `"role"`) require.Contains(t, err.Error(), "too many entitlements") }) - t.Run("cap is enforced across separate batches via persisted state", func(t *testing.T) { - s := &syncer{state: newState()} - // First batch fills the group exactly to the cap. - first := make([]*v2.Entitlement, 0, maxEntitlementsPerExclusionGroup) + t.Run("cap is enforced across pages of one scan", func(t *testing.T) { + // First page fills the group exactly to the cap. + tracker := &exclusionGroupTracker{} for i := 0; i < maxEntitlementsPerExclusionGroup; i++ { - first = append(first, newEntitlement(t, userResourceType, fmt.Sprintf("user%d", i), "member", et.WithExclusionGroup("role"))) + require.NoError(t, tracker.record(newEntitlement(t, userResourceType, fmt.Sprintf("user%d", i), "member", et.WithExclusionGroup("role")))) } - require.NoError(t, s.validateEntitlementExclusionGroups(first)) - // A later batch adding one more member to the same group pushes it over. - err := s.validateEntitlementExclusionGroups([]*v2.Entitlement{ + // A later page adding one more member to the same group pushes it over. + err := tracker.record( newEntitlement(t, userResourceType, "extra", "member", et.WithExclusionGroup("role")), - }) + ) require.Error(t, err) require.Contains(t, err.Error(), "too many entitlements") }) diff --git a/pkg/sync/type_scoped_entitlements_test.go b/pkg/sync/type_scoped_entitlements_test.go new file mode 100644 index 000000000..4a574138d --- /dev/null +++ b/pkg/sync/type_scoped_entitlements_test.go @@ -0,0 +1,520 @@ +package sync //nolint:revive,nolintlint // we can't change the package name for backwards compatibility + +// End-to-end harness for TYPE-SCOPED entitlements (v2.TypeScopedEntitlements + +// SpawnCursors), mirroring type_scoped_grants_test.go: entitlement rows for a +// whole resource type are served through connector-defined chunk cursors +// instead of one ListEntitlements call per group. Each chunk is its own +// source-cache scope, so warm syncs replay unchanged chunks and +// overlay/tombstone the changed ones. + +import ( + "context" + "fmt" + "path/filepath" + "sort" + "strings" + "sync" + "testing" + + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + "github.com/conductorone/baton-sdk/pkg/annotations" + "github.com/conductorone/baton-sdk/pkg/dotc1z" + "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" + "github.com/conductorone/baton-sdk/pkg/logging" + "github.com/conductorone/baton-sdk/pkg/sourcecache" + et "github.com/conductorone/baton-sdk/pkg/types/entitlement" + rs "github.com/conductorone/baton-sdk/pkg/types/resource" +) + +// chunkedEntitlementsMockConnector serves group entitlements exclusively +// through type-scoped cursors. +type chunkedEntitlementsMockConnector struct { + *mockConnector + + mu sync.Mutex + lookup sourcecache.Lookup + + groupIDs []string // sorted; chunk i = groupIDs[i*50:(i+1)*50] + ents map[string][]string // groupID -> entitlement slugs (e.g. "member") + resByID map[string]*v2.Resource + + tokenByChunk map[int]string // last validator issued per chunk + generation int + + addsByChunk map[int][]*v2.Entitlement + deletedIDsByChunk map[int][]string + + unstampedExtra bool + + planningCalls int + coldChunkRounds int + warmChunkRounds int + perResourceGroupLE int + unstampedPages int +} + +var typeScopedEntGroupRT = v2.ResourceType_builder{ + Id: "group", + DisplayName: "Group", + Traits: []v2.ResourceType_Trait{v2.ResourceType_TRAIT_GROUP}, + Annotations: annotations.New(&v2.TypeScopedEntitlements{}), +}.Build() + +func newChunkedEntitlementsMockConnector(t *testing.T, groupCount int, entsPerGroup int) *chunkedEntitlementsMockConnector { + t.Helper() + mc := &chunkedEntitlementsMockConnector{ + mockConnector: newMockConnector(), + ents: map[string][]string{}, + resByID: map[string]*v2.Resource{}, + tokenByChunk: map[int]string{}, + addsByChunk: map[int][]*v2.Entitlement{}, + deletedIDsByChunk: map[int][]string{}, + } + mc.rtDB = append(mc.rtDB, typeScopedEntGroupRT, userResourceType) + + userIDs := make([]string, 0, 10) + for i := 0; i < 10; i++ { + uid := fmt.Sprintf("user-%02d", i) + u, err := rs.NewUserResource(uid, userResourceType, uid, nil, rs.WithAnnotation(&v2.SkipEntitlementsAndGrants{})) + require.NoError(t, err) + mc.AddResource(context.Background(), u) + mc.resByID[uid] = u + userIDs = append(userIDs, uid) + } + _ = userIDs + + slugs := []string{"member", "admin", "viewer"} + for i := 0; i < groupCount; i++ { + gid := fmt.Sprintf("group-%03d", i) + g, err := rs.NewGroupResource(gid, typeScopedEntGroupRT, gid, nil) + require.NoError(t, err) + mc.AddResource(context.Background(), g) + mc.resByID[gid] = g + mc.groupIDs = append(mc.groupIDs, gid) + + for e := 0; e < entsPerGroup; e++ { + mc.ents[gid] = append(mc.ents[gid], slugs[e%len(slugs)]) + } + } + sort.Strings(mc.groupIDs) + return mc +} + +func (mc *chunkedEntitlementsMockConnector) SetSourceCache(_ context.Context, lookup sourcecache.Lookup) { + mc.mu.Lock() + defer mc.mu.Unlock() + if lookup == nil { + lookup = sourcecache.NoopLookup{} + } + mc.lookup = lookup +} + +func (mc *chunkedEntitlementsMockConnector) Validate(context.Context, *v2.ConnectorServiceValidateRequest, ...grpc.CallOption) (*v2.ConnectorServiceValidateResponse, error) { + return v2.ConnectorServiceValidateResponse_builder{ + Annotations: annotations.New(v2.SourceCacheCapability_builder{ + Mode: v2.SourceCacheCapability_MODE_READ_WRITE, + }.Build()), + }.Build(), nil +} + +func (mc *chunkedEntitlementsMockConnector) chunkCount() int { + return (len(mc.groupIDs) + chunkSize - 1) / chunkSize +} + +func (mc *chunkedEntitlementsMockConnector) chunkGroups(chunk int) []string { + lo := chunk * chunkSize + hi := lo + chunkSize + if hi > len(mc.groupIDs) { + hi = len(mc.groupIDs) + } + return mc.groupIDs[lo:hi] +} + +func entChunkScope(chunk int) string { + return fmt.Sprintf("groups/ents/chunk/%03d?sig=testsig", chunk) +} + +func (mc *chunkedEntitlementsMockConnector) groupEnt(gid, slug string) *v2.Entitlement { + return et.NewAssignmentEntitlement(mc.resByID[gid], slug, et.WithGrantableTo(userResourceType)) +} + +func (mc *chunkedEntitlementsMockConnector) chunkEntitlements(chunk int) []*v2.Entitlement { + var out []*v2.Entitlement + for _, gid := range mc.chunkGroups(chunk) { + for _, slug := range mc.ents[gid] { + out = append(out, mc.groupEnt(gid, slug)) + } + } + return out +} + +func (mc *chunkedEntitlementsMockConnector) mutateAddEntitlement(gid, slug string) { + mc.mu.Lock() + defer mc.mu.Unlock() + mc.ents[gid] = append(mc.ents[gid], slug) + chunk := sort.SearchStrings(mc.groupIDs, gid) / chunkSize + mc.addsByChunk[chunk] = append(mc.addsByChunk[chunk], mc.groupEnt(gid, slug)) +} + +func (mc *chunkedEntitlementsMockConnector) mutateRemoveEntitlement(gid, slug string) { + mc.mu.Lock() + defer mc.mu.Unlock() + kept := mc.ents[gid][:0] + for _, s := range mc.ents[gid] { + if s != slug { + kept = append(kept, s) + } + } + mc.ents[gid] = kept + chunk := sort.SearchStrings(mc.groupIDs, gid) / chunkSize + mc.deletedIDsByChunk[chunk] = append(mc.deletedIDsByChunk[chunk], mc.groupEnt(gid, slug).GetId()) +} + +func (mc *chunkedEntitlementsMockConnector) clearDeltas() { + mc.mu.Lock() + defer mc.mu.Unlock() + mc.addsByChunk = map[int][]*v2.Entitlement{} + mc.deletedIDsByChunk = map[int][]string{} +} + +func (mc *chunkedEntitlementsMockConnector) ListEntitlements( + ctx context.Context, in *v2.EntitlementsServiceListEntitlementsRequest, _ ...grpc.CallOption, +) (*v2.EntitlementsServiceListEntitlementsResponse, error) { + rid := in.GetResource().GetId() + if rid.GetResourceType() != typeScopedEntGroupRT.GetId() { + return v2.EntitlementsServiceListEntitlementsResponse_builder{}.Build(), nil + } + reqAnnos := annotations.Annotations(in.GetAnnotations()) + if !reqAnnos.Contains(&v2.TypeScopedEntitlements{}) { + mc.mu.Lock() + mc.perResourceGroupLE++ + mc.mu.Unlock() + return v2.EntitlementsServiceListEntitlementsResponse_builder{}.Build(), nil + } + + tok := in.GetPageToken() + if tok == "" { + mc.mu.Lock() + mc.planningCalls++ + n := mc.chunkCount() + mc.mu.Unlock() + tokens := make([]string, 0, n) + for i := 0; i < n; i++ { + tokens = append(tokens, fmt.Sprintf("chunk=%d", i)) + } + return v2.EntitlementsServiceListEntitlementsResponse_builder{ + Annotations: annotations.New(v2.SpawnCursors_builder{ + PageTokens: tokens, + EstimatedTotal: int64(len(mc.groupIDs)), + }.Build()), + }.Build(), nil + } + + if tok == "chunk=0&extra" { + mc.mu.Lock() + mc.unstampedPages++ + mc.mu.Unlock() + return v2.EntitlementsServiceListEntitlementsResponse_builder{ + List: mc.unstampedExtraEntitlements(), + }.Build(), nil + } + + var chunk, page int + if n, err := fmt.Sscanf(tok, "chunk=%d&page=%d", &chunk, &page); err != nil || n != 2 { + if _, err := fmt.Sscanf(tok, "chunk=%d", &chunk); err != nil { + return nil, fmt.Errorf("bad page token %q", tok) + } + page = 0 + } + scope := entChunkScope(chunk) + + mc.mu.Lock() + lookup := mc.lookup + lastTok := mc.tokenByChunk[chunk] + adds := mc.addsByChunk[chunk] + deleted := mc.deletedIDsByChunk[chunk] + gen := mc.generation + mc.mu.Unlock() + if lookup == nil { + lookup = sourcecache.NoopLookup{} + } + + newTok := fmt.Sprintf("tok-c%d-g%d", chunk, gen) + rotate := func() { + mc.mu.Lock() + mc.tokenByChunk[chunk] = newTok + mc.mu.Unlock() + } + + if page == 0 { + entry, found, err := lookup.LookupPreviousSourceCache(ctx, sourcecache.RowKindEntitlements, scope) + if err != nil { + return nil, err + } + if found && lastTok != "" && entry.ETag == lastTok { + mc.mu.Lock() + mc.warmChunkRounds++ + mc.mu.Unlock() + rotate() + next := "" + if mc.unstampedExtra && chunk == 0 { + next = "chunk=0&extra" + } + return v2.EntitlementsServiceListEntitlementsResponse_builder{ + List: adds, + Annotations: annotations.New(v2.SourceCacheReplay_builder{ + ScopeHash: scope, + Etag: newTok, + Overlay: true, + DeletedIds: deleted, + }.Build()), + NextPageToken: next, + }.Build(), nil + } + } + + ents := mc.chunkEntitlements(chunk) + half := len(ents) / 2 + if page == 0 { + mc.mu.Lock() + mc.coldChunkRounds++ + mc.mu.Unlock() + return v2.EntitlementsServiceListEntitlementsResponse_builder{ + List: ents[:half], + Annotations: annotations.New(v2.SourceCacheScope_builder{ + ScopeHash: scope, + }.Build()), + NextPageToken: fmt.Sprintf("chunk=%d&page=1", chunk), + }.Build(), nil + } + rotate() + next := "" + if mc.unstampedExtra && chunk == 0 { + next = "chunk=0&extra" + } + return v2.EntitlementsServiceListEntitlementsResponse_builder{ + List: ents[half:], + Annotations: annotations.New(v2.SourceCacheScope_builder{ + ScopeHash: scope, + Etag: newTok, + }.Build()), + NextPageToken: next, + }.Build(), nil +} + +func (mc *chunkedEntitlementsMockConnector) ListGrants(ctx context.Context, in *v2.GrantsServiceListGrantsRequest, _ ...grpc.CallOption) (*v2.GrantsServiceListGrantsResponse, error) { + // Type-scoped entitlements types still get a per-resource grants fan-out + // unless also annotated TypeScopedGrants; return empty so grants don't + // interfere with entitlement assertions. + return v2.GrantsServiceListGrantsResponse_builder{}.Build(), nil +} + +func (mc *chunkedEntitlementsMockConnector) unstampedExtraEntitlements() []*v2.Entitlement { + g := mc.resByID[mc.groupIDs[0]] + return []*v2.Entitlement{ + et.NewAssignmentEntitlement(g, "extra-a", et.WithGrantableTo(userResourceType)), + et.NewAssignmentEntitlement(g, "extra-b", et.WithGrantableTo(userResourceType)), + } +} + +func requireSameEntitlementIDs(t *testing.T, want, got map[string]*v2.Entitlement, msg string) { + t.Helper() + var missing, extra []string + for id := range want { + if _, ok := got[id]; !ok { + missing = append(missing, id) + } + } + for id := range got { + if _, ok := want[id]; !ok { + extra = append(extra, id) + } + } + sort.Strings(missing) + sort.Strings(extra) + require.Emptyf(t, missing, "%s: entitlements missing: %s", msg, strings.Join(missing, ", ")) + require.Emptyf(t, extra, "%s: unexpected entitlements: %s", msg, strings.Join(extra, ", ")) +} + +func listEntitlementsInFile(ctx context.Context, t *testing.T, path string) map[string]*v2.Entitlement { + t.Helper() + reopen, err := dotc1z.NewStore(ctx, path, + dotc1z.WithEngine(c1zstore.EnginePebble), + dotc1z.WithReadOnly(true), + ) + require.NoError(t, err) + defer func() { _ = reopen.Close(ctx) }() + + syncMeta, err := reopen.SyncMeta().LatestFullSync(ctx) + require.NoError(t, err) + require.NotNil(t, syncMeta) + require.NoError(t, reopen.SetCurrentSync(ctx, syncMeta.ID)) + + out := map[string]*v2.Entitlement{} + pageToken := "" + for { + resp, err := reopen.ListEntitlements(ctx, v2.EntitlementsServiceListEntitlementsRequest_builder{PageToken: pageToken}.Build()) + require.NoError(t, err) + for _, e := range resp.GetList() { + out[e.GetId()] = e + } + pageToken = resp.GetNextPageToken() + if pageToken == "" { + break + } + } + return out +} + +func runChunkedEntitlementsSync(ctx context.Context, t *testing.T, mc *chunkedEntitlementsMockConnector, path, prevPath, tmpDir string, extraOpts ...SyncOpt) { + t.Helper() + store, err := dotc1z.NewStore(ctx, path, + dotc1z.WithEngine(c1zstore.EnginePebble), + dotc1z.WithTmpDir(tmpDir), + ) + require.NoError(t, err) + opts := []SyncOpt{WithConnectorStore(store), WithTmpDir(tmpDir)} + if prevPath != "" { + opts = append(opts, WithPreviousSyncC1ZPath(prevPath)) + } + opts = append(opts, extraOpts...) + syncer, err := NewSyncer(ctx, mc, opts...) + require.NoError(t, err) + require.NoError(t, syncer.Sync(ctx)) + require.NoError(t, syncer.Close(ctx)) +} + +func TestTypeScopedEntitlementsMixedStampedUnstampedCursor(t *testing.T) { + ctx, err := logging.Init(context.Background()) + require.NoError(t, err) + tmpDir := t.TempDir() + + mc := newChunkedEntitlementsMockConnector(t, 10, 1) + mc.unstampedExtra = true + require.Equal(t, 1, mc.chunkCount()) + + sync1 := filepath.Join(tmpDir, "sync1.c1z") + runChunkedEntitlementsSync(ctx, t, mc, sync1, "", tmpDir) + require.Equal(t, 1, mc.coldChunkRounds) + require.Equal(t, 1, mc.unstampedPages) + + ents1 := listEntitlementsInFile(ctx, t, sync1) + require.Len(t, ents1, 12) // 10 member + 2 unstamped + + mc.generation = 1 + sync2 := filepath.Join(tmpDir, "sync2.c1z") + runChunkedEntitlementsSync(ctx, t, mc, sync2, sync1, tmpDir) + require.Equal(t, 1, mc.warmChunkRounds) + require.Equal(t, 1, mc.coldChunkRounds) + require.Equal(t, 2, mc.unstampedPages) + + requireSameEntitlementIDs(t, ents1, listEntitlementsInFile(ctx, t, sync2), "warm sync with mixed pages vs cold") + + mc.unstampedExtra = false + mc.generation = 2 + sync3 := filepath.Join(tmpDir, "sync3.c1z") + runChunkedEntitlementsSync(ctx, t, mc, sync3, sync2, tmpDir) + require.Equal(t, 2, mc.warmChunkRounds) + require.Equal(t, 2, mc.unstampedPages) + + ents3 := listEntitlementsInFile(ctx, t, sync3) + require.Len(t, ents3, 10, "unstamped rows must not ride replay") + for _, e := range mc.unstampedExtraEntitlements() { + require.NotContains(t, ents3, e.GetId()) + } +} + +func TestTypeScopedEntitlementsChunkedDelta(t *testing.T) { + for _, workers := range []int{0, 4} { + t.Run(fmt.Sprintf("workers=%d", workers), func(t *testing.T) { + runChunkedEntitlementsDeltaScenario(t, workers) + }) + } +} + +func runChunkedEntitlementsDeltaScenario(t *testing.T, workers int) { + ctx, err := logging.Init(context.Background()) + require.NoError(t, err) + tmpDir := t.TempDir() + + var workerOpts []SyncOpt + if workers > 0 { + workerOpts = append(workerOpts, WithWorkerCount(workers)) + } + + mc := newChunkedEntitlementsMockConnector(t, 120, 1) + require.Equal(t, 3, mc.chunkCount()) + + sync1 := filepath.Join(tmpDir, "sync1.c1z") + runChunkedEntitlementsSync(ctx, t, mc, sync1, "", tmpDir, workerOpts...) + + require.Equal(t, 1, mc.planningCalls) + require.Equal(t, 3, mc.coldChunkRounds) + require.Zero(t, mc.warmChunkRounds) + require.Zero(t, mc.perResourceGroupLE) + + ents1 := listEntitlementsInFile(ctx, t, sync1) + require.Len(t, ents1, 120) + + mc.generation = 1 + mc.mutateAddEntitlement("group-071", "admin") + mc.mutateRemoveEntitlement("group-115", "member") + + sync2 := filepath.Join(tmpDir, "sync2.c1z") + runChunkedEntitlementsSync(ctx, t, mc, sync2, sync1, tmpDir, workerOpts...) + + require.Equal(t, 2, mc.planningCalls) + require.Equal(t, 3, mc.coldChunkRounds) + require.Equal(t, 3, mc.warmChunkRounds) + require.Zero(t, mc.perResourceGroupLE) + + ents2 := listEntitlementsInFile(ctx, t, sync2) + require.Len(t, ents2, 120, "one add and one remove cancel out") + require.Contains(t, ents2, mc.groupEnt("group-071", "admin").GetId(), "overlay add must land") + + mc.clearDeltas() + control := filepath.Join(tmpDir, "control.c1z") + runChunkedEntitlementsSync(ctx, t, mc, control, "", tmpDir, workerOpts...) + requireSameEntitlementIDs(t, listEntitlementsInFile(ctx, t, control), ents2, "warm vs cold control") +} + +func TestTypeScopedEntitlementsTargetedSyncSkipsPerResource(t *testing.T) { + ctx, err := logging.Init(context.Background()) + require.NoError(t, err) + tmpDir := t.TempDir() + + mc := newChunkedEntitlementsMockConnector(t, 5, 1) + target := mc.resByID["group-000"] + + // Full sync first so the resource type (with annotation) is stored. + full := filepath.Join(tmpDir, "full.c1z") + runChunkedEntitlementsSync(ctx, t, mc, full, "", tmpDir) + require.Zero(t, mc.perResourceGroupLE) + + mc.planningCalls = 0 + mc.perResourceGroupLE = 0 + mc.coldChunkRounds = 0 + mc.warmChunkRounds = 0 + + targeted := filepath.Join(tmpDir, "targeted.c1z") + store, err := dotc1z.NewStore(ctx, targeted, + dotc1z.WithEngine(c1zstore.EnginePebble), + dotc1z.WithTmpDir(tmpDir), + ) + require.NoError(t, err) + syncer, err := NewSyncer(ctx, mc, + WithConnectorStore(store), + WithTmpDir(tmpDir), + WithPreviousSyncC1ZPath(full), + WithTargetedSyncResources([]*v2.Resource{target}), + ) + require.NoError(t, err) + require.NoError(t, syncer.Sync(ctx)) + require.NoError(t, syncer.Close(ctx)) + + require.Zero(t, mc.perResourceGroupLE, "targeted sync must not fan out per-resource ListEntitlements for a type-scoped type") + require.Zero(t, mc.planningCalls, "targeted sync must not run the type-scoped entitlements planner either") +} diff --git a/proto/c1/connector/v2/annotation_type_scoped_entitlements.proto b/proto/c1/connector/v2/annotation_type_scoped_entitlements.proto new file mode 100644 index 000000000..eada7f163 --- /dev/null +++ b/proto/c1/connector/v2/annotation_type_scoped_entitlements.proto @@ -0,0 +1,57 @@ +syntax = "proto3"; + +package c1.connector.v2; + +option go_package = "github.com/conductorone/baton-sdk/pb/c1/connector/v2"; + +// Type-scoped entitlement ingestion. +// +// Some providers expose change/enumeration APIs whose natural unit is a +// whole collection (or a connector-defined shard of one), not a single +// resource — e.g. Microsoft Graph delta queries, where one stream yields +// entitlement definitions for up to 50 groups. Forcing that shape through +// the per-resource ListEntitlements fan-out costs one request per resource +// even when nothing changed. +// +// A resource type annotated with TypeScopedEntitlements is EXCLUDED from +// the per-resource entitlements fan-out. Instead the syncer issues +// ListEntitlements calls whose resource carries ONLY the resource type +// (empty resource id); the connector answers with entitlements for the +// whole type, across as many paginated cursors as it chooses to spawn +// (see SpawnCursors in annotation_type_scoped_grants.proto). +// +// This is the entitlements-phase analogue of TypeScopedGrants: the +// connector takes over enumeration for the type, and every downstream +// behavior (storage, source-cache scopes/replay/tombstones, exclusion- +// group validation, rate limiting, page-token checkpointing) is unchanged +// because entitlements are self-describing rows. +// +// COMPLETENESS CONTRACT: a full sync of an annotated type must emit (or +// replay, via source-cache annotations) EVERY entitlement of that type. +// The syncer no longer visits each resource, so completeness rests +// entirely on the connector's enumeration. +// +// SKIP ANNOTATIONS: for an annotated type the syncer does not consult +// shouldSkipEntitlements on the type-scoped path. The connector owns skip +// semantics for resources of that type (omit rows; do not rely on +// per-resource SkipEntitlements* annotations to suppress the type-scoped +// call). +// +// ZERO-RESOURCE TYPES: the planner still enqueues the type-scoped action +// when the store holds no resources of the type. The connector must +// return an empty page (or SpawnCursors that resolve empty), not an +// error — an empty type is a valid outcome. +// +// TARGETED SYNC: SyncTargetedResource must not enqueue a per-resource +// SyncEntitlementsOp for a type-scoped type. The type-scoped planner +// action is the only legal entitlements path; a targeted sync of one +// resource of an annotated type syncs the resource (and children) but +// leaves type-scoped entitlement enumeration to a full entitlements +// phase. +// +// ROUTING MARKER ONLY: TypeScopedEntitlements is not a stream-coupled +// side-effect annotation. It is absent from the ingestion-invariant +// side-effect coverage map for the same reason as TypeScopedGrants — +// it routes the ListEntitlements call; it does not imply post-ingest +// repair or validation beyond ordinary entitlement handling. +message TypeScopedEntitlements {} diff --git a/proto/c1/connector/v2/annotation_type_scoped_grants.proto b/proto/c1/connector/v2/annotation_type_scoped_grants.proto index f48dc747b..04d333ffc 100644 --- a/proto/c1/connector/v2/annotation_type_scoped_grants.proto +++ b/proto/c1/connector/v2/annotation_type_scoped_grants.proto @@ -31,15 +31,32 @@ option go_package = "github.com/conductorone/baton-sdk/pb/c1/connector/v2"; // replay, via source-cache annotations) EVERY grant of that type. The // syncer no longer visits each resource, so completeness rests entirely on // the connector's enumeration. +// +// SKIP ANNOTATIONS: for an annotated type the syncer does not consult +// shouldSkipGrants on the type-scoped path. The connector owns skip +// semantics for resources of that type. +// +// ZERO-RESOURCE TYPES: the planner still enqueues the type-scoped action +// when the store holds no resources of the type. The connector must +// return an empty page (or SpawnCursors that resolve empty), not an error. +// +// TARGETED SYNC: SyncTargetedResource must not enqueue a per-resource +// SyncGrantsOp for a type-scoped type. Type-scoped grant enumeration is +// left to a full grants phase. +// +// ROUTING MARKER ONLY: TypeScopedGrants is absent from the ingestion- +// invariant side-effect coverage map — it routes ListGrants; it does not +// imply post-ingest repair beyond ordinary grant handling. message TypeScopedGrants {} -// SpawnCursors is attached to a ListGrants response to enqueue additional -// independent sibling cursors. Each token is delivered back to the -// connector as the page token of its own action — scheduled by the -// syncer's worker pool, rate-limited, and checkpointed like any other -// pagination. Honored on BOTH type-scoped and per-resource ListGrants -// responses; the spawned actions inherit the response's resource identity -// (type only for type-scoped calls, type+resource for per-resource calls). +// SpawnCursors is attached to a ListGrants or ListEntitlements response to +// enqueue additional independent sibling cursors. Each token is delivered +// back to the connector as the page token of its own action — scheduled by +// the syncer's worker pool, rate-limited, and checkpointed like any other +// pagination. Honored on BOTH type-scoped and per-resource ListGrants / +// ListEntitlements responses; the spawned actions inherit the response's +// resource identity (type only for type-scoped calls, type+resource for +// per-resource calls). // // Typical uses: // @@ -81,7 +98,7 @@ message SpawnCursors { } }]; - // Optional connector estimate of total grants across all cursors of this + // Optional connector estimate of total rows across all cursors of this // type, for progress reporting. Zero means unknown. int64 estimated_total = 2; } From 0269d7e6130bbbbead904d3ac3e8b6aabcd502ea Mon Sep 17 00:00:00 2001 From: Matt Kaniaris Date: Tue, 14 Jul 2026 16:26:54 -0600 Subject: [PATCH 04/17] wip --- .../v2/annotation_source_cache.pb.go | 17 +++++++++++++---- .../v2/annotation_source_cache.pb.validate.go | 11 +++++++++++ .../annotation_source_cache_protoopaque.pb.go | 11 ++++++++--- .../annotation_type_scoped_entitlements.pb.go | 11 +++++++---- ...type_scoped_entitlements_protoopaque.pb.go | 11 +++++++---- .../v2/annotation_type_scoped_grants.pb.go | 9 ++++++--- ...ation_type_scoped_grants_protoopaque.pb.go | 9 ++++++--- pkg/dotc1z/engine/pebble/grants.go | 12 ++++++++++++ pkg/dotc1z/engine/pebble/if_newer.go | 5 +++++ pkg/sync/ingest_invariants.go | 19 ++++++++++++++----- pkg/sync/source_cache_equivalence_test.go | 14 ++++++++++++++ pkg/sync/syncer.go | 12 ++++++++++-- pkg/synccompactor/attached/attached.go | 9 +++++++++ .../v2/annotation_source_cache.proto | 6 +++++- .../annotation_type_scoped_entitlements.proto | 11 +++++++---- .../v2/annotation_type_scoped_grants.proto | 9 ++++++--- 16 files changed, 140 insertions(+), 36 deletions(-) diff --git a/pb/c1/connector/v2/annotation_source_cache.pb.go b/pb/c1/connector/v2/annotation_source_cache.pb.go index 5e81b2954..6c077cac5 100644 --- a/pb/c1/connector/v2/annotation_source_cache.pb.go +++ b/pb/c1/connector/v2/annotation_source_cache.pb.go @@ -591,7 +591,11 @@ func (b0 SourceCacheLookupAsk_builder) Build() *SourceCacheLookupAsk { // the connector did not ask before) means unresolved: the connector may // ask again, subject to the bounce cap. type SourceCacheLookupAnswers struct { - state protoimpl.MessageState `protogen:"hybrid.v1"` + state protoimpl.MessageState `protogen:"hybrid.v1"` + // Cap = ask max_items (4096) × the per-request bounce cap (4): answers + // accumulate across bounces, so a request may legally carry every + // answer its asks produced. Keep in sync with + // sourcecache.MaxLookupBouncesPerRequest / maxAnswersPerMessage. Answers []*SourceCacheLookupAnswers_Answer `protobuf:"bytes,1,rep,name=answers,proto3" json:"answers,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache @@ -636,6 +640,10 @@ func (x *SourceCacheLookupAnswers) SetAnswers(v []*SourceCacheLookupAnswers_Answ type SourceCacheLookupAnswers_builder struct { _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + // Cap = ask max_items (4096) × the per-request bounce cap (4): answers + // accumulate across bounces, so a request may legally carry every + // answer its asks produced. Keep in sync with + // sourcecache.MaxLookupBouncesPerRequest / maxAnswersPerMessage. Answers []*SourceCacheLookupAnswers_Answer } @@ -858,9 +866,10 @@ const file_c1_connector_v2_annotation_source_cache_proto_rawDesc = "" + "\brow_kind\x18\x01 \x01(\tB\t\xfaB\x06r\x04 \x01(@R\arowKind\x12)\n" + "\n" + "scope_hash\x18\x02 \x01(\tB\n" + - "\xfaB\ar\x05 \x01(\x80\x02R\tscopeHash\"\xfa\x01\n" + - "\x18SourceCacheLookupAnswers\x12J\n" + - "\aanswers\x18\x01 \x03(\v20.c1.connector.v2.SourceCacheLookupAnswers.AnswerR\aanswers\x1a\x91\x01\n" + + "\xfaB\ar\x05 \x01(\x80\x02R\tscopeHash\"\x86\x02\n" + + "\x18SourceCacheLookupAnswers\x12V\n" + + "\aanswers\x18\x01 \x03(\v20.c1.connector.v2.SourceCacheLookupAnswers.AnswerB\n" + + "\xfaB\a\x92\x01\x04\x10\x80\x80\x01R\aanswers\x1a\x91\x01\n" + "\x06Answer\x12$\n" + "\brow_kind\x18\x01 \x01(\tB\t\xfaB\x06r\x04 \x01(@R\arowKind\x12)\n" + "\n" + diff --git a/pb/c1/connector/v2/annotation_source_cache.pb.validate.go b/pb/c1/connector/v2/annotation_source_cache.pb.validate.go index a301fbcbe..57afb268a 100644 --- a/pb/c1/connector/v2/annotation_source_cache.pb.validate.go +++ b/pb/c1/connector/v2/annotation_source_cache.pb.validate.go @@ -622,6 +622,17 @@ func (m *SourceCacheLookupAnswers) validate(all bool) error { var errors []error + if len(m.GetAnswers()) > 16384 { + err := SourceCacheLookupAnswersValidationError{ + field: "Answers", + reason: "value must contain no more than 16384 item(s)", + } + if !all { + return err + } + errors = append(errors, err) + } + for idx, item := range m.GetAnswers() { _, _ = idx, item diff --git a/pb/c1/connector/v2/annotation_source_cache_protoopaque.pb.go b/pb/c1/connector/v2/annotation_source_cache_protoopaque.pb.go index baf0e3b67..c8a115196 100644 --- a/pb/c1/connector/v2/annotation_source_cache_protoopaque.pb.go +++ b/pb/c1/connector/v2/annotation_source_cache_protoopaque.pb.go @@ -586,6 +586,10 @@ func (x *SourceCacheLookupAnswers) SetAnswers(v []*SourceCacheLookupAnswers_Answ type SourceCacheLookupAnswers_builder struct { _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + // Cap = ask max_items (4096) × the per-request bounce cap (4): answers + // accumulate across bounces, so a request may legally carry every + // answer its asks produced. Keep in sync with + // sourcecache.MaxLookupBouncesPerRequest / maxAnswersPerMessage. Answers []*SourceCacheLookupAnswers_Answer } @@ -804,9 +808,10 @@ const file_c1_connector_v2_annotation_source_cache_proto_rawDesc = "" + "\brow_kind\x18\x01 \x01(\tB\t\xfaB\x06r\x04 \x01(@R\arowKind\x12)\n" + "\n" + "scope_hash\x18\x02 \x01(\tB\n" + - "\xfaB\ar\x05 \x01(\x80\x02R\tscopeHash\"\xfa\x01\n" + - "\x18SourceCacheLookupAnswers\x12J\n" + - "\aanswers\x18\x01 \x03(\v20.c1.connector.v2.SourceCacheLookupAnswers.AnswerR\aanswers\x1a\x91\x01\n" + + "\xfaB\ar\x05 \x01(\x80\x02R\tscopeHash\"\x86\x02\n" + + "\x18SourceCacheLookupAnswers\x12V\n" + + "\aanswers\x18\x01 \x03(\v20.c1.connector.v2.SourceCacheLookupAnswers.AnswerB\n" + + "\xfaB\a\x92\x01\x04\x10\x80\x80\x01R\aanswers\x1a\x91\x01\n" + "\x06Answer\x12$\n" + "\brow_kind\x18\x01 \x01(\tB\t\xfaB\x06r\x04 \x01(@R\arowKind\x12)\n" + "\n" + diff --git a/pb/c1/connector/v2/annotation_type_scoped_entitlements.pb.go b/pb/c1/connector/v2/annotation_type_scoped_entitlements.pb.go index 8507dd430..0c7c5025e 100644 --- a/pb/c1/connector/v2/annotation_type_scoped_entitlements.pb.go +++ b/pb/c1/connector/v2/annotation_type_scoped_entitlements.pb.go @@ -33,10 +33,13 @@ const ( // // A resource type annotated with TypeScopedEntitlements is EXCLUDED from // the per-resource entitlements fan-out. Instead the syncer issues -// ListEntitlements calls whose resource carries ONLY the resource type -// (empty resource id); the connector answers with entitlements for the -// whole type, across as many paginated cursors as it chooses to spawn -// (see SpawnCursors in annotation_type_scoped_grants.proto). +// ListEntitlements calls carrying this annotation on the REQUEST as the +// routing marker; the request's resource is a self-referential stub +// ({type, type} — wire validation requires a non-empty resource id and +// the stub identifies the type, never a real resource). The connector +// answers with entitlements for the whole type, across as many paginated +// cursors as it chooses to spawn (see SpawnCursors in +// annotation_type_scoped_grants.proto). // // This is the entitlements-phase analogue of TypeScopedGrants: the // connector takes over enumeration for the type, and every downstream diff --git a/pb/c1/connector/v2/annotation_type_scoped_entitlements_protoopaque.pb.go b/pb/c1/connector/v2/annotation_type_scoped_entitlements_protoopaque.pb.go index 29950644f..45ac5c10f 100644 --- a/pb/c1/connector/v2/annotation_type_scoped_entitlements_protoopaque.pb.go +++ b/pb/c1/connector/v2/annotation_type_scoped_entitlements_protoopaque.pb.go @@ -33,10 +33,13 @@ const ( // // A resource type annotated with TypeScopedEntitlements is EXCLUDED from // the per-resource entitlements fan-out. Instead the syncer issues -// ListEntitlements calls whose resource carries ONLY the resource type -// (empty resource id); the connector answers with entitlements for the -// whole type, across as many paginated cursors as it chooses to spawn -// (see SpawnCursors in annotation_type_scoped_grants.proto). +// ListEntitlements calls carrying this annotation on the REQUEST as the +// routing marker; the request's resource is a self-referential stub +// ({type, type} — wire validation requires a non-empty resource id and +// the stub identifies the type, never a real resource). The connector +// answers with entitlements for the whole type, across as many paginated +// cursors as it chooses to spawn (see SpawnCursors in +// annotation_type_scoped_grants.proto). // // This is the entitlements-phase analogue of TypeScopedGrants: the // connector takes over enumeration for the type, and every downstream diff --git a/pb/c1/connector/v2/annotation_type_scoped_grants.pb.go b/pb/c1/connector/v2/annotation_type_scoped_grants.pb.go index bee03140f..bbccc920d 100644 --- a/pb/c1/connector/v2/annotation_type_scoped_grants.pb.go +++ b/pb/c1/connector/v2/annotation_type_scoped_grants.pb.go @@ -34,9 +34,12 @@ const ( // // A resource type annotated with TypeScopedGrants is EXCLUDED from the // per-resource grants fan-out. Instead the syncer issues ListGrants calls -// whose resource carries ONLY the resource type (empty resource id); the -// connector answers with grants for the whole type, across as many -// paginated cursors as it chooses to spawn (see SpawnCursors). +// carrying this annotation on the REQUEST as the routing marker; the +// request's resource is a self-referential stub ({type, type} — wire +// validation requires a non-empty resource id and the stub identifies +// the type, never a real resource). The connector answers with grants +// for the whole type, across as many paginated cursors as it chooses to +// spawn (see SpawnCursors). // // This is the grants-phase analogue of StaticEntitlements: the connector // takes over enumeration for the type, and every downstream behavior diff --git a/pb/c1/connector/v2/annotation_type_scoped_grants_protoopaque.pb.go b/pb/c1/connector/v2/annotation_type_scoped_grants_protoopaque.pb.go index 5533c069b..7df6da07d 100644 --- a/pb/c1/connector/v2/annotation_type_scoped_grants_protoopaque.pb.go +++ b/pb/c1/connector/v2/annotation_type_scoped_grants_protoopaque.pb.go @@ -34,9 +34,12 @@ const ( // // A resource type annotated with TypeScopedGrants is EXCLUDED from the // per-resource grants fan-out. Instead the syncer issues ListGrants calls -// whose resource carries ONLY the resource type (empty resource id); the -// connector answers with grants for the whole type, across as many -// paginated cursors as it chooses to spawn (see SpawnCursors). +// carrying this annotation on the REQUEST as the routing marker; the +// request's resource is a self-referential stub ({type, type} — wire +// validation requires a non-empty resource id and the stub identifies +// the type, never a real resource). The connector answers with grants +// for the whole type, across as many paginated cursors as it chooses to +// spawn (see SpawnCursors). // // This is the grants-phase analogue of StaticEntitlements: the connector // takes over enumeration for the type, and every downstream behavior diff --git a/pkg/dotc1z/engine/pebble/grants.go b/pkg/dotc1z/engine/pebble/grants.go index 1fa3cf5d8..7d1430745 100644 --- a/pkg/dotc1z/engine/pebble/grants.go +++ b/pkg/dotc1z/engine/pebble/grants.go @@ -849,6 +849,18 @@ func (e *Engine) UnsafePutUniqueGrantRecords(ctx context.Context, records ...*v3 if r == nil { continue } + // Dense ingestion facts (see PutGrantRecords). + // Thread-safe: atomic load once armed, CAS + + // concurrent-safe db.Set on the first hit. + if err := e.noteGrantRecordFacts(r); err != nil { + errMu.Lock() + if encErr == nil { + encErr = err + } + errMu.Unlock() + failed.Store(true) + return + } val, err := marshalRecord(r) if err != nil { errMu.Lock() diff --git a/pkg/dotc1z/engine/pebble/if_newer.go b/pkg/dotc1z/engine/pebble/if_newer.go index 14951cd86..b4bfc9ce0 100644 --- a/pkg/dotc1z/engine/pebble/if_newer.go +++ b/pkg/dotc1z/engine/pebble/if_newer.go @@ -51,6 +51,11 @@ func (e *Engine) PutGrantRecordsIfNewer(ctx context.Context, records ...*v3.Gran if r == nil { continue } + // Dense ingestion facts (see PutGrantRecords): partial-sync + // upserts can carry ExternalResourceMatch* annotations too. + if err := e.noteGrantRecordFacts(r); err != nil { + return err + } id, err := grantIdentityFromRecord(r) if err != nil { return err diff --git a/pkg/sync/ingest_invariants.go b/pkg/sync/ingest_invariants.go index 83e9d70f5..c42a14d45 100644 --- a/pkg/sync/ingest_invariants.go +++ b/pkg/sync/ingest_invariants.go @@ -44,6 +44,7 @@ package sync //nolint:revive,nolintlint // we can't change the package name for import ( "context" "fmt" + "sort" stdsync "sync" "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" @@ -107,23 +108,28 @@ func (c *childScheduleSet) has(childTypeID, parentTypeID, parentID string) bool // function of the store: running match processing when the store holds // match-annotated grants is required; the flag is only an optimization // that lets other syncs skip the scan. -func (s *syncer) repairExternalMatchFlag(ctx context.Context) { +// +// A fact-read error FAILS the sync: proceeding with an unset flag would +// silently skip match processing for replay-armed grants (the exact bug +// class I2 exists to prevent), and an engine-meta read error at this +// point means the store is unhealthy anyway. +func (s *syncer) repairExternalMatchFlag(ctx context.Context) error { if s.state.HasExternalResourcesGrants() { - return + return nil } facts, ok := s.store.(dotc1z.IngestFactStore) if !ok { - return // SQLite: the response-loop arm is the mechanism. + return nil // SQLite: the response-loop arm is the mechanism. } has, err := facts.HasExternalMatchGrants(ctx) if err != nil { - ctxzap.Extract(ctx).Warn("ingest invariant I2: could not read external-match fact; relying on stream-armed flag", zap.Error(err)) - return + return fmt.Errorf("ingest invariant I2: reading external-match fact: %w", err) } if has { ctxzap.Extract(ctx).Debug("ingest invariant I2: arming external-match processing from store-derived fact") s.state.SetHasExternalResourcesGrants() } + return nil } // checkGrantResourceReferences is invariant I3: every distinct @@ -220,6 +226,9 @@ func (s *syncer) checkChildScheduling(ctx context.Context) error { } } if len(violations) > 0 { + // Sorted so the error text is byte-stable regardless of store + // iteration order (the verdict already is). + sort.Strings(violations) err := fmt.Errorf( "ingest invariant I4 violated: %d child resource sync(s) were never scheduled: %v", len(violations), violations) diff --git a/pkg/sync/source_cache_equivalence_test.go b/pkg/sync/source_cache_equivalence_test.go index fd83bf085..2eb6b38ca 100644 --- a/pkg/sync/source_cache_equivalence_test.go +++ b/pkg/sync/source_cache_equivalence_test.go @@ -889,6 +889,14 @@ func (c *equivConnector) listEntitlementsInner( ) (*v2.EntitlementsServiceListEntitlementsResponse, error) { rid := in.GetResource().GetId() if rid.GetResourceType() == equivTeamRT.GetId() { + // Route on the REQUEST marker, not just the resource type: a + // syncer bug that drops the TypeScopedEntitlements annotation + // (e.g. on a continuation bounce or spawned cursor) must fail + // the harness, not silently keep working. + reqAnnos := annotations.Annotations(in.GetAnnotations()) + if !reqAnnos.Contains(&v2.TypeScopedEntitlements{}) { + return nil, fmt.Errorf("team entitlements call missing TypeScopedEntitlements request marker (page token %q)", in.GetPageToken()) + } return c.teamEntitlementsPage(ctx, lookup, in.GetPageToken()) } if rid.GetResourceType() != groupResourceType.GetId() { @@ -945,6 +953,12 @@ func (c *equivConnector) ListGrants(ctx context.Context, in *v2.GrantsServiceLis case equivOrgRT.GetId(): resp, err = c.orgGrantsPage(ctx, lookup, rid.GetResource()) case equivTeamRT.GetId(): + // Same marker enforcement as listEntitlementsInner: dropping the + // TypeScopedGrants request marker must fail the harness. + reqAnnos := annotations.Annotations(in.GetAnnotations()) + if !reqAnnos.Contains(&v2.TypeScopedGrants{}) { + return nil, fmt.Errorf("team grants call missing TypeScopedGrants request marker (page token %q)", in.GetPageToken()) + } resp, err = c.teamGrantsPage(ctx, lookup, in.GetPageToken()) default: return v2.GrantsServiceListGrantsResponse_builder{}.Build(), nil diff --git a/pkg/sync/syncer.go b/pkg/sync/syncer.go index e1541d7f8..328ccedf4 100644 --- a/pkg/sync/syncer.go +++ b/pkg/sync/syncer.go @@ -2574,7 +2574,9 @@ func (s *syncer) SyncExternalResources(ctx context.Context, action *Action) erro // Ingestion invariant I2 (repair): reconcile the match-processing // flag with the store-derived fact before anything gates on it — // replayed rows arm the engine's existence bit, never the stream arm. - s.repairExternalMatchFlag(ctx) + if err := s.repairExternalMatchFlag(ctx); err != nil { + return err + } if s.externalResourceEntitlementIdFilter != "" { err := s.SyncExternalResourcesWithGrantToEntitlement(ctx, s.externalResourceEntitlementIdFilter) @@ -3057,7 +3059,13 @@ func (s *syncer) processGrantsWithExternalPrincipals(ctx context.Context, princi } } grantsToDelete = append(grantsToDelete, grant) - continue + // No `continue`: a grant carrying MatchAll AND MatchID/Match + // annotations must still evaluate the more specific rules so + // rank resolution (MatchID > key/value > MatchAll) applies + // within one grant, not just across grants. Transformed + // grant IDs coincide per principal, so addExpanded picks the + // higher-rank version; grantsToDelete double-appends are + // idempotent (same shape as MatchID+Match today). } // Expansion annotation (may be nil for non-expandable grants). diff --git a/pkg/synccompactor/attached/attached.go b/pkg/synccompactor/attached/attached.go index 97c1e50b7..a953012db 100644 --- a/pkg/synccompactor/attached/attached.go +++ b/pkg/synccompactor/attached/attached.go @@ -119,6 +119,15 @@ func (c *Compactor) Compact(ctx context.Context) error { return fmt.Errorf("failed to process records: %w", err) } + // NOTE: unlike the pebble compactors, this path does not mark the + // output sync Compacted or clear source-cache state. That is safe, + // not an oversight: attached compaction is SQLite-only, SQLite + // stores never implement dotc1z.SourceCacheStore, and the syncer's + // replay gate type-asserts for that capability before it ever + // consults sync metadata — a SQLite artifact can't be a replay + // source regardless of its flags. If SQLite ever grows source-cache + // support, this path must adopt the pebble compactors' contract + // (SetCompacted + ClearSourceCacheEntries). return nil } diff --git a/proto/c1/connector/v2/annotation_source_cache.proto b/proto/c1/connector/v2/annotation_source_cache.proto index a139fff1c..9d0e41db9 100644 --- a/proto/c1/connector/v2/annotation_source_cache.proto +++ b/proto/c1/connector/v2/annotation_source_cache.proto @@ -246,5 +246,9 @@ message SourceCacheLookupAnswers { max_bytes: 65536 }]; } - repeated Answer answers = 1; + // Cap = ask max_items (4096) × the per-request bounce cap (4): answers + // accumulate across bounces, so a request may legally carry every + // answer its asks produced. Keep in sync with + // sourcecache.MaxLookupBouncesPerRequest / maxAnswersPerMessage. + repeated Answer answers = 1 [(validate.rules).repeated = {max_items: 16384}]; } diff --git a/proto/c1/connector/v2/annotation_type_scoped_entitlements.proto b/proto/c1/connector/v2/annotation_type_scoped_entitlements.proto index eada7f163..909eafbec 100644 --- a/proto/c1/connector/v2/annotation_type_scoped_entitlements.proto +++ b/proto/c1/connector/v2/annotation_type_scoped_entitlements.proto @@ -15,10 +15,13 @@ option go_package = "github.com/conductorone/baton-sdk/pb/c1/connector/v2"; // // A resource type annotated with TypeScopedEntitlements is EXCLUDED from // the per-resource entitlements fan-out. Instead the syncer issues -// ListEntitlements calls whose resource carries ONLY the resource type -// (empty resource id); the connector answers with entitlements for the -// whole type, across as many paginated cursors as it chooses to spawn -// (see SpawnCursors in annotation_type_scoped_grants.proto). +// ListEntitlements calls carrying this annotation on the REQUEST as the +// routing marker; the request's resource is a self-referential stub +// ({type, type} — wire validation requires a non-empty resource id and +// the stub identifies the type, never a real resource). The connector +// answers with entitlements for the whole type, across as many paginated +// cursors as it chooses to spawn (see SpawnCursors in +// annotation_type_scoped_grants.proto). // // This is the entitlements-phase analogue of TypeScopedGrants: the // connector takes over enumeration for the type, and every downstream diff --git a/proto/c1/connector/v2/annotation_type_scoped_grants.proto b/proto/c1/connector/v2/annotation_type_scoped_grants.proto index 04d333ffc..28eef7de1 100644 --- a/proto/c1/connector/v2/annotation_type_scoped_grants.proto +++ b/proto/c1/connector/v2/annotation_type_scoped_grants.proto @@ -17,9 +17,12 @@ option go_package = "github.com/conductorone/baton-sdk/pb/c1/connector/v2"; // // A resource type annotated with TypeScopedGrants is EXCLUDED from the // per-resource grants fan-out. Instead the syncer issues ListGrants calls -// whose resource carries ONLY the resource type (empty resource id); the -// connector answers with grants for the whole type, across as many -// paginated cursors as it chooses to spawn (see SpawnCursors). +// carrying this annotation on the REQUEST as the routing marker; the +// request's resource is a self-referential stub ({type, type} — wire +// validation requires a non-empty resource id and the stub identifies +// the type, never a real resource). The connector answers with grants +// for the whole type, across as many paginated cursors as it chooses to +// spawn (see SpawnCursors). // // This is the grants-phase analogue of StaticEntitlements: the connector // takes over enumeration for the type, and every downstream behavior From 8d3d16edfe60f293841698a4c97f4f23a938c897 Mon Sep 17 00:00:00 2001 From: Matt Kaniaris Date: Tue, 14 Jul 2026 18:37:53 -0600 Subject: [PATCH 05/17] rename some things to have better names --- .../v2/annotation_source_cache.pb.go | 224 +++++++++--------- .../v2/annotation_source_cache.pb.validate.go | 72 +++--- .../annotation_source_cache_protoopaque.pb.go | 218 +++++++++-------- .../annotation_type_scoped_entitlements.pb.go | 4 +- ...type_scoped_entitlements_protoopaque.pb.go | 4 +- .../v2/annotation_type_scoped_grants.pb.go | 40 ++-- ...notation_type_scoped_grants.pb.validate.go | 63 ++--- ...ation_type_scoped_grants_protoopaque.pb.go | 40 ++-- .../connectorapi/baton/v1/source_cache.pb.go | 57 +++-- .../baton/v1/source_cache.pb.validate.go | 8 +- .../baton/v1/source_cache_grpc.pb.go | 4 +- .../baton/v1/source_cache_protoopaque.pb.go | 63 +++-- pb/c1/storage/v3/records.pb.go | 142 +++++------ pb/c1/storage/v3/records.pb.validate.go | 10 +- pb/c1/storage/v3/records_protoopaque.pb.go | 171 +++++++------ pkg/connectorbuilder/continuation_test.go | 32 +-- pkg/connectorbuilder/resource_syncer.go | 4 +- .../type_scoped_entitlements_test.go | 4 +- pkg/dotc1z/assets.go | 2 +- pkg/dotc1z/c1file.go | 14 +- pkg/dotc1z/c1file_attached.go | 2 +- pkg/dotc1z/c1file_close_cancel_test.go | 2 +- pkg/dotc1z/c1zstore/grant_store.go | 4 +- pkg/dotc1z/clone_sync.go | 2 +- pkg/dotc1z/copy_isolate_sync.go | 2 +- pkg/dotc1z/diff.go | 2 +- pkg/dotc1z/engine/pebble/adapter.go | 10 +- .../engine/pebble/adapter_grants_store.go | 6 +- pkg/dotc1z/engine/pebble/entitlements.go | 2 +- pkg/dotc1z/engine/pebble/grants.go | 20 +- .../engine/pebble/grants_synth_encode.go | 4 +- .../engine/pebble/grants_synth_encode_test.go | 12 +- pkg/dotc1z/engine/pebble/if_newer.go | 4 +- pkg/dotc1z/engine/pebble/keys.go | 66 +++--- pkg/dotc1z/engine/pebble/raw_records.go | 40 ++-- pkg/dotc1z/engine/pebble/resources.go | 2 +- pkg/dotc1z/engine/pebble/source_cache.go | 80 +++---- .../engine/pebble/source_cache_engine_test.go | 22 +- pkg/dotc1z/entitlements.go | 2 +- pkg/dotc1z/grants.go | 2 +- pkg/dotc1z/resouce_types.go | 2 +- pkg/dotc1z/resources.go | 2 +- pkg/dotc1z/rollback_expansion.go | 2 +- pkg/dotc1z/source_cache.go | 54 ++--- pkg/dotc1z/sync_runs.go | 16 +- pkg/lambda/grpc/continuation_wire_test.go | 20 +- pkg/sourcecache/context.go | 12 +- pkg/sourcecache/continuation.go | 58 ++--- pkg/sourcecache/grpc_lookup.go | 12 +- pkg/sourcecache/grpc_lookup_test.go | 24 +- pkg/sourcecache/grpc_server.go | 10 +- pkg/sourcecache/sourcecache.go | 38 +-- ...rs_test.go => enqueue_page_tokens_test.go} | 32 +-- pkg/sync/progresslog/progresslog.go | 2 +- pkg/sync/source_cache.go | 84 +++---- pkg/sync/source_cache_continuation.go | 22 +- .../source_cache_continuation_e2e_test.go | 8 +- pkg/sync/source_cache_continuation_test.go | 72 +++--- .../source_cache_equivalence_soak_test.go | 31 +++ pkg/sync/source_cache_equivalence_test.go | 58 ++--- pkg/sync/source_cache_sideeffects_test.go | 50 ++-- pkg/sync/source_cache_stats.go | 4 +- pkg/sync/source_cache_test.go | 46 ++-- pkg/sync/state.go | 2 +- pkg/sync/syncer.go | 34 +-- pkg/sync/type_scoped_entitlements_test.go | 26 +- pkg/sync/type_scoped_grants_test.go | 30 +-- .../v2/annotation_source_cache.proto | 62 ++--- .../annotation_type_scoped_entitlements.proto | 4 +- .../v2/annotation_type_scoped_grants.proto | 8 +- .../connectorapi/baton/v1/source_cache.proto | 12 +- proto/c1/storage/v3/records.proto | 25 +- 72 files changed, 1177 insertions(+), 1147 deletions(-) rename pkg/sync/{spawn_cursors_test.go => enqueue_page_tokens_test.go} (95%) create mode 100644 pkg/sync/source_cache_equivalence_soak_test.go diff --git a/pb/c1/connector/v2/annotation_source_cache.pb.go b/pb/c1/connector/v2/annotation_source_cache.pb.go index 6c077cac5..db954c3cc 100644 --- a/pb/c1/connector/v2/annotation_source_cache.pb.go +++ b/pb/c1/connector/v2/annotation_source_cache.pb.go @@ -127,10 +127,10 @@ func (b0 SourceCacheCapability_builder) Build() *SourceCacheCapability { return m0 } -// SourceCacheScope is attached to a list-response page whose rows were +// SourceCacheRecord is attached to a list-response page whose rows were // freshly fetched from upstream. The SDK stamps the page's rows with -// scope_hash so a future sync can replay them as a unit. -type SourceCacheScope struct { +// scope_key so a future sync can replay them as a unit. +type SourceCacheRecord struct { state protoimpl.MessageState `protogen:"hybrid.v1"` // Connector-computed stable identifier for the canonical scope. Must be // byte-stable across syncs for the same logical scope. Prefer an ORDERED @@ -139,12 +139,12 @@ type SourceCacheScope struct { // identifiers that correlate with fetch order keep index writes nearly // append-ordered at scale. sourcecache.HashScope is the fallback when no // compact natural form exists. - ScopeHash string `protobuf:"bytes,1,opt,name=scope_hash,json=scopeHash,proto3" json:"scope_hash,omitempty"` + ScopeKey string `protobuf:"bytes,1,opt,name=scope_key,json=scopeKey,proto3" json:"scope_key,omitempty"` // Opaque validator to persist for this scope. May be empty on interim // pages of a multi-page scope (e.g. Graph @odata.nextLink pages); the - // SDK writes the scope's manifest entry when a non-empty etag arrives. + // SDK writes the scope's manifest entry when a non-empty cache_validator arrives. // A 200 response with zero rows still persists the entry. - Etag string `protobuf:"bytes,2,opt,name=etag,proto3" json:"etag,omitempty"` + CacheValidator string `protobuf:"bytes,2,opt,name=cache_validator,json=cacheValidator,proto3" json:"cache_validator,omitempty"` // Tombstones applied after this page's rows commit. Lets every page of // a multi-page delta round carry its own deletions as the provider // delivers them, instead of buffering a whole round onto the first @@ -178,20 +178,20 @@ type SourceCacheScope struct { sizeCache protoimpl.SizeCache } -func (x *SourceCacheScope) Reset() { - *x = SourceCacheScope{} +func (x *SourceCacheRecord) Reset() { + *x = SourceCacheRecord{} mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[1] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } -func (x *SourceCacheScope) String() string { +func (x *SourceCacheRecord) String() string { return protoimpl.X.MessageStringOf(x) } -func (*SourceCacheScope) ProtoMessage() {} +func (*SourceCacheRecord) ProtoMessage() {} -func (x *SourceCacheScope) ProtoReflect() protoreflect.Message { +func (x *SourceCacheRecord) ProtoReflect() protoreflect.Message { mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[1] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -203,51 +203,51 @@ func (x *SourceCacheScope) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -func (x *SourceCacheScope) GetScopeHash() string { +func (x *SourceCacheRecord) GetScopeKey() string { if x != nil { - return x.ScopeHash + return x.ScopeKey } return "" } -func (x *SourceCacheScope) GetEtag() string { +func (x *SourceCacheRecord) GetCacheValidator() string { if x != nil { - return x.Etag + return x.CacheValidator } return "" } -func (x *SourceCacheScope) GetDeletedIds() []string { +func (x *SourceCacheRecord) GetDeletedIds() []string { if x != nil { return x.DeletedIds } return nil } -func (x *SourceCacheScope) GetDeletedPrincipalIds() []string { +func (x *SourceCacheRecord) GetDeletedPrincipalIds() []string { if x != nil { return x.DeletedPrincipalIds } return nil } -func (x *SourceCacheScope) SetScopeHash(v string) { - x.ScopeHash = v +func (x *SourceCacheRecord) SetScopeKey(v string) { + x.ScopeKey = v } -func (x *SourceCacheScope) SetEtag(v string) { - x.Etag = v +func (x *SourceCacheRecord) SetCacheValidator(v string) { + x.CacheValidator = v } -func (x *SourceCacheScope) SetDeletedIds(v []string) { +func (x *SourceCacheRecord) SetDeletedIds(v []string) { x.DeletedIds = v } -func (x *SourceCacheScope) SetDeletedPrincipalIds(v []string) { +func (x *SourceCacheRecord) SetDeletedPrincipalIds(v []string) { x.DeletedPrincipalIds = v } -type SourceCacheScope_builder struct { +type SourceCacheRecord_builder struct { _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. // Connector-computed stable identifier for the canonical scope. Must be @@ -257,12 +257,12 @@ type SourceCacheScope_builder struct { // identifiers that correlate with fetch order keep index writes nearly // append-ordered at scale. sourcecache.HashScope is the fallback when no // compact natural form exists. - ScopeHash string + ScopeKey string // Opaque validator to persist for this scope. May be empty on interim // pages of a multi-page scope (e.g. Graph @odata.nextLink pages); the - // SDK writes the scope's manifest entry when a non-empty etag arrives. + // SDK writes the scope's manifest entry when a non-empty cache_validator arrives. // A 200 response with zero rows still persists the entry. - Etag string + CacheValidator string // Tombstones applied after this page's rows commit. Lets every page of // a multi-page delta round carry its own deletions as the provider // delivers them, instead of buffering a whole round onto the first @@ -294,37 +294,37 @@ type SourceCacheScope_builder struct { DeletedPrincipalIds []string } -func (b0 SourceCacheScope_builder) Build() *SourceCacheScope { - m0 := &SourceCacheScope{} +func (b0 SourceCacheRecord_builder) Build() *SourceCacheRecord { + m0 := &SourceCacheRecord{} b, x := &b0, m0 _, _ = b, x - x.ScopeHash = b.ScopeHash - x.Etag = b.Etag + x.ScopeKey = b.ScopeKey + x.CacheValidator = b.CacheValidator x.DeletedIds = b.DeletedIds x.DeletedPrincipalIds = b.DeletedPrincipalIds return m0 } // SourceCacheReplay is attached to a list-response page to tell the SDK to -// copy the previous sync's rows for scope_hash into the current sync. +// copy the previous sync's rows for scope_key into the current sync. // // The row kind (resources, entitlements, grants) is determined by which // RPC the annotation arrived on, never by the annotation itself. // -// The connector must only emit this for a scope whose etag it received +// The connector must only emit this for a scope whose validator it received // from the SDK's source-cache lookup during this same sync. A replay for // an unknown scope fails the sync: the connector has already skipped row // generation, so there is nothing to fall back to. type SourceCacheReplay struct { - state protoimpl.MessageState `protogen:"hybrid.v1"` - ScopeHash string `protobuf:"bytes,1,opt,name=scope_hash,json=scopeHash,proto3" json:"scope_hash,omitempty"` + state protoimpl.MessageState `protogen:"hybrid.v1"` + ScopeKey string `protobuf:"bytes,1,opt,name=scope_key,json=scopeKey,proto3" json:"scope_key,omitempty"` // Validator to persist for this scope in the current sync. For an HTTP - // 304 this is the unchanged etag. For a delta query this is the NEW + // 304 this is the unchanged validator. For a delta query this is the NEW // token; it may instead be supplied by the final overlay page's - // SourceCacheScope.etag, in which case this field may be left empty. - Etag string `protobuf:"bytes,2,opt,name=etag,proto3" json:"etag,omitempty"` + // SourceCacheRecord.cache_validator, in which case this field may be left empty. + CacheValidator string `protobuf:"bytes,2,opt,name=cache_validator,json=cacheValidator,proto3" json:"cache_validator,omitempty"` // When true, the response (and subsequent pages carrying - // SourceCacheScope with the same scope_hash) contains changed rows to + // SourceCacheRecord with the same scope_key) contains changed rows to // upsert on top of the replayed base. When false the response must // contain no rows for this scope. Overlay bool `protobuf:"varint,3,opt,name=overlay,proto3" json:"overlay,omitempty"` @@ -332,10 +332,10 @@ type SourceCacheReplay struct { // RowKindResources) to delete from the current sync after the replay // copy and this page's upserts. Used for delta-query tombstones (e.g. // Microsoft Graph @removed entries). Subsequent pages of the round - // carry their tombstones on SourceCacheScope.deleted_ids. + // carry their tombstones on SourceCacheRecord.deleted_ids. DeletedIds []string `protobuf:"bytes,4,rep,name=deleted_ids,json=deletedIds,proto3" json:"deleted_ids,omitempty"` // Principal-scoped tombstones for this page; see - // SourceCacheScope.deleted_principal_ids for semantics and + // SourceCacheRecord.deleted_principal_ids for semantics and // preconditions. DeletedPrincipalIds []string `protobuf:"bytes,5,rep,name=deleted_principal_ids,json=deletedPrincipalIds,proto3" json:"deleted_principal_ids,omitempty"` unknownFields protoimpl.UnknownFields @@ -367,16 +367,16 @@ func (x *SourceCacheReplay) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -func (x *SourceCacheReplay) GetScopeHash() string { +func (x *SourceCacheReplay) GetScopeKey() string { if x != nil { - return x.ScopeHash + return x.ScopeKey } return "" } -func (x *SourceCacheReplay) GetEtag() string { +func (x *SourceCacheReplay) GetCacheValidator() string { if x != nil { - return x.Etag + return x.CacheValidator } return "" } @@ -402,12 +402,12 @@ func (x *SourceCacheReplay) GetDeletedPrincipalIds() []string { return nil } -func (x *SourceCacheReplay) SetScopeHash(v string) { - x.ScopeHash = v +func (x *SourceCacheReplay) SetScopeKey(v string) { + x.ScopeKey = v } -func (x *SourceCacheReplay) SetEtag(v string) { - x.Etag = v +func (x *SourceCacheReplay) SetCacheValidator(v string) { + x.CacheValidator = v } func (x *SourceCacheReplay) SetOverlay(v bool) { @@ -425,14 +425,14 @@ func (x *SourceCacheReplay) SetDeletedPrincipalIds(v []string) { type SourceCacheReplay_builder struct { _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. - ScopeHash string + ScopeKey string // Validator to persist for this scope in the current sync. For an HTTP - // 304 this is the unchanged etag. For a delta query this is the NEW + // 304 this is the unchanged validator. For a delta query this is the NEW // token; it may instead be supplied by the final overlay page's - // SourceCacheScope.etag, in which case this field may be left empty. - Etag string + // SourceCacheRecord.cache_validator, in which case this field may be left empty. + CacheValidator string // When true, the response (and subsequent pages carrying - // SourceCacheScope with the same scope_hash) contains changed rows to + // SourceCacheRecord with the same scope_key) contains changed rows to // upsert on top of the replayed base. When false the response must // contain no rows for this scope. Overlay bool @@ -440,10 +440,10 @@ type SourceCacheReplay_builder struct { // RowKindResources) to delete from the current sync after the replay // copy and this page's upserts. Used for delta-query tombstones (e.g. // Microsoft Graph @removed entries). Subsequent pages of the round - // carry their tombstones on SourceCacheScope.deleted_ids. + // carry their tombstones on SourceCacheRecord.deleted_ids. DeletedIds []string // Principal-scoped tombstones for this page; see - // SourceCacheScope.deleted_principal_ids for semantics and + // SourceCacheRecord.deleted_principal_ids for semantics and // preconditions. DeletedPrincipalIds []string } @@ -452,8 +452,8 @@ func (b0 SourceCacheReplay_builder) Build() *SourceCacheReplay { m0 := &SourceCacheReplay{} b, x := &b0, m0 _, _ = b, x - x.ScopeHash = b.ScopeHash - x.Etag = b.Etag + x.ScopeKey = b.ScopeKey + x.CacheValidator = b.CacheValidator x.Overlay = b.Overlay x.DeletedIds = b.DeletedIds x.DeletedPrincipalIds = b.DeletedPrincipalIds @@ -582,11 +582,11 @@ func (b0 SourceCacheLookupAsk_builder) Build() *SourceCacheLookupAsk { // REQUEST, carrying the resolution of a prior ask's queries. // // Every query of the prior ask is answered — found=true with the -// previous sync's etag, or found=false meaning fetch fresh. A found -// answer whose etag does not fit the per-request answer size budget is +// previous sync's validator, or found=false meaning fetch fresh. A found +// answer whose validator does not fit the per-request answer size budget is // delivered as found=false: the scope degrades to a cold fetch // (correct, just slower) instead of staying unresolved, because answers -// accumulate across bounces and an etag that did not fit once can never +// accumulate across bounces and a validator that did not fit once can never // fit a later re-invoke. An ABSENT query (a phase-2 lookup for a scope // the connector did not ask before) means unresolved: the connector may // ask again, subject to the bounce cap. @@ -660,7 +660,7 @@ type SourceCacheLookupAsk_Query struct { // One of the sourcecache.RowKind values: "resources", // "entitlements", "grants". RowKind string `protobuf:"bytes,1,opt,name=row_kind,json=rowKind,proto3" json:"row_kind,omitempty"` - ScopeHash string `protobuf:"bytes,2,opt,name=scope_hash,json=scopeHash,proto3" json:"scope_hash,omitempty"` + ScopeKey string `protobuf:"bytes,2,opt,name=scope_key,json=scopeKey,proto3" json:"scope_key,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -697,9 +697,9 @@ func (x *SourceCacheLookupAsk_Query) GetRowKind() string { return "" } -func (x *SourceCacheLookupAsk_Query) GetScopeHash() string { +func (x *SourceCacheLookupAsk_Query) GetScopeKey() string { if x != nil { - return x.ScopeHash + return x.ScopeKey } return "" } @@ -708,8 +708,8 @@ func (x *SourceCacheLookupAsk_Query) SetRowKind(v string) { x.RowKind = v } -func (x *SourceCacheLookupAsk_Query) SetScopeHash(v string) { - x.ScopeHash = v +func (x *SourceCacheLookupAsk_Query) SetScopeKey(v string) { + x.ScopeKey = v } type SourceCacheLookupAsk_Query_builder struct { @@ -717,8 +717,8 @@ type SourceCacheLookupAsk_Query_builder struct { // One of the sourcecache.RowKind values: "resources", // "entitlements", "grants". - RowKind string - ScopeHash string + RowKind string + ScopeKey string } func (b0 SourceCacheLookupAsk_Query_builder) Build() *SourceCacheLookupAsk_Query { @@ -726,20 +726,20 @@ func (b0 SourceCacheLookupAsk_Query_builder) Build() *SourceCacheLookupAsk_Query b, x := &b0, m0 _, _ = b, x x.RowKind = b.RowKind - x.ScopeHash = b.ScopeHash + x.ScopeKey = b.ScopeKey return m0 } type SourceCacheLookupAnswers_Answer struct { - state protoimpl.MessageState `protogen:"hybrid.v1"` - RowKind string `protobuf:"bytes,1,opt,name=row_kind,json=rowKind,proto3" json:"row_kind,omitempty"` - ScopeHash string `protobuf:"bytes,2,opt,name=scope_hash,json=scopeHash,proto3" json:"scope_hash,omitempty"` - Found bool `protobuf:"varint,3,opt,name=found,proto3" json:"found,omitempty"` + state protoimpl.MessageState `protogen:"hybrid.v1"` + RowKind string `protobuf:"bytes,1,opt,name=row_kind,json=rowKind,proto3" json:"row_kind,omitempty"` + ScopeKey string `protobuf:"bytes,2,opt,name=scope_key,json=scopeKey,proto3" json:"scope_key,omitempty"` + Found bool `protobuf:"varint,3,opt,name=found,proto3" json:"found,omitempty"` // The previous sync's validator; empty when found is false. Cap // matches the lookup RPC (Graph delta tokens run long). - Etag string `protobuf:"bytes,4,opt,name=etag,proto3" json:"etag,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + CacheValidator string `protobuf:"bytes,4,opt,name=cache_validator,json=cacheValidator,proto3" json:"cache_validator,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *SourceCacheLookupAnswers_Answer) Reset() { @@ -774,9 +774,9 @@ func (x *SourceCacheLookupAnswers_Answer) GetRowKind() string { return "" } -func (x *SourceCacheLookupAnswers_Answer) GetScopeHash() string { +func (x *SourceCacheLookupAnswers_Answer) GetScopeKey() string { if x != nil { - return x.ScopeHash + return x.ScopeKey } return "" } @@ -788,9 +788,9 @@ func (x *SourceCacheLookupAnswers_Answer) GetFound() bool { return false } -func (x *SourceCacheLookupAnswers_Answer) GetEtag() string { +func (x *SourceCacheLookupAnswers_Answer) GetCacheValidator() string { if x != nil { - return x.Etag + return x.CacheValidator } return "" } @@ -799,27 +799,27 @@ func (x *SourceCacheLookupAnswers_Answer) SetRowKind(v string) { x.RowKind = v } -func (x *SourceCacheLookupAnswers_Answer) SetScopeHash(v string) { - x.ScopeHash = v +func (x *SourceCacheLookupAnswers_Answer) SetScopeKey(v string) { + x.ScopeKey = v } func (x *SourceCacheLookupAnswers_Answer) SetFound(v bool) { x.Found = v } -func (x *SourceCacheLookupAnswers_Answer) SetEtag(v string) { - x.Etag = v +func (x *SourceCacheLookupAnswers_Answer) SetCacheValidator(v string) { + x.CacheValidator = v } type SourceCacheLookupAnswers_Answer_builder struct { _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. - RowKind string - ScopeHash string - Found bool + RowKind string + ScopeKey string + Found bool // The previous sync's validator; empty when found is false. Cap // matches the lookup RPC (Graph delta tokens run long). - Etag string + CacheValidator string } func (b0 SourceCacheLookupAnswers_Answer_builder) Build() *SourceCacheLookupAnswers_Answer { @@ -827,9 +827,9 @@ func (b0 SourceCacheLookupAnswers_Answer_builder) Build() *SourceCacheLookupAnsw b, x := &b0, m0 _, _ = b, x x.RowKind = b.RowKind - x.ScopeHash = b.ScopeHash + x.ScopeKey = b.ScopeKey x.Found = b.Found - x.Etag = b.Etag + x.CacheValidator = b.CacheValidator return m0 } @@ -843,47 +843,43 @@ const file_c1_connector_v2_annotation_source_cache_proto_rawDesc = "" + "\x04Mode\x12\x14\n" + "\x10MODE_UNSPECIFIED\x10\x00\x12\x11\n" + "\rMODE_DISABLED\x10\x01\x12\x13\n" + - "\x0fMODE_READ_WRITE\x10\x02\"\x9a\x01\n" + - "\x10SourceCacheScope\x12\x1d\n" + - "\n" + - "scope_hash\x18\x01 \x01(\tR\tscopeHash\x12\x12\n" + - "\x04etag\x18\x02 \x01(\tR\x04etag\x12\x1f\n" + + "\x0fMODE_READ_WRITE\x10\x02\"\xae\x01\n" + + "\x11SourceCacheRecord\x12\x1b\n" + + "\tscope_key\x18\x01 \x01(\tR\bscopeKey\x12'\n" + + "\x0fcache_validator\x18\x02 \x01(\tR\x0ecacheValidator\x12\x1f\n" + "\vdeleted_ids\x18\x03 \x03(\tR\n" + "deletedIds\x122\n" + - "\x15deleted_principal_ids\x18\x04 \x03(\tR\x13deletedPrincipalIds\"\xb5\x01\n" + - "\x11SourceCacheReplay\x12\x1d\n" + - "\n" + - "scope_hash\x18\x01 \x01(\tR\tscopeHash\x12\x12\n" + - "\x04etag\x18\x02 \x01(\tR\x04etag\x12\x18\n" + + "\x15deleted_principal_ids\x18\x04 \x03(\tR\x13deletedPrincipalIds\"\xc8\x01\n" + + "\x11SourceCacheReplay\x12\x1b\n" + + "\tscope_key\x18\x01 \x01(\tR\bscopeKey\x12'\n" + + "\x0fcache_validator\x18\x02 \x01(\tR\x0ecacheValidator\x12\x18\n" + "\aoverlay\x18\x03 \x01(\bR\aoverlay\x12\x1f\n" + "\vdeleted_ids\x18\x04 \x03(\tR\n" + "deletedIds\x122\n" + "\x15deleted_principal_ids\x18\x05 \x03(\tR\x13deletedPrincipalIds\"\x18\n" + - "\x16SourceCacheLookupOffer\"\xc4\x01\n" + + "\x16SourceCacheLookupOffer\"\xc2\x01\n" + "\x14SourceCacheLookupAsk\x12R\n" + - "\aqueries\x18\x01 \x03(\v2+.c1.connector.v2.SourceCacheLookupAsk.QueryB\v\xfaB\b\x92\x01\x05\b\x01\x10\x80 R\aqueries\x1aX\n" + + "\aqueries\x18\x01 \x03(\v2+.c1.connector.v2.SourceCacheLookupAsk.QueryB\v\xfaB\b\x92\x01\x05\b\x01\x10\x80 R\aqueries\x1aV\n" + "\x05Query\x12$\n" + - "\brow_kind\x18\x01 \x01(\tB\t\xfaB\x06r\x04 \x01(@R\arowKind\x12)\n" + - "\n" + - "scope_hash\x18\x02 \x01(\tB\n" + - "\xfaB\ar\x05 \x01(\x80\x02R\tscopeHash\"\x86\x02\n" + + "\brow_kind\x18\x01 \x01(\tB\t\xfaB\x06r\x04 \x01(@R\arowKind\x12'\n" + + "\tscope_key\x18\x02 \x01(\tB\n" + + "\xfaB\ar\x05 \x01(\x80\x02R\bscopeKey\"\x99\x02\n" + "\x18SourceCacheLookupAnswers\x12V\n" + "\aanswers\x18\x01 \x03(\v20.c1.connector.v2.SourceCacheLookupAnswers.AnswerB\n" + - "\xfaB\a\x92\x01\x04\x10\x80\x80\x01R\aanswers\x1a\x91\x01\n" + + "\xfaB\a\x92\x01\x04\x10\x80\x80\x01R\aanswers\x1a\xa4\x01\n" + "\x06Answer\x12$\n" + - "\brow_kind\x18\x01 \x01(\tB\t\xfaB\x06r\x04 \x01(@R\arowKind\x12)\n" + - "\n" + - "scope_hash\x18\x02 \x01(\tB\n" + - "\xfaB\ar\x05 \x01(\x80\x02R\tscopeHash\x12\x14\n" + - "\x05found\x18\x03 \x01(\bR\x05found\x12 \n" + - "\x04etag\x18\x04 \x01(\tB\f\xfaB\tr\a(\x80\x80\x04\xd0\x01\x01R\x04etagB6Z4github.com/conductorone/baton-sdk/pb/c1/connector/v2b\x06proto3" + "\brow_kind\x18\x01 \x01(\tB\t\xfaB\x06r\x04 \x01(@R\arowKind\x12'\n" + + "\tscope_key\x18\x02 \x01(\tB\n" + + "\xfaB\ar\x05 \x01(\x80\x02R\bscopeKey\x12\x14\n" + + "\x05found\x18\x03 \x01(\bR\x05found\x125\n" + + "\x0fcache_validator\x18\x04 \x01(\tB\f\xfaB\tr\a(\x80\x80\x04\xd0\x01\x01R\x0ecacheValidatorB6Z4github.com/conductorone/baton-sdk/pb/c1/connector/v2b\x06proto3" var file_c1_connector_v2_annotation_source_cache_proto_enumTypes = make([]protoimpl.EnumInfo, 1) var file_c1_connector_v2_annotation_source_cache_proto_msgTypes = make([]protoimpl.MessageInfo, 8) var file_c1_connector_v2_annotation_source_cache_proto_goTypes = []any{ (SourceCacheCapability_Mode)(0), // 0: c1.connector.v2.SourceCacheCapability.Mode (*SourceCacheCapability)(nil), // 1: c1.connector.v2.SourceCacheCapability - (*SourceCacheScope)(nil), // 2: c1.connector.v2.SourceCacheScope + (*SourceCacheRecord)(nil), // 2: c1.connector.v2.SourceCacheRecord (*SourceCacheReplay)(nil), // 3: c1.connector.v2.SourceCacheReplay (*SourceCacheLookupOffer)(nil), // 4: c1.connector.v2.SourceCacheLookupOffer (*SourceCacheLookupAsk)(nil), // 5: c1.connector.v2.SourceCacheLookupAsk diff --git a/pb/c1/connector/v2/annotation_source_cache.pb.validate.go b/pb/c1/connector/v2/annotation_source_cache.pb.validate.go index 57afb268a..a60200dce 100644 --- a/pb/c1/connector/v2/annotation_source_cache.pb.validate.go +++ b/pb/c1/connector/v2/annotation_source_cache.pb.validate.go @@ -139,46 +139,46 @@ var _ interface { ErrorName() string } = SourceCacheCapabilityValidationError{} -// Validate checks the field values on SourceCacheScope with the rules defined +// Validate checks the field values on SourceCacheRecord with the rules defined // in the proto definition for this message. If any rules are violated, the // first error encountered is returned, or nil if there are no violations. -func (m *SourceCacheScope) Validate() error { +func (m *SourceCacheRecord) Validate() error { return m.validate(false) } -// ValidateAll checks the field values on SourceCacheScope with the rules +// ValidateAll checks the field values on SourceCacheRecord with the rules // defined in the proto definition for this message. If any rules are // violated, the result is a list of violation errors wrapped in -// SourceCacheScopeMultiError, or nil if none found. -func (m *SourceCacheScope) ValidateAll() error { +// SourceCacheRecordMultiError, or nil if none found. +func (m *SourceCacheRecord) ValidateAll() error { return m.validate(true) } -func (m *SourceCacheScope) validate(all bool) error { +func (m *SourceCacheRecord) validate(all bool) error { if m == nil { return nil } var errors []error - // no validation rules for ScopeHash + // no validation rules for ScopeKey - // no validation rules for Etag + // no validation rules for CacheValidator if len(errors) > 0 { - return SourceCacheScopeMultiError(errors) + return SourceCacheRecordMultiError(errors) } return nil } -// SourceCacheScopeMultiError is an error wrapping multiple validation errors -// returned by SourceCacheScope.ValidateAll() if the designated constraints +// SourceCacheRecordMultiError is an error wrapping multiple validation errors +// returned by SourceCacheRecord.ValidateAll() if the designated constraints // aren't met. -type SourceCacheScopeMultiError []error +type SourceCacheRecordMultiError []error // Error returns a concatenation of all the error messages it wraps. -func (m SourceCacheScopeMultiError) Error() string { +func (m SourceCacheRecordMultiError) Error() string { msgs := make([]string, 0, len(m)) for _, err := range m { msgs = append(msgs, err.Error()) @@ -187,11 +187,11 @@ func (m SourceCacheScopeMultiError) Error() string { } // AllErrors returns a list of validation violation errors. -func (m SourceCacheScopeMultiError) AllErrors() []error { return m } +func (m SourceCacheRecordMultiError) AllErrors() []error { return m } -// SourceCacheScopeValidationError is the validation error returned by -// SourceCacheScope.Validate if the designated constraints aren't met. -type SourceCacheScopeValidationError struct { +// SourceCacheRecordValidationError is the validation error returned by +// SourceCacheRecord.Validate if the designated constraints aren't met. +type SourceCacheRecordValidationError struct { field string reason string cause error @@ -199,22 +199,24 @@ type SourceCacheScopeValidationError struct { } // Field function returns field value. -func (e SourceCacheScopeValidationError) Field() string { return e.field } +func (e SourceCacheRecordValidationError) Field() string { return e.field } // Reason function returns reason value. -func (e SourceCacheScopeValidationError) Reason() string { return e.reason } +func (e SourceCacheRecordValidationError) Reason() string { return e.reason } // Cause function returns cause value. -func (e SourceCacheScopeValidationError) Cause() error { return e.cause } +func (e SourceCacheRecordValidationError) Cause() error { return e.cause } // Key function returns key value. -func (e SourceCacheScopeValidationError) Key() bool { return e.key } +func (e SourceCacheRecordValidationError) Key() bool { return e.key } // ErrorName returns error name. -func (e SourceCacheScopeValidationError) ErrorName() string { return "SourceCacheScopeValidationError" } +func (e SourceCacheRecordValidationError) ErrorName() string { + return "SourceCacheRecordValidationError" +} // Error satisfies the builtin error interface -func (e SourceCacheScopeValidationError) Error() string { +func (e SourceCacheRecordValidationError) Error() string { cause := "" if e.cause != nil { cause = fmt.Sprintf(" | caused by: %v", e.cause) @@ -226,14 +228,14 @@ func (e SourceCacheScopeValidationError) Error() string { } return fmt.Sprintf( - "invalid %sSourceCacheScope.%s: %s%s", + "invalid %sSourceCacheRecord.%s: %s%s", key, e.field, e.reason, cause) } -var _ error = SourceCacheScopeValidationError{} +var _ error = SourceCacheRecordValidationError{} var _ interface { Field() string @@ -241,7 +243,7 @@ var _ interface { Key() bool Cause() error ErrorName() string -} = SourceCacheScopeValidationError{} +} = SourceCacheRecordValidationError{} // Validate checks the field values on SourceCacheReplay with the rules defined // in the proto definition for this message. If any rules are violated, the @@ -265,9 +267,9 @@ func (m *SourceCacheReplay) validate(all bool) error { var errors []error - // no validation rules for ScopeHash + // no validation rules for ScopeKey - // no validation rules for Etag + // no validation rules for CacheValidator // no validation rules for Overlay @@ -780,9 +782,9 @@ func (m *SourceCacheLookupAsk_Query) validate(all bool) error { errors = append(errors, err) } - if l := len(m.GetScopeHash()); l < 1 || l > 256 { + if l := len(m.GetScopeKey()); l < 1 || l > 256 { err := SourceCacheLookupAsk_QueryValidationError{ - field: "ScopeHash", + field: "ScopeKey", reason: "value length must be between 1 and 256 bytes, inclusive", } if !all { @@ -904,9 +906,9 @@ func (m *SourceCacheLookupAnswers_Answer) validate(all bool) error { errors = append(errors, err) } - if l := len(m.GetScopeHash()); l < 1 || l > 256 { + if l := len(m.GetScopeKey()); l < 1 || l > 256 { err := SourceCacheLookupAnswers_AnswerValidationError{ - field: "ScopeHash", + field: "ScopeKey", reason: "value length must be between 1 and 256 bytes, inclusive", } if !all { @@ -917,11 +919,11 @@ func (m *SourceCacheLookupAnswers_Answer) validate(all bool) error { // no validation rules for Found - if m.GetEtag() != "" { + if m.GetCacheValidator() != "" { - if len(m.GetEtag()) > 65536 { + if len(m.GetCacheValidator()) > 65536 { err := SourceCacheLookupAnswers_AnswerValidationError{ - field: "Etag", + field: "CacheValidator", reason: "value length must be at most 65536 bytes", } if !all { diff --git a/pb/c1/connector/v2/annotation_source_cache_protoopaque.pb.go b/pb/c1/connector/v2/annotation_source_cache_protoopaque.pb.go index c8a115196..e44b120f5 100644 --- a/pb/c1/connector/v2/annotation_source_cache_protoopaque.pb.go +++ b/pb/c1/connector/v2/annotation_source_cache_protoopaque.pb.go @@ -127,33 +127,33 @@ func (b0 SourceCacheCapability_builder) Build() *SourceCacheCapability { return m0 } -// SourceCacheScope is attached to a list-response page whose rows were +// SourceCacheRecord is attached to a list-response page whose rows were // freshly fetched from upstream. The SDK stamps the page's rows with -// scope_hash so a future sync can replay them as a unit. -type SourceCacheScope struct { +// scope_key so a future sync can replay them as a unit. +type SourceCacheRecord struct { state protoimpl.MessageState `protogen:"opaque.v1"` - xxx_hidden_ScopeHash string `protobuf:"bytes,1,opt,name=scope_hash,json=scopeHash,proto3"` - xxx_hidden_Etag string `protobuf:"bytes,2,opt,name=etag,proto3"` + xxx_hidden_ScopeKey string `protobuf:"bytes,1,opt,name=scope_key,json=scopeKey,proto3"` + xxx_hidden_CacheValidator string `protobuf:"bytes,2,opt,name=cache_validator,json=cacheValidator,proto3"` xxx_hidden_DeletedIds []string `protobuf:"bytes,3,rep,name=deleted_ids,json=deletedIds,proto3"` xxx_hidden_DeletedPrincipalIds []string `protobuf:"bytes,4,rep,name=deleted_principal_ids,json=deletedPrincipalIds,proto3"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } -func (x *SourceCacheScope) Reset() { - *x = SourceCacheScope{} +func (x *SourceCacheRecord) Reset() { + *x = SourceCacheRecord{} mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[1] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } -func (x *SourceCacheScope) String() string { +func (x *SourceCacheRecord) String() string { return protoimpl.X.MessageStringOf(x) } -func (*SourceCacheScope) ProtoMessage() {} +func (*SourceCacheRecord) ProtoMessage() {} -func (x *SourceCacheScope) ProtoReflect() protoreflect.Message { +func (x *SourceCacheRecord) ProtoReflect() protoreflect.Message { mi := &file_c1_connector_v2_annotation_source_cache_proto_msgTypes[1] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -165,51 +165,51 @@ func (x *SourceCacheScope) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -func (x *SourceCacheScope) GetScopeHash() string { +func (x *SourceCacheRecord) GetScopeKey() string { if x != nil { - return x.xxx_hidden_ScopeHash + return x.xxx_hidden_ScopeKey } return "" } -func (x *SourceCacheScope) GetEtag() string { +func (x *SourceCacheRecord) GetCacheValidator() string { if x != nil { - return x.xxx_hidden_Etag + return x.xxx_hidden_CacheValidator } return "" } -func (x *SourceCacheScope) GetDeletedIds() []string { +func (x *SourceCacheRecord) GetDeletedIds() []string { if x != nil { return x.xxx_hidden_DeletedIds } return nil } -func (x *SourceCacheScope) GetDeletedPrincipalIds() []string { +func (x *SourceCacheRecord) GetDeletedPrincipalIds() []string { if x != nil { return x.xxx_hidden_DeletedPrincipalIds } return nil } -func (x *SourceCacheScope) SetScopeHash(v string) { - x.xxx_hidden_ScopeHash = v +func (x *SourceCacheRecord) SetScopeKey(v string) { + x.xxx_hidden_ScopeKey = v } -func (x *SourceCacheScope) SetEtag(v string) { - x.xxx_hidden_Etag = v +func (x *SourceCacheRecord) SetCacheValidator(v string) { + x.xxx_hidden_CacheValidator = v } -func (x *SourceCacheScope) SetDeletedIds(v []string) { +func (x *SourceCacheRecord) SetDeletedIds(v []string) { x.xxx_hidden_DeletedIds = v } -func (x *SourceCacheScope) SetDeletedPrincipalIds(v []string) { +func (x *SourceCacheRecord) SetDeletedPrincipalIds(v []string) { x.xxx_hidden_DeletedPrincipalIds = v } -type SourceCacheScope_builder struct { +type SourceCacheRecord_builder struct { _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. // Connector-computed stable identifier for the canonical scope. Must be @@ -219,12 +219,12 @@ type SourceCacheScope_builder struct { // identifiers that correlate with fetch order keep index writes nearly // append-ordered at scale. sourcecache.HashScope is the fallback when no // compact natural form exists. - ScopeHash string + ScopeKey string // Opaque validator to persist for this scope. May be empty on interim // pages of a multi-page scope (e.g. Graph @odata.nextLink pages); the - // SDK writes the scope's manifest entry when a non-empty etag arrives. + // SDK writes the scope's manifest entry when a non-empty cache_validator arrives. // A 200 response with zero rows still persists the entry. - Etag string + CacheValidator string // Tombstones applied after this page's rows commit. Lets every page of // a multi-page delta round carry its own deletions as the provider // delivers them, instead of buffering a whole round onto the first @@ -256,31 +256,31 @@ type SourceCacheScope_builder struct { DeletedPrincipalIds []string } -func (b0 SourceCacheScope_builder) Build() *SourceCacheScope { - m0 := &SourceCacheScope{} +func (b0 SourceCacheRecord_builder) Build() *SourceCacheRecord { + m0 := &SourceCacheRecord{} b, x := &b0, m0 _, _ = b, x - x.xxx_hidden_ScopeHash = b.ScopeHash - x.xxx_hidden_Etag = b.Etag + x.xxx_hidden_ScopeKey = b.ScopeKey + x.xxx_hidden_CacheValidator = b.CacheValidator x.xxx_hidden_DeletedIds = b.DeletedIds x.xxx_hidden_DeletedPrincipalIds = b.DeletedPrincipalIds return m0 } // SourceCacheReplay is attached to a list-response page to tell the SDK to -// copy the previous sync's rows for scope_hash into the current sync. +// copy the previous sync's rows for scope_key into the current sync. // // The row kind (resources, entitlements, grants) is determined by which // RPC the annotation arrived on, never by the annotation itself. // -// The connector must only emit this for a scope whose etag it received +// The connector must only emit this for a scope whose validator it received // from the SDK's source-cache lookup during this same sync. A replay for // an unknown scope fails the sync: the connector has already skipped row // generation, so there is nothing to fall back to. type SourceCacheReplay struct { state protoimpl.MessageState `protogen:"opaque.v1"` - xxx_hidden_ScopeHash string `protobuf:"bytes,1,opt,name=scope_hash,json=scopeHash,proto3"` - xxx_hidden_Etag string `protobuf:"bytes,2,opt,name=etag,proto3"` + xxx_hidden_ScopeKey string `protobuf:"bytes,1,opt,name=scope_key,json=scopeKey,proto3"` + xxx_hidden_CacheValidator string `protobuf:"bytes,2,opt,name=cache_validator,json=cacheValidator,proto3"` xxx_hidden_Overlay bool `protobuf:"varint,3,opt,name=overlay,proto3"` xxx_hidden_DeletedIds []string `protobuf:"bytes,4,rep,name=deleted_ids,json=deletedIds,proto3"` xxx_hidden_DeletedPrincipalIds []string `protobuf:"bytes,5,rep,name=deleted_principal_ids,json=deletedPrincipalIds,proto3"` @@ -313,16 +313,16 @@ func (x *SourceCacheReplay) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -func (x *SourceCacheReplay) GetScopeHash() string { +func (x *SourceCacheReplay) GetScopeKey() string { if x != nil { - return x.xxx_hidden_ScopeHash + return x.xxx_hidden_ScopeKey } return "" } -func (x *SourceCacheReplay) GetEtag() string { +func (x *SourceCacheReplay) GetCacheValidator() string { if x != nil { - return x.xxx_hidden_Etag + return x.xxx_hidden_CacheValidator } return "" } @@ -348,12 +348,12 @@ func (x *SourceCacheReplay) GetDeletedPrincipalIds() []string { return nil } -func (x *SourceCacheReplay) SetScopeHash(v string) { - x.xxx_hidden_ScopeHash = v +func (x *SourceCacheReplay) SetScopeKey(v string) { + x.xxx_hidden_ScopeKey = v } -func (x *SourceCacheReplay) SetEtag(v string) { - x.xxx_hidden_Etag = v +func (x *SourceCacheReplay) SetCacheValidator(v string) { + x.xxx_hidden_CacheValidator = v } func (x *SourceCacheReplay) SetOverlay(v bool) { @@ -371,14 +371,14 @@ func (x *SourceCacheReplay) SetDeletedPrincipalIds(v []string) { type SourceCacheReplay_builder struct { _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. - ScopeHash string + ScopeKey string // Validator to persist for this scope in the current sync. For an HTTP - // 304 this is the unchanged etag. For a delta query this is the NEW + // 304 this is the unchanged validator. For a delta query this is the NEW // token; it may instead be supplied by the final overlay page's - // SourceCacheScope.etag, in which case this field may be left empty. - Etag string + // SourceCacheRecord.cache_validator, in which case this field may be left empty. + CacheValidator string // When true, the response (and subsequent pages carrying - // SourceCacheScope with the same scope_hash) contains changed rows to + // SourceCacheRecord with the same scope_key) contains changed rows to // upsert on top of the replayed base. When false the response must // contain no rows for this scope. Overlay bool @@ -386,10 +386,10 @@ type SourceCacheReplay_builder struct { // RowKindResources) to delete from the current sync after the replay // copy and this page's upserts. Used for delta-query tombstones (e.g. // Microsoft Graph @removed entries). Subsequent pages of the round - // carry their tombstones on SourceCacheScope.deleted_ids. + // carry their tombstones on SourceCacheRecord.deleted_ids. DeletedIds []string // Principal-scoped tombstones for this page; see - // SourceCacheScope.deleted_principal_ids for semantics and + // SourceCacheRecord.deleted_principal_ids for semantics and // preconditions. DeletedPrincipalIds []string } @@ -398,8 +398,8 @@ func (b0 SourceCacheReplay_builder) Build() *SourceCacheReplay { m0 := &SourceCacheReplay{} b, x := &b0, m0 _, _ = b, x - x.xxx_hidden_ScopeHash = b.ScopeHash - x.xxx_hidden_Etag = b.Etag + x.xxx_hidden_ScopeKey = b.ScopeKey + x.xxx_hidden_CacheValidator = b.CacheValidator x.xxx_hidden_Overlay = b.Overlay x.xxx_hidden_DeletedIds = b.DeletedIds x.xxx_hidden_DeletedPrincipalIds = b.DeletedPrincipalIds @@ -530,11 +530,11 @@ func (b0 SourceCacheLookupAsk_builder) Build() *SourceCacheLookupAsk { // REQUEST, carrying the resolution of a prior ask's queries. // // Every query of the prior ask is answered — found=true with the -// previous sync's etag, or found=false meaning fetch fresh. A found -// answer whose etag does not fit the per-request answer size budget is +// previous sync's validator, or found=false meaning fetch fresh. A found +// answer whose validator does not fit the per-request answer size budget is // delivered as found=false: the scope degrades to a cold fetch // (correct, just slower) instead of staying unresolved, because answers -// accumulate across bounces and an etag that did not fit once can never +// accumulate across bounces and a validator that did not fit once can never // fit a later re-invoke. An ABSENT query (a phase-2 lookup for a scope // the connector did not ask before) means unresolved: the connector may // ask again, subject to the bounce cap. @@ -602,11 +602,11 @@ func (b0 SourceCacheLookupAnswers_builder) Build() *SourceCacheLookupAnswers { } type SourceCacheLookupAsk_Query struct { - state protoimpl.MessageState `protogen:"opaque.v1"` - xxx_hidden_RowKind string `protobuf:"bytes,1,opt,name=row_kind,json=rowKind,proto3"` - xxx_hidden_ScopeHash string `protobuf:"bytes,2,opt,name=scope_hash,json=scopeHash,proto3"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"opaque.v1"` + xxx_hidden_RowKind string `protobuf:"bytes,1,opt,name=row_kind,json=rowKind,proto3"` + xxx_hidden_ScopeKey string `protobuf:"bytes,2,opt,name=scope_key,json=scopeKey,proto3"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *SourceCacheLookupAsk_Query) Reset() { @@ -641,9 +641,9 @@ func (x *SourceCacheLookupAsk_Query) GetRowKind() string { return "" } -func (x *SourceCacheLookupAsk_Query) GetScopeHash() string { +func (x *SourceCacheLookupAsk_Query) GetScopeKey() string { if x != nil { - return x.xxx_hidden_ScopeHash + return x.xxx_hidden_ScopeKey } return "" } @@ -652,8 +652,8 @@ func (x *SourceCacheLookupAsk_Query) SetRowKind(v string) { x.xxx_hidden_RowKind = v } -func (x *SourceCacheLookupAsk_Query) SetScopeHash(v string) { - x.xxx_hidden_ScopeHash = v +func (x *SourceCacheLookupAsk_Query) SetScopeKey(v string) { + x.xxx_hidden_ScopeKey = v } type SourceCacheLookupAsk_Query_builder struct { @@ -661,8 +661,8 @@ type SourceCacheLookupAsk_Query_builder struct { // One of the sourcecache.RowKind values: "resources", // "entitlements", "grants". - RowKind string - ScopeHash string + RowKind string + ScopeKey string } func (b0 SourceCacheLookupAsk_Query_builder) Build() *SourceCacheLookupAsk_Query { @@ -670,18 +670,18 @@ func (b0 SourceCacheLookupAsk_Query_builder) Build() *SourceCacheLookupAsk_Query b, x := &b0, m0 _, _ = b, x x.xxx_hidden_RowKind = b.RowKind - x.xxx_hidden_ScopeHash = b.ScopeHash + x.xxx_hidden_ScopeKey = b.ScopeKey return m0 } type SourceCacheLookupAnswers_Answer struct { - state protoimpl.MessageState `protogen:"opaque.v1"` - xxx_hidden_RowKind string `protobuf:"bytes,1,opt,name=row_kind,json=rowKind,proto3"` - xxx_hidden_ScopeHash string `protobuf:"bytes,2,opt,name=scope_hash,json=scopeHash,proto3"` - xxx_hidden_Found bool `protobuf:"varint,3,opt,name=found,proto3"` - xxx_hidden_Etag string `protobuf:"bytes,4,opt,name=etag,proto3"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"opaque.v1"` + xxx_hidden_RowKind string `protobuf:"bytes,1,opt,name=row_kind,json=rowKind,proto3"` + xxx_hidden_ScopeKey string `protobuf:"bytes,2,opt,name=scope_key,json=scopeKey,proto3"` + xxx_hidden_Found bool `protobuf:"varint,3,opt,name=found,proto3"` + xxx_hidden_CacheValidator string `protobuf:"bytes,4,opt,name=cache_validator,json=cacheValidator,proto3"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *SourceCacheLookupAnswers_Answer) Reset() { @@ -716,9 +716,9 @@ func (x *SourceCacheLookupAnswers_Answer) GetRowKind() string { return "" } -func (x *SourceCacheLookupAnswers_Answer) GetScopeHash() string { +func (x *SourceCacheLookupAnswers_Answer) GetScopeKey() string { if x != nil { - return x.xxx_hidden_ScopeHash + return x.xxx_hidden_ScopeKey } return "" } @@ -730,9 +730,9 @@ func (x *SourceCacheLookupAnswers_Answer) GetFound() bool { return false } -func (x *SourceCacheLookupAnswers_Answer) GetEtag() string { +func (x *SourceCacheLookupAnswers_Answer) GetCacheValidator() string { if x != nil { - return x.xxx_hidden_Etag + return x.xxx_hidden_CacheValidator } return "" } @@ -741,27 +741,27 @@ func (x *SourceCacheLookupAnswers_Answer) SetRowKind(v string) { x.xxx_hidden_RowKind = v } -func (x *SourceCacheLookupAnswers_Answer) SetScopeHash(v string) { - x.xxx_hidden_ScopeHash = v +func (x *SourceCacheLookupAnswers_Answer) SetScopeKey(v string) { + x.xxx_hidden_ScopeKey = v } func (x *SourceCacheLookupAnswers_Answer) SetFound(v bool) { x.xxx_hidden_Found = v } -func (x *SourceCacheLookupAnswers_Answer) SetEtag(v string) { - x.xxx_hidden_Etag = v +func (x *SourceCacheLookupAnswers_Answer) SetCacheValidator(v string) { + x.xxx_hidden_CacheValidator = v } type SourceCacheLookupAnswers_Answer_builder struct { _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. - RowKind string - ScopeHash string - Found bool + RowKind string + ScopeKey string + Found bool // The previous sync's validator; empty when found is false. Cap // matches the lookup RPC (Graph delta tokens run long). - Etag string + CacheValidator string } func (b0 SourceCacheLookupAnswers_Answer_builder) Build() *SourceCacheLookupAnswers_Answer { @@ -769,9 +769,9 @@ func (b0 SourceCacheLookupAnswers_Answer_builder) Build() *SourceCacheLookupAnsw b, x := &b0, m0 _, _ = b, x x.xxx_hidden_RowKind = b.RowKind - x.xxx_hidden_ScopeHash = b.ScopeHash + x.xxx_hidden_ScopeKey = b.ScopeKey x.xxx_hidden_Found = b.Found - x.xxx_hidden_Etag = b.Etag + x.xxx_hidden_CacheValidator = b.CacheValidator return m0 } @@ -785,47 +785,43 @@ const file_c1_connector_v2_annotation_source_cache_proto_rawDesc = "" + "\x04Mode\x12\x14\n" + "\x10MODE_UNSPECIFIED\x10\x00\x12\x11\n" + "\rMODE_DISABLED\x10\x01\x12\x13\n" + - "\x0fMODE_READ_WRITE\x10\x02\"\x9a\x01\n" + - "\x10SourceCacheScope\x12\x1d\n" + - "\n" + - "scope_hash\x18\x01 \x01(\tR\tscopeHash\x12\x12\n" + - "\x04etag\x18\x02 \x01(\tR\x04etag\x12\x1f\n" + + "\x0fMODE_READ_WRITE\x10\x02\"\xae\x01\n" + + "\x11SourceCacheRecord\x12\x1b\n" + + "\tscope_key\x18\x01 \x01(\tR\bscopeKey\x12'\n" + + "\x0fcache_validator\x18\x02 \x01(\tR\x0ecacheValidator\x12\x1f\n" + "\vdeleted_ids\x18\x03 \x03(\tR\n" + "deletedIds\x122\n" + - "\x15deleted_principal_ids\x18\x04 \x03(\tR\x13deletedPrincipalIds\"\xb5\x01\n" + - "\x11SourceCacheReplay\x12\x1d\n" + - "\n" + - "scope_hash\x18\x01 \x01(\tR\tscopeHash\x12\x12\n" + - "\x04etag\x18\x02 \x01(\tR\x04etag\x12\x18\n" + + "\x15deleted_principal_ids\x18\x04 \x03(\tR\x13deletedPrincipalIds\"\xc8\x01\n" + + "\x11SourceCacheReplay\x12\x1b\n" + + "\tscope_key\x18\x01 \x01(\tR\bscopeKey\x12'\n" + + "\x0fcache_validator\x18\x02 \x01(\tR\x0ecacheValidator\x12\x18\n" + "\aoverlay\x18\x03 \x01(\bR\aoverlay\x12\x1f\n" + "\vdeleted_ids\x18\x04 \x03(\tR\n" + "deletedIds\x122\n" + "\x15deleted_principal_ids\x18\x05 \x03(\tR\x13deletedPrincipalIds\"\x18\n" + - "\x16SourceCacheLookupOffer\"\xc4\x01\n" + + "\x16SourceCacheLookupOffer\"\xc2\x01\n" + "\x14SourceCacheLookupAsk\x12R\n" + - "\aqueries\x18\x01 \x03(\v2+.c1.connector.v2.SourceCacheLookupAsk.QueryB\v\xfaB\b\x92\x01\x05\b\x01\x10\x80 R\aqueries\x1aX\n" + + "\aqueries\x18\x01 \x03(\v2+.c1.connector.v2.SourceCacheLookupAsk.QueryB\v\xfaB\b\x92\x01\x05\b\x01\x10\x80 R\aqueries\x1aV\n" + "\x05Query\x12$\n" + - "\brow_kind\x18\x01 \x01(\tB\t\xfaB\x06r\x04 \x01(@R\arowKind\x12)\n" + - "\n" + - "scope_hash\x18\x02 \x01(\tB\n" + - "\xfaB\ar\x05 \x01(\x80\x02R\tscopeHash\"\x86\x02\n" + + "\brow_kind\x18\x01 \x01(\tB\t\xfaB\x06r\x04 \x01(@R\arowKind\x12'\n" + + "\tscope_key\x18\x02 \x01(\tB\n" + + "\xfaB\ar\x05 \x01(\x80\x02R\bscopeKey\"\x99\x02\n" + "\x18SourceCacheLookupAnswers\x12V\n" + "\aanswers\x18\x01 \x03(\v20.c1.connector.v2.SourceCacheLookupAnswers.AnswerB\n" + - "\xfaB\a\x92\x01\x04\x10\x80\x80\x01R\aanswers\x1a\x91\x01\n" + + "\xfaB\a\x92\x01\x04\x10\x80\x80\x01R\aanswers\x1a\xa4\x01\n" + "\x06Answer\x12$\n" + - "\brow_kind\x18\x01 \x01(\tB\t\xfaB\x06r\x04 \x01(@R\arowKind\x12)\n" + - "\n" + - "scope_hash\x18\x02 \x01(\tB\n" + - "\xfaB\ar\x05 \x01(\x80\x02R\tscopeHash\x12\x14\n" + - "\x05found\x18\x03 \x01(\bR\x05found\x12 \n" + - "\x04etag\x18\x04 \x01(\tB\f\xfaB\tr\a(\x80\x80\x04\xd0\x01\x01R\x04etagB6Z4github.com/conductorone/baton-sdk/pb/c1/connector/v2b\x06proto3" + "\brow_kind\x18\x01 \x01(\tB\t\xfaB\x06r\x04 \x01(@R\arowKind\x12'\n" + + "\tscope_key\x18\x02 \x01(\tB\n" + + "\xfaB\ar\x05 \x01(\x80\x02R\bscopeKey\x12\x14\n" + + "\x05found\x18\x03 \x01(\bR\x05found\x125\n" + + "\x0fcache_validator\x18\x04 \x01(\tB\f\xfaB\tr\a(\x80\x80\x04\xd0\x01\x01R\x0ecacheValidatorB6Z4github.com/conductorone/baton-sdk/pb/c1/connector/v2b\x06proto3" var file_c1_connector_v2_annotation_source_cache_proto_enumTypes = make([]protoimpl.EnumInfo, 1) var file_c1_connector_v2_annotation_source_cache_proto_msgTypes = make([]protoimpl.MessageInfo, 8) var file_c1_connector_v2_annotation_source_cache_proto_goTypes = []any{ (SourceCacheCapability_Mode)(0), // 0: c1.connector.v2.SourceCacheCapability.Mode (*SourceCacheCapability)(nil), // 1: c1.connector.v2.SourceCacheCapability - (*SourceCacheScope)(nil), // 2: c1.connector.v2.SourceCacheScope + (*SourceCacheRecord)(nil), // 2: c1.connector.v2.SourceCacheRecord (*SourceCacheReplay)(nil), // 3: c1.connector.v2.SourceCacheReplay (*SourceCacheLookupOffer)(nil), // 4: c1.connector.v2.SourceCacheLookupOffer (*SourceCacheLookupAsk)(nil), // 5: c1.connector.v2.SourceCacheLookupAsk diff --git a/pb/c1/connector/v2/annotation_type_scoped_entitlements.pb.go b/pb/c1/connector/v2/annotation_type_scoped_entitlements.pb.go index 0c7c5025e..81fad6a4a 100644 --- a/pb/c1/connector/v2/annotation_type_scoped_entitlements.pb.go +++ b/pb/c1/connector/v2/annotation_type_scoped_entitlements.pb.go @@ -38,7 +38,7 @@ const ( // ({type, type} — wire validation requires a non-empty resource id and // the stub identifies the type, never a real resource). The connector // answers with entitlements for the whole type, across as many paginated -// cursors as it chooses to spawn (see SpawnCursors in +// cursors as it chooses to spawn (see EnqueuePageTokens in // annotation_type_scoped_grants.proto). // // This is the entitlements-phase analogue of TypeScopedGrants: the @@ -60,7 +60,7 @@ const ( // // ZERO-RESOURCE TYPES: the planner still enqueues the type-scoped action // when the store holds no resources of the type. The connector must -// return an empty page (or SpawnCursors that resolve empty), not an +// return an empty page (or EnqueuePageTokens that resolve empty), not an // error — an empty type is a valid outcome. // // TARGETED SYNC: SyncTargetedResource must not enqueue a per-resource diff --git a/pb/c1/connector/v2/annotation_type_scoped_entitlements_protoopaque.pb.go b/pb/c1/connector/v2/annotation_type_scoped_entitlements_protoopaque.pb.go index 45ac5c10f..d65f6c11c 100644 --- a/pb/c1/connector/v2/annotation_type_scoped_entitlements_protoopaque.pb.go +++ b/pb/c1/connector/v2/annotation_type_scoped_entitlements_protoopaque.pb.go @@ -38,7 +38,7 @@ const ( // ({type, type} — wire validation requires a non-empty resource id and // the stub identifies the type, never a real resource). The connector // answers with entitlements for the whole type, across as many paginated -// cursors as it chooses to spawn (see SpawnCursors in +// cursors as it chooses to spawn (see EnqueuePageTokens in // annotation_type_scoped_grants.proto). // // This is the entitlements-phase analogue of TypeScopedGrants: the @@ -60,7 +60,7 @@ const ( // // ZERO-RESOURCE TYPES: the planner still enqueues the type-scoped action // when the store holds no resources of the type. The connector must -// return an empty page (or SpawnCursors that resolve empty), not an +// return an empty page (or EnqueuePageTokens that resolve empty), not an // error — an empty type is a valid outcome. // // TARGETED SYNC: SyncTargetedResource must not enqueue a per-resource diff --git a/pb/c1/connector/v2/annotation_type_scoped_grants.pb.go b/pb/c1/connector/v2/annotation_type_scoped_grants.pb.go index bbccc920d..07fc8ee76 100644 --- a/pb/c1/connector/v2/annotation_type_scoped_grants.pb.go +++ b/pb/c1/connector/v2/annotation_type_scoped_grants.pb.go @@ -39,7 +39,7 @@ const ( // validation requires a non-empty resource id and the stub identifies // the type, never a real resource). The connector answers with grants // for the whole type, across as many paginated cursors as it chooses to -// spawn (see SpawnCursors). +// spawn (see EnqueuePageTokens). // // This is the grants-phase analogue of StaticEntitlements: the connector // takes over enumeration for the type, and every downstream behavior @@ -58,7 +58,7 @@ const ( // // ZERO-RESOURCE TYPES: the planner still enqueues the type-scoped action // when the store holds no resources of the type. The connector must -// return an empty page (or SpawnCursors that resolve empty), not an error. +// return an empty page (or EnqueuePageTokens that resolve empty), not an error. // // TARGETED SYNC: SyncTargetedResource must not enqueue a per-resource // SyncGrantsOp for a type-scoped type. Type-scoped grant enumeration is @@ -110,7 +110,7 @@ func (b0 TypeScopedGrants_builder) Build() *TypeScopedGrants { return m0 } -// SpawnCursors is attached to a ListGrants or ListEntitlements response to +// EnqueuePageTokens is attached to a ListGrants or ListEntitlements response to // enqueue additional independent sibling cursors. Each token is delivered // back to the connector as the page token of its own action — scheduled by // the syncer's worker pool, rate-limited, and checkpointed like any other @@ -145,7 +145,7 @@ func (b0 TypeScopedGrants_builder) Build() *TypeScopedGrants { // connector needs to serve the cursor must be inside the token (or // re-derivable from it) — for per-resource spawns the resource identity // rides the action, so tokens only need the page coordinate. -type SpawnCursors struct { +type EnqueuePageTokens struct { state protoimpl.MessageState `protogen:"hybrid.v1"` // Page tokens for the sibling cursors to enqueue, one action each. // Capped per response (spawned actions persist in the checkpointed @@ -158,20 +158,20 @@ type SpawnCursors struct { sizeCache protoimpl.SizeCache } -func (x *SpawnCursors) Reset() { - *x = SpawnCursors{} +func (x *EnqueuePageTokens) Reset() { + *x = EnqueuePageTokens{} mi := &file_c1_connector_v2_annotation_type_scoped_grants_proto_msgTypes[1] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } -func (x *SpawnCursors) String() string { +func (x *EnqueuePageTokens) String() string { return protoimpl.X.MessageStringOf(x) } -func (*SpawnCursors) ProtoMessage() {} +func (*EnqueuePageTokens) ProtoMessage() {} -func (x *SpawnCursors) ProtoReflect() protoreflect.Message { +func (x *EnqueuePageTokens) ProtoReflect() protoreflect.Message { mi := &file_c1_connector_v2_annotation_type_scoped_grants_proto_msgTypes[1] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -183,29 +183,29 @@ func (x *SpawnCursors) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -func (x *SpawnCursors) GetPageTokens() []string { +func (x *EnqueuePageTokens) GetPageTokens() []string { if x != nil { return x.PageTokens } return nil } -func (x *SpawnCursors) GetEstimatedTotal() int64 { +func (x *EnqueuePageTokens) GetEstimatedTotal() int64 { if x != nil { return x.EstimatedTotal } return 0 } -func (x *SpawnCursors) SetPageTokens(v []string) { +func (x *EnqueuePageTokens) SetPageTokens(v []string) { x.PageTokens = v } -func (x *SpawnCursors) SetEstimatedTotal(v int64) { +func (x *EnqueuePageTokens) SetEstimatedTotal(v int64) { x.EstimatedTotal = v } -type SpawnCursors_builder struct { +type EnqueuePageTokens_builder struct { _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. // Page tokens for the sibling cursors to enqueue, one action each. @@ -217,8 +217,8 @@ type SpawnCursors_builder struct { EstimatedTotal int64 } -func (b0 SpawnCursors_builder) Build() *SpawnCursors { - m0 := &SpawnCursors{} +func (b0 EnqueuePageTokens_builder) Build() *EnqueuePageTokens { + m0 := &EnqueuePageTokens{} b, x := &b0, m0 _, _ = b, x x.PageTokens = b.PageTokens @@ -231,16 +231,16 @@ var File_c1_connector_v2_annotation_type_scoped_grants_proto protoreflect.FileDe const file_c1_connector_v2_annotation_type_scoped_grants_proto_rawDesc = "" + "\n" + "3c1/connector/v2/annotation_type_scoped_grants.proto\x12\x0fc1.connector.v2\x1a\x17validate/validate.proto\"\x12\n" + - "\x10TypeScopedGrants\"m\n" + - "\fSpawnCursors\x124\n" + + "\x10TypeScopedGrants\"r\n" + + "\x11EnqueuePageTokens\x124\n" + "\vpage_tokens\x18\x01 \x03(\tB\x13\xfaB\x10\x92\x01\r\x10\x80\b\"\br\x06\x10\x01\x18\x80\x80@R\n" + "pageTokens\x12'\n" + "\x0festimated_total\x18\x02 \x01(\x03R\x0eestimatedTotalB6Z4github.com/conductorone/baton-sdk/pb/c1/connector/v2b\x06proto3" var file_c1_connector_v2_annotation_type_scoped_grants_proto_msgTypes = make([]protoimpl.MessageInfo, 2) var file_c1_connector_v2_annotation_type_scoped_grants_proto_goTypes = []any{ - (*TypeScopedGrants)(nil), // 0: c1.connector.v2.TypeScopedGrants - (*SpawnCursors)(nil), // 1: c1.connector.v2.SpawnCursors + (*TypeScopedGrants)(nil), // 0: c1.connector.v2.TypeScopedGrants + (*EnqueuePageTokens)(nil), // 1: c1.connector.v2.EnqueuePageTokens } var file_c1_connector_v2_annotation_type_scoped_grants_proto_depIdxs = []int32{ 0, // [0:0] is the sub-list for method output_type diff --git a/pb/c1/connector/v2/annotation_type_scoped_grants.pb.validate.go b/pb/c1/connector/v2/annotation_type_scoped_grants.pb.validate.go index 49a808027..ee529553a 100644 --- a/pb/c1/connector/v2/annotation_type_scoped_grants.pb.validate.go +++ b/pb/c1/connector/v2/annotation_type_scoped_grants.pb.validate.go @@ -135,22 +135,22 @@ var _ interface { ErrorName() string } = TypeScopedGrantsValidationError{} -// Validate checks the field values on SpawnCursors with the rules defined in -// the proto definition for this message. If any rules are violated, the first -// error encountered is returned, or nil if there are no violations. -func (m *SpawnCursors) Validate() error { +// Validate checks the field values on EnqueuePageTokens with the rules defined +// in the proto definition for this message. If any rules are violated, the +// first error encountered is returned, or nil if there are no violations. +func (m *EnqueuePageTokens) Validate() error { return m.validate(false) } -// ValidateAll checks the field values on SpawnCursors with the rules defined -// in the proto definition for this message. If any rules are violated, the -// result is a list of violation errors wrapped in SpawnCursorsMultiError, or -// nil if none found. -func (m *SpawnCursors) ValidateAll() error { +// ValidateAll checks the field values on EnqueuePageTokens with the rules +// defined in the proto definition for this message. If any rules are +// violated, the result is a list of violation errors wrapped in +// EnqueuePageTokensMultiError, or nil if none found. +func (m *EnqueuePageTokens) ValidateAll() error { return m.validate(true) } -func (m *SpawnCursors) validate(all bool) error { +func (m *EnqueuePageTokens) validate(all bool) error { if m == nil { return nil } @@ -158,7 +158,7 @@ func (m *SpawnCursors) validate(all bool) error { var errors []error if len(m.GetPageTokens()) > 1024 { - err := SpawnCursorsValidationError{ + err := EnqueuePageTokensValidationError{ field: "PageTokens", reason: "value must contain no more than 1024 item(s)", } @@ -172,7 +172,7 @@ func (m *SpawnCursors) validate(all bool) error { _, _ = idx, item if l := utf8.RuneCountInString(item); l < 1 || l > 1048576 { - err := SpawnCursorsValidationError{ + err := EnqueuePageTokensValidationError{ field: fmt.Sprintf("PageTokens[%v]", idx), reason: "value length must be between 1 and 1048576 runes, inclusive", } @@ -187,18 +187,19 @@ func (m *SpawnCursors) validate(all bool) error { // no validation rules for EstimatedTotal if len(errors) > 0 { - return SpawnCursorsMultiError(errors) + return EnqueuePageTokensMultiError(errors) } return nil } -// SpawnCursorsMultiError is an error wrapping multiple validation errors -// returned by SpawnCursors.ValidateAll() if the designated constraints aren't met. -type SpawnCursorsMultiError []error +// EnqueuePageTokensMultiError is an error wrapping multiple validation errors +// returned by EnqueuePageTokens.ValidateAll() if the designated constraints +// aren't met. +type EnqueuePageTokensMultiError []error // Error returns a concatenation of all the error messages it wraps. -func (m SpawnCursorsMultiError) Error() string { +func (m EnqueuePageTokensMultiError) Error() string { msgs := make([]string, 0, len(m)) for _, err := range m { msgs = append(msgs, err.Error()) @@ -207,11 +208,11 @@ func (m SpawnCursorsMultiError) Error() string { } // AllErrors returns a list of validation violation errors. -func (m SpawnCursorsMultiError) AllErrors() []error { return m } +func (m EnqueuePageTokensMultiError) AllErrors() []error { return m } -// SpawnCursorsValidationError is the validation error returned by -// SpawnCursors.Validate if the designated constraints aren't met. -type SpawnCursorsValidationError struct { +// EnqueuePageTokensValidationError is the validation error returned by +// EnqueuePageTokens.Validate if the designated constraints aren't met. +type EnqueuePageTokensValidationError struct { field string reason string cause error @@ -219,22 +220,24 @@ type SpawnCursorsValidationError struct { } // Field function returns field value. -func (e SpawnCursorsValidationError) Field() string { return e.field } +func (e EnqueuePageTokensValidationError) Field() string { return e.field } // Reason function returns reason value. -func (e SpawnCursorsValidationError) Reason() string { return e.reason } +func (e EnqueuePageTokensValidationError) Reason() string { return e.reason } // Cause function returns cause value. -func (e SpawnCursorsValidationError) Cause() error { return e.cause } +func (e EnqueuePageTokensValidationError) Cause() error { return e.cause } // Key function returns key value. -func (e SpawnCursorsValidationError) Key() bool { return e.key } +func (e EnqueuePageTokensValidationError) Key() bool { return e.key } // ErrorName returns error name. -func (e SpawnCursorsValidationError) ErrorName() string { return "SpawnCursorsValidationError" } +func (e EnqueuePageTokensValidationError) ErrorName() string { + return "EnqueuePageTokensValidationError" +} // Error satisfies the builtin error interface -func (e SpawnCursorsValidationError) Error() string { +func (e EnqueuePageTokensValidationError) Error() string { cause := "" if e.cause != nil { cause = fmt.Sprintf(" | caused by: %v", e.cause) @@ -246,14 +249,14 @@ func (e SpawnCursorsValidationError) Error() string { } return fmt.Sprintf( - "invalid %sSpawnCursors.%s: %s%s", + "invalid %sEnqueuePageTokens.%s: %s%s", key, e.field, e.reason, cause) } -var _ error = SpawnCursorsValidationError{} +var _ error = EnqueuePageTokensValidationError{} var _ interface { Field() string @@ -261,4 +264,4 @@ var _ interface { Key() bool Cause() error ErrorName() string -} = SpawnCursorsValidationError{} +} = EnqueuePageTokensValidationError{} diff --git a/pb/c1/connector/v2/annotation_type_scoped_grants_protoopaque.pb.go b/pb/c1/connector/v2/annotation_type_scoped_grants_protoopaque.pb.go index 7df6da07d..00baef3c4 100644 --- a/pb/c1/connector/v2/annotation_type_scoped_grants_protoopaque.pb.go +++ b/pb/c1/connector/v2/annotation_type_scoped_grants_protoopaque.pb.go @@ -39,7 +39,7 @@ const ( // validation requires a non-empty resource id and the stub identifies // the type, never a real resource). The connector answers with grants // for the whole type, across as many paginated cursors as it chooses to -// spawn (see SpawnCursors). +// spawn (see EnqueuePageTokens). // // This is the grants-phase analogue of StaticEntitlements: the connector // takes over enumeration for the type, and every downstream behavior @@ -58,7 +58,7 @@ const ( // // ZERO-RESOURCE TYPES: the planner still enqueues the type-scoped action // when the store holds no resources of the type. The connector must -// return an empty page (or SpawnCursors that resolve empty), not an error. +// return an empty page (or EnqueuePageTokens that resolve empty), not an error. // // TARGETED SYNC: SyncTargetedResource must not enqueue a per-resource // SyncGrantsOp for a type-scoped type. Type-scoped grant enumeration is @@ -110,7 +110,7 @@ func (b0 TypeScopedGrants_builder) Build() *TypeScopedGrants { return m0 } -// SpawnCursors is attached to a ListGrants or ListEntitlements response to +// EnqueuePageTokens is attached to a ListGrants or ListEntitlements response to // enqueue additional independent sibling cursors. Each token is delivered // back to the connector as the page token of its own action — scheduled by // the syncer's worker pool, rate-limited, and checkpointed like any other @@ -145,7 +145,7 @@ func (b0 TypeScopedGrants_builder) Build() *TypeScopedGrants { // connector needs to serve the cursor must be inside the token (or // re-derivable from it) — for per-resource spawns the resource identity // rides the action, so tokens only need the page coordinate. -type SpawnCursors struct { +type EnqueuePageTokens struct { state protoimpl.MessageState `protogen:"opaque.v1"` xxx_hidden_PageTokens []string `protobuf:"bytes,1,rep,name=page_tokens,json=pageTokens,proto3"` xxx_hidden_EstimatedTotal int64 `protobuf:"varint,2,opt,name=estimated_total,json=estimatedTotal,proto3"` @@ -153,20 +153,20 @@ type SpawnCursors struct { sizeCache protoimpl.SizeCache } -func (x *SpawnCursors) Reset() { - *x = SpawnCursors{} +func (x *EnqueuePageTokens) Reset() { + *x = EnqueuePageTokens{} mi := &file_c1_connector_v2_annotation_type_scoped_grants_proto_msgTypes[1] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } -func (x *SpawnCursors) String() string { +func (x *EnqueuePageTokens) String() string { return protoimpl.X.MessageStringOf(x) } -func (*SpawnCursors) ProtoMessage() {} +func (*EnqueuePageTokens) ProtoMessage() {} -func (x *SpawnCursors) ProtoReflect() protoreflect.Message { +func (x *EnqueuePageTokens) ProtoReflect() protoreflect.Message { mi := &file_c1_connector_v2_annotation_type_scoped_grants_proto_msgTypes[1] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) @@ -178,29 +178,29 @@ func (x *SpawnCursors) ProtoReflect() protoreflect.Message { return mi.MessageOf(x) } -func (x *SpawnCursors) GetPageTokens() []string { +func (x *EnqueuePageTokens) GetPageTokens() []string { if x != nil { return x.xxx_hidden_PageTokens } return nil } -func (x *SpawnCursors) GetEstimatedTotal() int64 { +func (x *EnqueuePageTokens) GetEstimatedTotal() int64 { if x != nil { return x.xxx_hidden_EstimatedTotal } return 0 } -func (x *SpawnCursors) SetPageTokens(v []string) { +func (x *EnqueuePageTokens) SetPageTokens(v []string) { x.xxx_hidden_PageTokens = v } -func (x *SpawnCursors) SetEstimatedTotal(v int64) { +func (x *EnqueuePageTokens) SetEstimatedTotal(v int64) { x.xxx_hidden_EstimatedTotal = v } -type SpawnCursors_builder struct { +type EnqueuePageTokens_builder struct { _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. // Page tokens for the sibling cursors to enqueue, one action each. @@ -212,8 +212,8 @@ type SpawnCursors_builder struct { EstimatedTotal int64 } -func (b0 SpawnCursors_builder) Build() *SpawnCursors { - m0 := &SpawnCursors{} +func (b0 EnqueuePageTokens_builder) Build() *EnqueuePageTokens { + m0 := &EnqueuePageTokens{} b, x := &b0, m0 _, _ = b, x x.xxx_hidden_PageTokens = b.PageTokens @@ -226,16 +226,16 @@ var File_c1_connector_v2_annotation_type_scoped_grants_proto protoreflect.FileDe const file_c1_connector_v2_annotation_type_scoped_grants_proto_rawDesc = "" + "\n" + "3c1/connector/v2/annotation_type_scoped_grants.proto\x12\x0fc1.connector.v2\x1a\x17validate/validate.proto\"\x12\n" + - "\x10TypeScopedGrants\"m\n" + - "\fSpawnCursors\x124\n" + + "\x10TypeScopedGrants\"r\n" + + "\x11EnqueuePageTokens\x124\n" + "\vpage_tokens\x18\x01 \x03(\tB\x13\xfaB\x10\x92\x01\r\x10\x80\b\"\br\x06\x10\x01\x18\x80\x80@R\n" + "pageTokens\x12'\n" + "\x0festimated_total\x18\x02 \x01(\x03R\x0eestimatedTotalB6Z4github.com/conductorone/baton-sdk/pb/c1/connector/v2b\x06proto3" var file_c1_connector_v2_annotation_type_scoped_grants_proto_msgTypes = make([]protoimpl.MessageInfo, 2) var file_c1_connector_v2_annotation_type_scoped_grants_proto_goTypes = []any{ - (*TypeScopedGrants)(nil), // 0: c1.connector.v2.TypeScopedGrants - (*SpawnCursors)(nil), // 1: c1.connector.v2.SpawnCursors + (*TypeScopedGrants)(nil), // 0: c1.connector.v2.TypeScopedGrants + (*EnqueuePageTokens)(nil), // 1: c1.connector.v2.EnqueuePageTokens } var file_c1_connector_v2_annotation_type_scoped_grants_proto_depIdxs = []int32{ 0, // [0:0] is the sub-list for method output_type diff --git a/pb/c1/connectorapi/baton/v1/source_cache.pb.go b/pb/c1/connectorapi/baton/v1/source_cache.pb.go index 5d0c2090c..ef50c399a 100644 --- a/pb/c1/connectorapi/baton/v1/source_cache.pb.go +++ b/pb/c1/connectorapi/baton/v1/source_cache.pb.go @@ -29,11 +29,11 @@ type LookupRequest struct { // (pkg/sourcecache.RowKind values). Entries are partitioned by row // kind, so one scope hash can carry a different validator per kind. RowKind string `protobuf:"bytes,1,opt,name=row_kind,json=rowKind,proto3" json:"row_kind,omitempty"` - // Connector-defined stable scope identifier (conventionally a hex hash - // of the canonical scope; see pkg/sourcecache.HashScope). Opaque to the + // Connector-defined stable scope identifier (a stable identifier for the canonical scope + // (often a hex hash; see pkg/sourcecache.HashScope). Opaque to the // parent; matched verbatim against the previous sync's source-cache // entries. - ScopeHash string `protobuf:"bytes,2,opt,name=scope_hash,json=scopeHash,proto3" json:"scope_hash,omitempty"` + ScopeKey string `protobuf:"bytes,2,opt,name=scope_key,json=scopeKey,proto3" json:"scope_key,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -70,9 +70,9 @@ func (x *LookupRequest) GetRowKind() string { return "" } -func (x *LookupRequest) GetScopeHash() string { +func (x *LookupRequest) GetScopeKey() string { if x != nil { - return x.ScopeHash + return x.ScopeKey } return "" } @@ -81,8 +81,8 @@ func (x *LookupRequest) SetRowKind(v string) { x.RowKind = v } -func (x *LookupRequest) SetScopeHash(v string) { - x.ScopeHash = v +func (x *LookupRequest) SetScopeKey(v string) { + x.ScopeKey = v } type LookupRequest_builder struct { @@ -92,11 +92,11 @@ type LookupRequest_builder struct { // (pkg/sourcecache.RowKind values). Entries are partitioned by row // kind, so one scope hash can carry a different validator per kind. RowKind string - // Connector-defined stable scope identifier (conventionally a hex hash - // of the canonical scope; see pkg/sourcecache.HashScope). Opaque to the + // Connector-defined stable scope identifier (a stable identifier for the canonical scope + // (often a hex hash; see pkg/sourcecache.HashScope). Opaque to the // parent; matched verbatim against the previous sync's source-cache // entries. - ScopeHash string + ScopeKey string } func (b0 LookupRequest_builder) Build() *LookupRequest { @@ -104,13 +104,13 @@ func (b0 LookupRequest_builder) Build() *LookupRequest { b, x := &b0, m0 _, _ = b, x x.RowKind = b.RowKind - x.ScopeHash = b.ScopeHash + x.ScopeKey = b.ScopeKey return m0 } type LookupResponse struct { state protoimpl.MessageState `protogen:"hybrid.v1"` - // False means no prior entry exists for (row_kind, scope_hash): the + // False means no prior entry exists for (row_kind, scope_key): the // connector must fetch fresh and must not emit SourceCacheReplay for // this scope. Found bool `protobuf:"varint,1,opt,name=found,proto3" json:"found,omitempty"` @@ -118,9 +118,9 @@ type LookupResponse struct { // ETag, delta token, ...). Empty when found is false. The cap is a // sanity bound sized for Microsoft Graph delta tokens, which are known // to run to thousands of characters; storage imposes no limit. - Etag string `protobuf:"bytes,2,opt,name=etag,proto3" json:"etag,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + CacheValidator string `protobuf:"bytes,2,opt,name=cache_validator,json=cacheValidator,proto3" json:"cache_validator,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *LookupResponse) Reset() { @@ -155,9 +155,9 @@ func (x *LookupResponse) GetFound() bool { return false } -func (x *LookupResponse) GetEtag() string { +func (x *LookupResponse) GetCacheValidator() string { if x != nil { - return x.Etag + return x.CacheValidator } return "" } @@ -166,14 +166,14 @@ func (x *LookupResponse) SetFound(v bool) { x.Found = v } -func (x *LookupResponse) SetEtag(v string) { - x.Etag = v +func (x *LookupResponse) SetCacheValidator(v string) { + x.CacheValidator = v } type LookupResponse_builder struct { _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. - // False means no prior entry exists for (row_kind, scope_hash): the + // False means no prior entry exists for (row_kind, scope_key): the // connector must fetch fresh and must not emit SourceCacheReplay for // this scope. Found bool @@ -181,7 +181,7 @@ type LookupResponse_builder struct { // ETag, delta token, ...). Empty when found is false. The cap is a // sanity bound sized for Microsoft Graph delta tokens, which are known // to run to thousands of characters; storage imposes no limit. - Etag string + CacheValidator string } func (b0 LookupResponse_builder) Build() *LookupResponse { @@ -189,7 +189,7 @@ func (b0 LookupResponse_builder) Build() *LookupResponse { b, x := &b0, m0 _, _ = b, x x.Found = b.Found - x.Etag = b.Etag + x.CacheValidator = b.CacheValidator return m0 } @@ -197,15 +197,14 @@ var File_c1_connectorapi_baton_v1_source_cache_proto protoreflect.FileDescriptor const file_c1_connectorapi_baton_v1_source_cache_proto_rawDesc = "" + "\n" + - "+c1/connectorapi/baton/v1/source_cache.proto\x12\x18c1.connectorapi.baton.v1\x1a\x17validate/validate.proto\"`\n" + + "+c1/connectorapi/baton/v1/source_cache.proto\x12\x18c1.connectorapi.baton.v1\x1a\x17validate/validate.proto\"^\n" + "\rLookupRequest\x12$\n" + - "\brow_kind\x18\x01 \x01(\tB\t\xfaB\x06r\x04\x10\x01\x18@R\arowKind\x12)\n" + - "\n" + - "scope_hash\x18\x02 \x01(\tB\n" + - "\xfaB\ar\x05\x10\x01\x18\x80\x02R\tscopeHash\"E\n" + + "\brow_kind\x18\x01 \x01(\tB\t\xfaB\x06r\x04\x10\x01\x18@R\arowKind\x12'\n" + + "\tscope_key\x18\x02 \x01(\tB\n" + + "\xfaB\ar\x05\x10\x01\x18\x80\x02R\bscopeKey\"Z\n" + "\x0eLookupResponse\x12\x14\n" + - "\x05found\x18\x01 \x01(\bR\x05found\x12\x1d\n" + - "\x04etag\x18\x02 \x01(\tB\t\xfaB\x06r\x04\x18\x80\x80\x04R\x04etag2x\n" + + "\x05found\x18\x01 \x01(\bR\x05found\x122\n" + + "\x0fcache_validator\x18\x02 \x01(\tB\t\xfaB\x06r\x04\x18\x80\x80\x04R\x0ecacheValidator2x\n" + "\x17BatonSourceCacheService\x12]\n" + "\x06Lookup\x12'.c1.connectorapi.baton.v1.LookupRequest\x1a(.c1.connectorapi.baton.v1.LookupResponse\"\x00B7Z5gitlab.com/ductone/c1/pkg/pb/c1/connectorapi/baton/v1b\x06proto3" diff --git a/pb/c1/connectorapi/baton/v1/source_cache.pb.validate.go b/pb/c1/connectorapi/baton/v1/source_cache.pb.validate.go index dfc7cca7c..55912ddab 100644 --- a/pb/c1/connectorapi/baton/v1/source_cache.pb.validate.go +++ b/pb/c1/connectorapi/baton/v1/source_cache.pb.validate.go @@ -68,9 +68,9 @@ func (m *LookupRequest) validate(all bool) error { errors = append(errors, err) } - if l := utf8.RuneCountInString(m.GetScopeHash()); l < 1 || l > 256 { + if l := utf8.RuneCountInString(m.GetScopeKey()); l < 1 || l > 256 { err := LookupRequestValidationError{ - field: "ScopeHash", + field: "ScopeKey", reason: "value length must be between 1 and 256 runes, inclusive", } if !all { @@ -181,9 +181,9 @@ func (m *LookupResponse) validate(all bool) error { // no validation rules for Found - if utf8.RuneCountInString(m.GetEtag()) > 65536 { + if utf8.RuneCountInString(m.GetCacheValidator()) > 65536 { err := LookupResponseValidationError{ - field: "Etag", + field: "CacheValidator", reason: "value length must be at most 65536 runes", } if !all { diff --git a/pb/c1/connectorapi/baton/v1/source_cache_grpc.pb.go b/pb/c1/connectorapi/baton/v1/source_cache_grpc.pb.go index 8db25cd34..6c5bd6a71 100644 --- a/pb/c1/connectorapi/baton/v1/source_cache_grpc.pb.go +++ b/pb/c1/connectorapi/baton/v1/source_cache_grpc.pb.go @@ -27,7 +27,7 @@ const ( // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. // // BatonSourceCacheService is the dedicated parent-side RPC the connector -// subprocess uses to ask "do I have a previous validator (etag / delta +// subprocess uses to ask "do I have a previous validator (HTTP ETag / delta // token) for this scope?" before revalidating upstream. // // This is intentionally NOT routed through the session-store gRPC service: @@ -65,7 +65,7 @@ func (c *batonSourceCacheServiceClient) Lookup(ctx context.Context, in *LookupRe // for forward compatibility. // // BatonSourceCacheService is the dedicated parent-side RPC the connector -// subprocess uses to ask "do I have a previous validator (etag / delta +// subprocess uses to ask "do I have a previous validator (HTTP ETag / delta // token) for this scope?" before revalidating upstream. // // This is intentionally NOT routed through the session-store gRPC service: diff --git a/pb/c1/connectorapi/baton/v1/source_cache_protoopaque.pb.go b/pb/c1/connectorapi/baton/v1/source_cache_protoopaque.pb.go index 9bfc8953c..50e612196 100644 --- a/pb/c1/connectorapi/baton/v1/source_cache_protoopaque.pb.go +++ b/pb/c1/connectorapi/baton/v1/source_cache_protoopaque.pb.go @@ -24,11 +24,11 @@ const ( ) type LookupRequest struct { - state protoimpl.MessageState `protogen:"opaque.v1"` - xxx_hidden_RowKind string `protobuf:"bytes,1,opt,name=row_kind,json=rowKind,proto3"` - xxx_hidden_ScopeHash string `protobuf:"bytes,2,opt,name=scope_hash,json=scopeHash,proto3"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"opaque.v1"` + xxx_hidden_RowKind string `protobuf:"bytes,1,opt,name=row_kind,json=rowKind,proto3"` + xxx_hidden_ScopeKey string `protobuf:"bytes,2,opt,name=scope_key,json=scopeKey,proto3"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *LookupRequest) Reset() { @@ -63,9 +63,9 @@ func (x *LookupRequest) GetRowKind() string { return "" } -func (x *LookupRequest) GetScopeHash() string { +func (x *LookupRequest) GetScopeKey() string { if x != nil { - return x.xxx_hidden_ScopeHash + return x.xxx_hidden_ScopeKey } return "" } @@ -74,8 +74,8 @@ func (x *LookupRequest) SetRowKind(v string) { x.xxx_hidden_RowKind = v } -func (x *LookupRequest) SetScopeHash(v string) { - x.xxx_hidden_ScopeHash = v +func (x *LookupRequest) SetScopeKey(v string) { + x.xxx_hidden_ScopeKey = v } type LookupRequest_builder struct { @@ -85,11 +85,11 @@ type LookupRequest_builder struct { // (pkg/sourcecache.RowKind values). Entries are partitioned by row // kind, so one scope hash can carry a different validator per kind. RowKind string - // Connector-defined stable scope identifier (conventionally a hex hash - // of the canonical scope; see pkg/sourcecache.HashScope). Opaque to the + // Connector-defined stable scope identifier (a stable identifier for the canonical scope + // (often a hex hash; see pkg/sourcecache.HashScope). Opaque to the // parent; matched verbatim against the previous sync's source-cache // entries. - ScopeHash string + ScopeKey string } func (b0 LookupRequest_builder) Build() *LookupRequest { @@ -97,16 +97,16 @@ func (b0 LookupRequest_builder) Build() *LookupRequest { b, x := &b0, m0 _, _ = b, x x.xxx_hidden_RowKind = b.RowKind - x.xxx_hidden_ScopeHash = b.ScopeHash + x.xxx_hidden_ScopeKey = b.ScopeKey return m0 } type LookupResponse struct { - state protoimpl.MessageState `protogen:"opaque.v1"` - xxx_hidden_Found bool `protobuf:"varint,1,opt,name=found,proto3"` - xxx_hidden_Etag string `protobuf:"bytes,2,opt,name=etag,proto3"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"opaque.v1"` + xxx_hidden_Found bool `protobuf:"varint,1,opt,name=found,proto3"` + xxx_hidden_CacheValidator string `protobuf:"bytes,2,opt,name=cache_validator,json=cacheValidator,proto3"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *LookupResponse) Reset() { @@ -141,9 +141,9 @@ func (x *LookupResponse) GetFound() bool { return false } -func (x *LookupResponse) GetEtag() string { +func (x *LookupResponse) GetCacheValidator() string { if x != nil { - return x.xxx_hidden_Etag + return x.xxx_hidden_CacheValidator } return "" } @@ -152,14 +152,14 @@ func (x *LookupResponse) SetFound(v bool) { x.xxx_hidden_Found = v } -func (x *LookupResponse) SetEtag(v string) { - x.xxx_hidden_Etag = v +func (x *LookupResponse) SetCacheValidator(v string) { + x.xxx_hidden_CacheValidator = v } type LookupResponse_builder struct { _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. - // False means no prior entry exists for (row_kind, scope_hash): the + // False means no prior entry exists for (row_kind, scope_key): the // connector must fetch fresh and must not emit SourceCacheReplay for // this scope. Found bool @@ -167,7 +167,7 @@ type LookupResponse_builder struct { // ETag, delta token, ...). Empty when found is false. The cap is a // sanity bound sized for Microsoft Graph delta tokens, which are known // to run to thousands of characters; storage imposes no limit. - Etag string + CacheValidator string } func (b0 LookupResponse_builder) Build() *LookupResponse { @@ -175,7 +175,7 @@ func (b0 LookupResponse_builder) Build() *LookupResponse { b, x := &b0, m0 _, _ = b, x x.xxx_hidden_Found = b.Found - x.xxx_hidden_Etag = b.Etag + x.xxx_hidden_CacheValidator = b.CacheValidator return m0 } @@ -183,15 +183,14 @@ var File_c1_connectorapi_baton_v1_source_cache_proto protoreflect.FileDescriptor const file_c1_connectorapi_baton_v1_source_cache_proto_rawDesc = "" + "\n" + - "+c1/connectorapi/baton/v1/source_cache.proto\x12\x18c1.connectorapi.baton.v1\x1a\x17validate/validate.proto\"`\n" + + "+c1/connectorapi/baton/v1/source_cache.proto\x12\x18c1.connectorapi.baton.v1\x1a\x17validate/validate.proto\"^\n" + "\rLookupRequest\x12$\n" + - "\brow_kind\x18\x01 \x01(\tB\t\xfaB\x06r\x04\x10\x01\x18@R\arowKind\x12)\n" + - "\n" + - "scope_hash\x18\x02 \x01(\tB\n" + - "\xfaB\ar\x05\x10\x01\x18\x80\x02R\tscopeHash\"E\n" + + "\brow_kind\x18\x01 \x01(\tB\t\xfaB\x06r\x04\x10\x01\x18@R\arowKind\x12'\n" + + "\tscope_key\x18\x02 \x01(\tB\n" + + "\xfaB\ar\x05\x10\x01\x18\x80\x02R\bscopeKey\"Z\n" + "\x0eLookupResponse\x12\x14\n" + - "\x05found\x18\x01 \x01(\bR\x05found\x12\x1d\n" + - "\x04etag\x18\x02 \x01(\tB\t\xfaB\x06r\x04\x18\x80\x80\x04R\x04etag2x\n" + + "\x05found\x18\x01 \x01(\bR\x05found\x122\n" + + "\x0fcache_validator\x18\x02 \x01(\tB\t\xfaB\x06r\x04\x18\x80\x80\x04R\x0ecacheValidator2x\n" + "\x17BatonSourceCacheService\x12]\n" + "\x06Lookup\x12'.c1.connectorapi.baton.v1.LookupRequest\x1a(.c1.connectorapi.baton.v1.LookupResponse\"\x00B7Z5gitlab.com/ductone/c1/pkg/pb/c1/connectorapi/baton/v1b\x06proto3" diff --git a/pb/c1/storage/v3/records.pb.go b/pb/c1/storage/v3/records.pb.go index e02826a96..e54fd02c2 100644 --- a/pb/c1/storage/v3/records.pb.go +++ b/pb/c1/storage/v3/records.pb.go @@ -457,9 +457,9 @@ type ResourceRecord struct { DiscoveredAt *timestamppb.Timestamp `protobuf:"bytes,8,opt,name=discovered_at,json=discoveredAt,proto3" json:"discovered_at,omitempty"` // Source-cache scope stamp (see annotation_source_cache.proto). // Empty for rows not produced under a source-cache scope. - SourceScopeHash string `protobuf:"bytes,9,opt,name=source_scope_hash,json=sourceScopeHash,proto3" json:"source_scope_hash,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + SourceScopeKey string `protobuf:"bytes,9,opt,name=source_scope_key,json=sourceScopeKey,proto3" json:"source_scope_key,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *ResourceRecord) Reset() { @@ -536,9 +536,9 @@ func (x *ResourceRecord) GetDiscoveredAt() *timestamppb.Timestamp { return nil } -func (x *ResourceRecord) GetSourceScopeHash() string { +func (x *ResourceRecord) GetSourceScopeKey() string { if x != nil { - return x.SourceScopeHash + return x.SourceScopeKey } return "" } @@ -571,8 +571,8 @@ func (x *ResourceRecord) SetDiscoveredAt(v *timestamppb.Timestamp) { x.DiscoveredAt = v } -func (x *ResourceRecord) SetSourceScopeHash(v string) { - x.SourceScopeHash = v +func (x *ResourceRecord) SetSourceScopeKey(v string) { + x.SourceScopeKey = v } func (x *ResourceRecord) HasParent() bool { @@ -609,7 +609,7 @@ type ResourceRecord_builder struct { DiscoveredAt *timestamppb.Timestamp // Source-cache scope stamp (see annotation_source_cache.proto). // Empty for rows not produced under a source-cache scope. - SourceScopeHash string + SourceScopeKey string } func (b0 ResourceRecord_builder) Build() *ResourceRecord { @@ -623,7 +623,7 @@ func (b0 ResourceRecord_builder) Build() *ResourceRecord { x.Parent = b.Parent x.Annotations = b.Annotations x.DiscoveredAt = b.DiscoveredAt - x.SourceScopeHash = b.SourceScopeHash + x.SourceScopeKey = b.SourceScopeKey return m0 } @@ -652,9 +652,9 @@ type EntitlementRecord struct { GrantableToResourceTypeIds []string `protobuf:"bytes,10,rep,name=grantable_to_resource_type_ids,json=grantableToResourceTypeIds,proto3" json:"grantable_to_resource_type_ids,omitempty"` // Source-cache scope stamp (see annotation_source_cache.proto). // Empty for rows not produced under a source-cache scope. - SourceScopeHash string `protobuf:"bytes,11,opt,name=source_scope_hash,json=sourceScopeHash,proto3" json:"source_scope_hash,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + SourceScopeKey string `protobuf:"bytes,11,opt,name=source_scope_key,json=sourceScopeKey,proto3" json:"source_scope_key,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *EntitlementRecord) Reset() { @@ -745,9 +745,9 @@ func (x *EntitlementRecord) GetGrantableToResourceTypeIds() []string { return nil } -func (x *EntitlementRecord) GetSourceScopeHash() string { +func (x *EntitlementRecord) GetSourceScopeKey() string { if x != nil { - return x.SourceScopeHash + return x.SourceScopeKey } return "" } @@ -788,8 +788,8 @@ func (x *EntitlementRecord) SetGrantableToResourceTypeIds(v []string) { x.GrantableToResourceTypeIds = v } -func (x *EntitlementRecord) SetSourceScopeHash(v string) { - x.SourceScopeHash = v +func (x *EntitlementRecord) SetSourceScopeKey(v string) { + x.SourceScopeKey = v } func (x *EntitlementRecord) HasResource() bool { @@ -840,7 +840,7 @@ type EntitlementRecord_builder struct { GrantableToResourceTypeIds []string // Source-cache scope stamp (see annotation_source_cache.proto). // Empty for rows not produced under a source-cache scope. - SourceScopeHash string + SourceScopeKey string } func (b0 EntitlementRecord_builder) Build() *EntitlementRecord { @@ -856,7 +856,7 @@ func (b0 EntitlementRecord_builder) Build() *EntitlementRecord { x.DiscoveredAt = b.DiscoveredAt x.Slug = b.Slug x.GrantableToResourceTypeIds = b.GrantableToResourceTypeIds - x.SourceScopeHash = b.SourceScopeHash + x.SourceScopeKey = b.SourceScopeKey return m0 } @@ -888,9 +888,9 @@ type GrantRecord struct { // expander-derived grants, which are recreated by expansion each sync // and never replayed. StoreExpandedGrants preserves this field on // rewrites of existing rows, exactly like expansion/needs_expansion. - SourceScopeHash string `protobuf:"bytes,10,opt,name=source_scope_hash,json=sourceScopeHash,proto3" json:"source_scope_hash,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + SourceScopeKey string `protobuf:"bytes,10,opt,name=source_scope_key,json=sourceScopeKey,proto3" json:"source_scope_key,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *GrantRecord) Reset() { @@ -974,9 +974,9 @@ func (x *GrantRecord) GetSources() map[string]*GrantSourceRecord { return nil } -func (x *GrantRecord) GetSourceScopeHash() string { +func (x *GrantRecord) GetSourceScopeKey() string { if x != nil { - return x.SourceScopeHash + return x.SourceScopeKey } return "" } @@ -1013,8 +1013,8 @@ func (x *GrantRecord) SetSources(v map[string]*GrantSourceRecord) { x.Sources = v } -func (x *GrantRecord) SetSourceScopeHash(v string) { - x.SourceScopeHash = v +func (x *GrantRecord) SetSourceScopeKey(v string) { + x.SourceScopeKey = v } func (x *GrantRecord) HasEntitlement() bool { @@ -1090,7 +1090,7 @@ type GrantRecord_builder struct { // expander-derived grants, which are recreated by expansion each sync // and never replayed. StoreExpandedGrants preserves this field on // rewrites of existing rows, exactly like expansion/needs_expansion. - SourceScopeHash string + SourceScopeKey string } func (b0 GrantRecord_builder) Build() *GrantRecord { @@ -1105,7 +1105,7 @@ func (b0 GrantRecord_builder) Build() *GrantRecord { x.NeedsExpansion = b.NeedsExpansion x.Annotations = b.Annotations x.Sources = b.Sources - x.SourceScopeHash = b.SourceScopeHash + x.SourceScopeKey = b.SourceScopeKey return m0 } @@ -1249,7 +1249,7 @@ type SyncRunRecord struct { // compacted marks a sync produced by compaction (fold or rebuild) // rather than by a real connector run. Compacted artifacts are // keep-newer UPSERT merges — base rows a newer input deleted survive — - // so no input sync's upstream validators (source-cache etags) describe + // so no input sync's upstream validators (source-cache validators) describe // their contents. // // "Can this sync be used as a source-cache replay source?" is the @@ -1424,7 +1424,7 @@ type SyncRunRecord_builder struct { // compacted marks a sync produced by compaction (fold or rebuild) // rather than by a real connector run. Compacted artifacts are // keep-newer UPSERT merges — base rows a newer input deleted survive — - // so no input sync's upstream validators (source-cache etags) describe + // so no input sync's upstream validators (source-cache validators) describe // their contents. // // "Can this sync be used as a source-cache replay source?" is the @@ -1755,7 +1755,7 @@ func (b0 SessionRecord_builder) Build() *SessionRecord { // SourceCacheEntryRecord is the per-scope manifest entry for source-cache // replay (see c1/connector/v2/annotation_source_cache.proto). One entry -// per (row_kind, scope_hash), written for every freshly fetched scope — +// per (row_kind, scope_key), written for every freshly fetched scope — // including zero-row responses — and rewritten on replay with the scope's // current validator. The previous sync's entries (read from the previous // c1z) are the lookup surface connectors revalidate against. @@ -1764,13 +1764,14 @@ type SourceCacheEntryRecord struct { // Row kind partition: "resources", "entitlements", or "grants" // (pkg/sourcecache.RowKind values). RowKind string `protobuf:"bytes,1,opt,name=row_kind,json=rowKind,proto3" json:"row_kind,omitempty"` - // Connector-computed scope hash (lowercase hex). - ScopeHash string `protobuf:"bytes,2,opt,name=scope_hash,json=scopeHash,proto3" json:"scope_hash,omitempty"` + // Connector-computed stable scope identifier (conventionally lowercase + // hex when hashed via sourcecache.HashScope; any byte-stable string). + ScopeKey string `protobuf:"bytes,2,opt,name=scope_key,json=scopeKey,proto3" json:"scope_key,omitempty"` // Opaque upstream validator: HTTP ETag, delta token, etc. - Etag string `protobuf:"bytes,3,opt,name=etag,proto3" json:"etag,omitempty"` - DiscoveredAt *timestamppb.Timestamp `protobuf:"bytes,4,opt,name=discovered_at,json=discoveredAt,proto3" json:"discovered_at,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + CacheValidator string `protobuf:"bytes,3,opt,name=cache_validator,json=cacheValidator,proto3" json:"cache_validator,omitempty"` + DiscoveredAt *timestamppb.Timestamp `protobuf:"bytes,4,opt,name=discovered_at,json=discoveredAt,proto3" json:"discovered_at,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *SourceCacheEntryRecord) Reset() { @@ -1805,16 +1806,16 @@ func (x *SourceCacheEntryRecord) GetRowKind() string { return "" } -func (x *SourceCacheEntryRecord) GetScopeHash() string { +func (x *SourceCacheEntryRecord) GetScopeKey() string { if x != nil { - return x.ScopeHash + return x.ScopeKey } return "" } -func (x *SourceCacheEntryRecord) GetEtag() string { +func (x *SourceCacheEntryRecord) GetCacheValidator() string { if x != nil { - return x.Etag + return x.CacheValidator } return "" } @@ -1830,12 +1831,12 @@ func (x *SourceCacheEntryRecord) SetRowKind(v string) { x.RowKind = v } -func (x *SourceCacheEntryRecord) SetScopeHash(v string) { - x.ScopeHash = v +func (x *SourceCacheEntryRecord) SetScopeKey(v string) { + x.ScopeKey = v } -func (x *SourceCacheEntryRecord) SetEtag(v string) { - x.Etag = v +func (x *SourceCacheEntryRecord) SetCacheValidator(v string) { + x.CacheValidator = v } func (x *SourceCacheEntryRecord) SetDiscoveredAt(v *timestamppb.Timestamp) { @@ -1859,11 +1860,12 @@ type SourceCacheEntryRecord_builder struct { // Row kind partition: "resources", "entitlements", or "grants" // (pkg/sourcecache.RowKind values). RowKind string - // Connector-computed scope hash (lowercase hex). - ScopeHash string + // Connector-computed stable scope identifier (conventionally lowercase + // hex when hashed via sourcecache.HashScope; any byte-stable string). + ScopeKey string // Opaque upstream validator: HTTP ETag, delta token, etc. - Etag string - DiscoveredAt *timestamppb.Timestamp + CacheValidator string + DiscoveredAt *timestamppb.Timestamp } func (b0 SourceCacheEntryRecord_builder) Build() *SourceCacheEntryRecord { @@ -1871,8 +1873,8 @@ func (b0 SourceCacheEntryRecord_builder) Build() *SourceCacheEntryRecord { b, x := &b0, m0 _, _ = b, x x.RowKind = b.RowKind - x.ScopeHash = b.ScopeHash - x.Etag = b.Etag + x.ScopeKey = b.ScopeKey + x.CacheValidator = b.CacheValidator x.DiscoveredAt = b.DiscoveredAt return m0 } @@ -1901,7 +1903,7 @@ const file_c1_storage_v3_records_proto_rawDesc = "" + "\rdiscovered_at\x18\x06 \x01(\v2\x1a.google.protobuf.TimestampR\fdiscoveredAt\x12 \n" + "\vdescription\x18\a \x01(\tR\vdescription\x12-\n" + "\x12sourced_externally\x18\b \x01(\bR\x11sourcedExternally:!\x82\xf9+\x1d\n" + - "\x0eresource_types\x12\vexternal_idJ\x04\b\x01\x10\x02R\async_id\"\x98\x04\n" + + "\x0eresource_types\x12\vexternal_idJ\x04\b\x01\x10\x02R\async_id\"\x95\x04\n" + "\x0eResourceRecord\x12(\n" + "\x10resource_type_id\x18\x02 \x01(\tR\x0eresourceTypeId\x12\x1f\n" + "\vresource_id\x18\x03 \x01(\tR\n" + @@ -1911,10 +1913,10 @@ const file_c1_storage_v3_records_proto_rawDesc = "" + "\x06parent\x18\x06 \x01(\v2\x1a.c1.storage.v3.ResourceRefB.\x8a\xf9+*\n" + "\tby_parent\x1a\x10resource_type_id\x1a\vresource_idR\x06parent\x126\n" + "\vannotations\x18\a \x03(\v2\x14.google.protobuf.AnyR\vannotations\x12?\n" + - "\rdiscovered_at\x18\b \x01(\v2\x1a.google.protobuf.TimestampR\fdiscoveredAt\x12Z\n" + - "\x11source_scope_hash\x18\t \x01(\tB.\x8a\xf9+*\n" + - "\x0fby_source_scope\"\x17source_scope_hash != ''R\x0fsourceScopeHash:.\x82\xf9+*\n" + - "\tresources\x12\x10resource_type_id\x12\vresource_idJ\x04\b\x01\x10\x02R\async_id\"\xda\x04\n" + + "\rdiscovered_at\x18\b \x01(\v2\x1a.google.protobuf.TimestampR\fdiscoveredAt\x12W\n" + + "\x10source_scope_key\x18\t \x01(\tB-\x8a\xf9+)\n" + + "\x0fby_source_scope\"\x16source_scope_key != ''R\x0esourceScopeKey:.\x82\xf9+*\n" + + "\tresources\x12\x10resource_type_id\x12\vresource_idJ\x04\b\x01\x10\x02R\async_id\"\xd7\x04\n" + "\x11EntitlementRecord\x12\x1f\n" + "\vexternal_id\x18\x02 \x01(\tR\n" + "externalId\x12h\n" + @@ -1927,10 +1929,10 @@ const file_c1_storage_v3_records_proto_rawDesc = "" + "\rdiscovered_at\x18\b \x01(\v2\x1a.google.protobuf.TimestampR\fdiscoveredAt\x12\x12\n" + "\x04slug\x18\t \x01(\tR\x04slug\x12B\n" + "\x1egrantable_to_resource_type_ids\x18\n" + - " \x03(\tR\x1agrantableToResourceTypeIds\x12Z\n" + - "\x11source_scope_hash\x18\v \x01(\tB.\x8a\xf9+*\n" + - "\x0fby_source_scope\"\x17source_scope_hash != ''R\x0fsourceScopeHash:\x1f\x82\xf9+\x1b\n" + - "\fentitlements\x12\vexternal_idJ\x04\b\x01\x10\x02R\async_id\"\xf6\x06\n" + + " \x03(\tR\x1agrantableToResourceTypeIds\x12W\n" + + "\x10source_scope_key\x18\v \x01(\tB-\x8a\xf9+)\n" + + "\x0fby_source_scope\"\x16source_scope_key != ''R\x0esourceScopeKey:\x1f\x82\xf9+\x1b\n" + + "\fentitlements\x12\vexternal_idJ\x04\b\x01\x10\x02R\async_id\"\xf3\x06\n" + "\vGrantRecord\x12\x1f\n" + "\vexternal_id\x18\x02 \x01(\tR\n" + "externalId\x12\x98\x01\n" + @@ -1943,10 +1945,10 @@ const file_c1_storage_v3_records_proto_rawDesc = "" + "\x0fneeds_expansion\x18\a \x01(\bB0\x8a\xf9+,\n" + "\x12by_needs_expansion\"\x16needs_expansion = trueR\x0eneedsExpansion\x126\n" + "\vannotations\x18\b \x03(\v2\x14.google.protobuf.AnyR\vannotations\x12A\n" + - "\asources\x18\t \x03(\v2'.c1.storage.v3.GrantRecord.SourcesEntryR\asources\x12Z\n" + - "\x11source_scope_hash\x18\n" + - " \x01(\tB.\x8a\xf9+*\n" + - "\x0fby_source_scope\"\x17source_scope_hash != ''R\x0fsourceScopeHash\x1a\\\n" + + "\asources\x18\t \x03(\v2'.c1.storage.v3.GrantRecord.SourcesEntryR\asources\x12W\n" + + "\x10source_scope_key\x18\n" + + " \x01(\tB-\x8a\xf9+)\n" + + "\x0fby_source_scope\"\x16source_scope_key != ''R\x0esourceScopeKey\x1a\\\n" + "\fSourcesEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x126\n" + "\x05value\x18\x02 \x01(\v2 .c1.storage.v3.GrantSourceRecordR\x05value:\x028\x01:\x19\x82\xf9+\x15\n" + @@ -1997,15 +1999,13 @@ const file_c1_storage_v3_records_proto_rawDesc = "" + "\async_id\x18\x01 \x01(\tR\x06syncId\x12\x10\n" + "\x03key\x18\x02 \x01(\tR\x03key\x12\x14\n" + "\x05value\x18\x03 \x01(\fR\x05value:\x1c\x82\xf9+\x18\n" + - "\bsessions\x12\async_id\x12\x03key\"\xd9\x01\n" + + "\bsessions\x12\async_id\x12\x03key\"\xeb\x01\n" + "\x16SourceCacheEntryRecord\x12\x19\n" + - "\brow_kind\x18\x01 \x01(\tR\arowKind\x12\x1d\n" + - "\n" + - "scope_hash\x18\x02 \x01(\tR\tscopeHash\x12\x12\n" + - "\x04etag\x18\x03 \x01(\tR\x04etag\x12?\n" + - "\rdiscovered_at\x18\x04 \x01(\v2\x1a.google.protobuf.TimestampR\fdiscoveredAt:0\x82\xf9+,\n" + - "\x14source_cache_entries\x12\brow_kind\x12\n" + - "scope_hash*\xae\x01\n" + + "\brow_kind\x18\x01 \x01(\tR\arowKind\x12\x1b\n" + + "\tscope_key\x18\x02 \x01(\tR\bscopeKey\x12'\n" + + "\x0fcache_validator\x18\x03 \x01(\tR\x0ecacheValidator\x12?\n" + + "\rdiscovered_at\x18\x04 \x01(\v2\x1a.google.protobuf.TimestampR\fdiscoveredAt:/\x82\xf9++\n" + + "\x14source_cache_entries\x12\brow_kind\x12\tscope_key*\xae\x01\n" + "\bSyncType\x12\x19\n" + "\x15SYNC_TYPE_UNSPECIFIED\x10\x00\x12\x12\n" + "\x0eSYNC_TYPE_FULL\x10\x01\x12\x15\n" + diff --git a/pb/c1/storage/v3/records.pb.validate.go b/pb/c1/storage/v3/records.pb.validate.go index 4d040b1e1..bc3827e0a 100644 --- a/pb/c1/storage/v3/records.pb.validate.go +++ b/pb/c1/storage/v3/records.pb.validate.go @@ -544,7 +544,7 @@ func (m *ResourceRecord) validate(all bool) error { } } - // no validation rules for SourceScopeHash + // no validation rules for SourceScopeKey if len(errors) > 0 { return ResourceRecordMultiError(errors) @@ -748,7 +748,7 @@ func (m *EntitlementRecord) validate(all bool) error { // no validation rules for Slug - // no validation rules for SourceScopeHash + // no validation rules for SourceScopeKey if len(errors) > 0 { return EntitlementRecordMultiError(errors) @@ -1052,7 +1052,7 @@ func (m *GrantRecord) validate(all bool) error { } } - // no validation rules for SourceScopeHash + // no validation rules for SourceScopeKey if len(errors) > 0 { return GrantRecordMultiError(errors) @@ -1716,9 +1716,9 @@ func (m *SourceCacheEntryRecord) validate(all bool) error { // no validation rules for RowKind - // no validation rules for ScopeHash + // no validation rules for ScopeKey - // no validation rules for Etag + // no validation rules for CacheValidator if all { switch v := interface{}(m.GetDiscoveredAt()).(type) { diff --git a/pb/c1/storage/v3/records_protoopaque.pb.go b/pb/c1/storage/v3/records_protoopaque.pb.go index 21f9567b2..28a95a3c5 100644 --- a/pb/c1/storage/v3/records_protoopaque.pb.go +++ b/pb/c1/storage/v3/records_protoopaque.pb.go @@ -441,17 +441,17 @@ func (b0 ResourceTypeRecord_builder) Build() *ResourceTypeRecord { } type ResourceRecord struct { - state protoimpl.MessageState `protogen:"opaque.v1"` - xxx_hidden_ResourceTypeId string `protobuf:"bytes,2,opt,name=resource_type_id,json=resourceTypeId,proto3"` - xxx_hidden_ResourceId string `protobuf:"bytes,3,opt,name=resource_id,json=resourceId,proto3"` - xxx_hidden_DisplayName string `protobuf:"bytes,4,opt,name=display_name,json=displayName,proto3"` - xxx_hidden_Description string `protobuf:"bytes,5,opt,name=description,proto3"` - xxx_hidden_Parent *ResourceRef `protobuf:"bytes,6,opt,name=parent,proto3"` - xxx_hidden_Annotations *[]*anypb.Any `protobuf:"bytes,7,rep,name=annotations,proto3"` - xxx_hidden_DiscoveredAt *timestamppb.Timestamp `protobuf:"bytes,8,opt,name=discovered_at,json=discoveredAt,proto3"` - xxx_hidden_SourceScopeHash string `protobuf:"bytes,9,opt,name=source_scope_hash,json=sourceScopeHash,proto3"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"opaque.v1"` + xxx_hidden_ResourceTypeId string `protobuf:"bytes,2,opt,name=resource_type_id,json=resourceTypeId,proto3"` + xxx_hidden_ResourceId string `protobuf:"bytes,3,opt,name=resource_id,json=resourceId,proto3"` + xxx_hidden_DisplayName string `protobuf:"bytes,4,opt,name=display_name,json=displayName,proto3"` + xxx_hidden_Description string `protobuf:"bytes,5,opt,name=description,proto3"` + xxx_hidden_Parent *ResourceRef `protobuf:"bytes,6,opt,name=parent,proto3"` + xxx_hidden_Annotations *[]*anypb.Any `protobuf:"bytes,7,rep,name=annotations,proto3"` + xxx_hidden_DiscoveredAt *timestamppb.Timestamp `protobuf:"bytes,8,opt,name=discovered_at,json=discoveredAt,proto3"` + xxx_hidden_SourceScopeKey string `protobuf:"bytes,9,opt,name=source_scope_key,json=sourceScopeKey,proto3"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *ResourceRecord) Reset() { @@ -530,9 +530,9 @@ func (x *ResourceRecord) GetDiscoveredAt() *timestamppb.Timestamp { return nil } -func (x *ResourceRecord) GetSourceScopeHash() string { +func (x *ResourceRecord) GetSourceScopeKey() string { if x != nil { - return x.xxx_hidden_SourceScopeHash + return x.xxx_hidden_SourceScopeKey } return "" } @@ -565,8 +565,8 @@ func (x *ResourceRecord) SetDiscoveredAt(v *timestamppb.Timestamp) { x.xxx_hidden_DiscoveredAt = v } -func (x *ResourceRecord) SetSourceScopeHash(v string) { - x.xxx_hidden_SourceScopeHash = v +func (x *ResourceRecord) SetSourceScopeKey(v string) { + x.xxx_hidden_SourceScopeKey = v } func (x *ResourceRecord) HasParent() bool { @@ -603,7 +603,7 @@ type ResourceRecord_builder struct { DiscoveredAt *timestamppb.Timestamp // Source-cache scope stamp (see annotation_source_cache.proto). // Empty for rows not produced under a source-cache scope. - SourceScopeHash string + SourceScopeKey string } func (b0 ResourceRecord_builder) Build() *ResourceRecord { @@ -617,7 +617,7 @@ func (b0 ResourceRecord_builder) Build() *ResourceRecord { x.xxx_hidden_Parent = b.Parent x.xxx_hidden_Annotations = &b.Annotations x.xxx_hidden_DiscoveredAt = b.DiscoveredAt - x.xxx_hidden_SourceScopeHash = b.SourceScopeHash + x.xxx_hidden_SourceScopeKey = b.SourceScopeKey return m0 } @@ -632,7 +632,7 @@ type EntitlementRecord struct { xxx_hidden_DiscoveredAt *timestamppb.Timestamp `protobuf:"bytes,8,opt,name=discovered_at,json=discoveredAt,proto3"` xxx_hidden_Slug string `protobuf:"bytes,9,opt,name=slug,proto3"` xxx_hidden_GrantableToResourceTypeIds []string `protobuf:"bytes,10,rep,name=grantable_to_resource_type_ids,json=grantableToResourceTypeIds,proto3"` - xxx_hidden_SourceScopeHash string `protobuf:"bytes,11,opt,name=source_scope_hash,json=sourceScopeHash,proto3"` + xxx_hidden_SourceScopeKey string `protobuf:"bytes,11,opt,name=source_scope_key,json=sourceScopeKey,proto3"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -727,9 +727,9 @@ func (x *EntitlementRecord) GetGrantableToResourceTypeIds() []string { return nil } -func (x *EntitlementRecord) GetSourceScopeHash() string { +func (x *EntitlementRecord) GetSourceScopeKey() string { if x != nil { - return x.xxx_hidden_SourceScopeHash + return x.xxx_hidden_SourceScopeKey } return "" } @@ -770,8 +770,8 @@ func (x *EntitlementRecord) SetGrantableToResourceTypeIds(v []string) { x.xxx_hidden_GrantableToResourceTypeIds = v } -func (x *EntitlementRecord) SetSourceScopeHash(v string) { - x.xxx_hidden_SourceScopeHash = v +func (x *EntitlementRecord) SetSourceScopeKey(v string) { + x.xxx_hidden_SourceScopeKey = v } func (x *EntitlementRecord) HasResource() bool { @@ -822,7 +822,7 @@ type EntitlementRecord_builder struct { GrantableToResourceTypeIds []string // Source-cache scope stamp (see annotation_source_cache.proto). // Empty for rows not produced under a source-cache scope. - SourceScopeHash string + SourceScopeKey string } func (b0 EntitlementRecord_builder) Build() *EntitlementRecord { @@ -838,23 +838,23 @@ func (b0 EntitlementRecord_builder) Build() *EntitlementRecord { x.xxx_hidden_DiscoveredAt = b.DiscoveredAt x.xxx_hidden_Slug = b.Slug x.xxx_hidden_GrantableToResourceTypeIds = b.GrantableToResourceTypeIds - x.xxx_hidden_SourceScopeHash = b.SourceScopeHash + x.xxx_hidden_SourceScopeKey = b.SourceScopeKey return m0 } type GrantRecord struct { - state protoimpl.MessageState `protogen:"opaque.v1"` - xxx_hidden_ExternalId string `protobuf:"bytes,2,opt,name=external_id,json=externalId,proto3"` - xxx_hidden_Entitlement *EntitlementRef `protobuf:"bytes,3,opt,name=entitlement,proto3"` - xxx_hidden_Principal *PrincipalRef `protobuf:"bytes,4,opt,name=principal,proto3"` - xxx_hidden_DiscoveredAt *timestamppb.Timestamp `protobuf:"bytes,5,opt,name=discovered_at,json=discoveredAt,proto3"` - xxx_hidden_Expansion *GrantExpandableRecord `protobuf:"bytes,6,opt,name=expansion,proto3"` - xxx_hidden_NeedsExpansion bool `protobuf:"varint,7,opt,name=needs_expansion,json=needsExpansion,proto3"` - xxx_hidden_Annotations *[]*anypb.Any `protobuf:"bytes,8,rep,name=annotations,proto3"` - xxx_hidden_Sources map[string]*GrantSourceRecord `protobuf:"bytes,9,rep,name=sources,proto3" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` - xxx_hidden_SourceScopeHash string `protobuf:"bytes,10,opt,name=source_scope_hash,json=sourceScopeHash,proto3"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"opaque.v1"` + xxx_hidden_ExternalId string `protobuf:"bytes,2,opt,name=external_id,json=externalId,proto3"` + xxx_hidden_Entitlement *EntitlementRef `protobuf:"bytes,3,opt,name=entitlement,proto3"` + xxx_hidden_Principal *PrincipalRef `protobuf:"bytes,4,opt,name=principal,proto3"` + xxx_hidden_DiscoveredAt *timestamppb.Timestamp `protobuf:"bytes,5,opt,name=discovered_at,json=discoveredAt,proto3"` + xxx_hidden_Expansion *GrantExpandableRecord `protobuf:"bytes,6,opt,name=expansion,proto3"` + xxx_hidden_NeedsExpansion bool `protobuf:"varint,7,opt,name=needs_expansion,json=needsExpansion,proto3"` + xxx_hidden_Annotations *[]*anypb.Any `protobuf:"bytes,8,rep,name=annotations,proto3"` + xxx_hidden_Sources map[string]*GrantSourceRecord `protobuf:"bytes,9,rep,name=sources,proto3" protobuf_key:"bytes,1,opt,name=key" protobuf_val:"bytes,2,opt,name=value"` + xxx_hidden_SourceScopeKey string `protobuf:"bytes,10,opt,name=source_scope_key,json=sourceScopeKey,proto3"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *GrantRecord) Reset() { @@ -940,9 +940,9 @@ func (x *GrantRecord) GetSources() map[string]*GrantSourceRecord { return nil } -func (x *GrantRecord) GetSourceScopeHash() string { +func (x *GrantRecord) GetSourceScopeKey() string { if x != nil { - return x.xxx_hidden_SourceScopeHash + return x.xxx_hidden_SourceScopeKey } return "" } @@ -979,8 +979,8 @@ func (x *GrantRecord) SetSources(v map[string]*GrantSourceRecord) { x.xxx_hidden_Sources = v } -func (x *GrantRecord) SetSourceScopeHash(v string) { - x.xxx_hidden_SourceScopeHash = v +func (x *GrantRecord) SetSourceScopeKey(v string) { + x.xxx_hidden_SourceScopeKey = v } func (x *GrantRecord) HasEntitlement() bool { @@ -1056,7 +1056,7 @@ type GrantRecord_builder struct { // expander-derived grants, which are recreated by expansion each sync // and never replayed. StoreExpandedGrants preserves this field on // rewrites of existing rows, exactly like expansion/needs_expansion. - SourceScopeHash string + SourceScopeKey string } func (b0 GrantRecord_builder) Build() *GrantRecord { @@ -1071,7 +1071,7 @@ func (b0 GrantRecord_builder) Build() *GrantRecord { x.xxx_hidden_NeedsExpansion = b.NeedsExpansion x.xxx_hidden_Annotations = &b.Annotations x.xxx_hidden_Sources = b.Sources - x.xxx_hidden_SourceScopeHash = b.SourceScopeHash + x.xxx_hidden_SourceScopeKey = b.SourceScopeKey return m0 } @@ -1377,7 +1377,7 @@ type SyncRunRecord_builder struct { // compacted marks a sync produced by compaction (fold or rebuild) // rather than by a real connector run. Compacted artifacts are // keep-newer UPSERT merges — base rows a newer input deleted survive — - // so no input sync's upstream validators (source-cache etags) describe + // so no input sync's upstream validators (source-cache validators) describe // their contents. // // "Can this sync be used as a source-cache replay source?" is the @@ -1702,18 +1702,18 @@ func (b0 SessionRecord_builder) Build() *SessionRecord { // SourceCacheEntryRecord is the per-scope manifest entry for source-cache // replay (see c1/connector/v2/annotation_source_cache.proto). One entry -// per (row_kind, scope_hash), written for every freshly fetched scope — +// per (row_kind, scope_key), written for every freshly fetched scope — // including zero-row responses — and rewritten on replay with the scope's // current validator. The previous sync's entries (read from the previous // c1z) are the lookup surface connectors revalidate against. type SourceCacheEntryRecord struct { - state protoimpl.MessageState `protogen:"opaque.v1"` - xxx_hidden_RowKind string `protobuf:"bytes,1,opt,name=row_kind,json=rowKind,proto3"` - xxx_hidden_ScopeHash string `protobuf:"bytes,2,opt,name=scope_hash,json=scopeHash,proto3"` - xxx_hidden_Etag string `protobuf:"bytes,3,opt,name=etag,proto3"` - xxx_hidden_DiscoveredAt *timestamppb.Timestamp `protobuf:"bytes,4,opt,name=discovered_at,json=discoveredAt,proto3"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"opaque.v1"` + xxx_hidden_RowKind string `protobuf:"bytes,1,opt,name=row_kind,json=rowKind,proto3"` + xxx_hidden_ScopeKey string `protobuf:"bytes,2,opt,name=scope_key,json=scopeKey,proto3"` + xxx_hidden_CacheValidator string `protobuf:"bytes,3,opt,name=cache_validator,json=cacheValidator,proto3"` + xxx_hidden_DiscoveredAt *timestamppb.Timestamp `protobuf:"bytes,4,opt,name=discovered_at,json=discoveredAt,proto3"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *SourceCacheEntryRecord) Reset() { @@ -1748,16 +1748,16 @@ func (x *SourceCacheEntryRecord) GetRowKind() string { return "" } -func (x *SourceCacheEntryRecord) GetScopeHash() string { +func (x *SourceCacheEntryRecord) GetScopeKey() string { if x != nil { - return x.xxx_hidden_ScopeHash + return x.xxx_hidden_ScopeKey } return "" } -func (x *SourceCacheEntryRecord) GetEtag() string { +func (x *SourceCacheEntryRecord) GetCacheValidator() string { if x != nil { - return x.xxx_hidden_Etag + return x.xxx_hidden_CacheValidator } return "" } @@ -1773,12 +1773,12 @@ func (x *SourceCacheEntryRecord) SetRowKind(v string) { x.xxx_hidden_RowKind = v } -func (x *SourceCacheEntryRecord) SetScopeHash(v string) { - x.xxx_hidden_ScopeHash = v +func (x *SourceCacheEntryRecord) SetScopeKey(v string) { + x.xxx_hidden_ScopeKey = v } -func (x *SourceCacheEntryRecord) SetEtag(v string) { - x.xxx_hidden_Etag = v +func (x *SourceCacheEntryRecord) SetCacheValidator(v string) { + x.xxx_hidden_CacheValidator = v } func (x *SourceCacheEntryRecord) SetDiscoveredAt(v *timestamppb.Timestamp) { @@ -1802,11 +1802,12 @@ type SourceCacheEntryRecord_builder struct { // Row kind partition: "resources", "entitlements", or "grants" // (pkg/sourcecache.RowKind values). RowKind string - // Connector-computed scope hash (lowercase hex). - ScopeHash string + // Connector-computed stable scope identifier (conventionally lowercase + // hex when hashed via sourcecache.HashScope; any byte-stable string). + ScopeKey string // Opaque upstream validator: HTTP ETag, delta token, etc. - Etag string - DiscoveredAt *timestamppb.Timestamp + CacheValidator string + DiscoveredAt *timestamppb.Timestamp } func (b0 SourceCacheEntryRecord_builder) Build() *SourceCacheEntryRecord { @@ -1814,8 +1815,8 @@ func (b0 SourceCacheEntryRecord_builder) Build() *SourceCacheEntryRecord { b, x := &b0, m0 _, _ = b, x x.xxx_hidden_RowKind = b.RowKind - x.xxx_hidden_ScopeHash = b.ScopeHash - x.xxx_hidden_Etag = b.Etag + x.xxx_hidden_ScopeKey = b.ScopeKey + x.xxx_hidden_CacheValidator = b.CacheValidator x.xxx_hidden_DiscoveredAt = b.DiscoveredAt return m0 } @@ -1844,7 +1845,7 @@ const file_c1_storage_v3_records_proto_rawDesc = "" + "\rdiscovered_at\x18\x06 \x01(\v2\x1a.google.protobuf.TimestampR\fdiscoveredAt\x12 \n" + "\vdescription\x18\a \x01(\tR\vdescription\x12-\n" + "\x12sourced_externally\x18\b \x01(\bR\x11sourcedExternally:!\x82\xf9+\x1d\n" + - "\x0eresource_types\x12\vexternal_idJ\x04\b\x01\x10\x02R\async_id\"\x98\x04\n" + + "\x0eresource_types\x12\vexternal_idJ\x04\b\x01\x10\x02R\async_id\"\x95\x04\n" + "\x0eResourceRecord\x12(\n" + "\x10resource_type_id\x18\x02 \x01(\tR\x0eresourceTypeId\x12\x1f\n" + "\vresource_id\x18\x03 \x01(\tR\n" + @@ -1854,10 +1855,10 @@ const file_c1_storage_v3_records_proto_rawDesc = "" + "\x06parent\x18\x06 \x01(\v2\x1a.c1.storage.v3.ResourceRefB.\x8a\xf9+*\n" + "\tby_parent\x1a\x10resource_type_id\x1a\vresource_idR\x06parent\x126\n" + "\vannotations\x18\a \x03(\v2\x14.google.protobuf.AnyR\vannotations\x12?\n" + - "\rdiscovered_at\x18\b \x01(\v2\x1a.google.protobuf.TimestampR\fdiscoveredAt\x12Z\n" + - "\x11source_scope_hash\x18\t \x01(\tB.\x8a\xf9+*\n" + - "\x0fby_source_scope\"\x17source_scope_hash != ''R\x0fsourceScopeHash:.\x82\xf9+*\n" + - "\tresources\x12\x10resource_type_id\x12\vresource_idJ\x04\b\x01\x10\x02R\async_id\"\xda\x04\n" + + "\rdiscovered_at\x18\b \x01(\v2\x1a.google.protobuf.TimestampR\fdiscoveredAt\x12W\n" + + "\x10source_scope_key\x18\t \x01(\tB-\x8a\xf9+)\n" + + "\x0fby_source_scope\"\x16source_scope_key != ''R\x0esourceScopeKey:.\x82\xf9+*\n" + + "\tresources\x12\x10resource_type_id\x12\vresource_idJ\x04\b\x01\x10\x02R\async_id\"\xd7\x04\n" + "\x11EntitlementRecord\x12\x1f\n" + "\vexternal_id\x18\x02 \x01(\tR\n" + "externalId\x12h\n" + @@ -1870,10 +1871,10 @@ const file_c1_storage_v3_records_proto_rawDesc = "" + "\rdiscovered_at\x18\b \x01(\v2\x1a.google.protobuf.TimestampR\fdiscoveredAt\x12\x12\n" + "\x04slug\x18\t \x01(\tR\x04slug\x12B\n" + "\x1egrantable_to_resource_type_ids\x18\n" + - " \x03(\tR\x1agrantableToResourceTypeIds\x12Z\n" + - "\x11source_scope_hash\x18\v \x01(\tB.\x8a\xf9+*\n" + - "\x0fby_source_scope\"\x17source_scope_hash != ''R\x0fsourceScopeHash:\x1f\x82\xf9+\x1b\n" + - "\fentitlements\x12\vexternal_idJ\x04\b\x01\x10\x02R\async_id\"\xf6\x06\n" + + " \x03(\tR\x1agrantableToResourceTypeIds\x12W\n" + + "\x10source_scope_key\x18\v \x01(\tB-\x8a\xf9+)\n" + + "\x0fby_source_scope\"\x16source_scope_key != ''R\x0esourceScopeKey:\x1f\x82\xf9+\x1b\n" + + "\fentitlements\x12\vexternal_idJ\x04\b\x01\x10\x02R\async_id\"\xf3\x06\n" + "\vGrantRecord\x12\x1f\n" + "\vexternal_id\x18\x02 \x01(\tR\n" + "externalId\x12\x98\x01\n" + @@ -1886,10 +1887,10 @@ const file_c1_storage_v3_records_proto_rawDesc = "" + "\x0fneeds_expansion\x18\a \x01(\bB0\x8a\xf9+,\n" + "\x12by_needs_expansion\"\x16needs_expansion = trueR\x0eneedsExpansion\x126\n" + "\vannotations\x18\b \x03(\v2\x14.google.protobuf.AnyR\vannotations\x12A\n" + - "\asources\x18\t \x03(\v2'.c1.storage.v3.GrantRecord.SourcesEntryR\asources\x12Z\n" + - "\x11source_scope_hash\x18\n" + - " \x01(\tB.\x8a\xf9+*\n" + - "\x0fby_source_scope\"\x17source_scope_hash != ''R\x0fsourceScopeHash\x1a\\\n" + + "\asources\x18\t \x03(\v2'.c1.storage.v3.GrantRecord.SourcesEntryR\asources\x12W\n" + + "\x10source_scope_key\x18\n" + + " \x01(\tB-\x8a\xf9+)\n" + + "\x0fby_source_scope\"\x16source_scope_key != ''R\x0esourceScopeKey\x1a\\\n" + "\fSourcesEntry\x12\x10\n" + "\x03key\x18\x01 \x01(\tR\x03key\x126\n" + "\x05value\x18\x02 \x01(\v2 .c1.storage.v3.GrantSourceRecordR\x05value:\x028\x01:\x19\x82\xf9+\x15\n" + @@ -1940,15 +1941,13 @@ const file_c1_storage_v3_records_proto_rawDesc = "" + "\async_id\x18\x01 \x01(\tR\x06syncId\x12\x10\n" + "\x03key\x18\x02 \x01(\tR\x03key\x12\x14\n" + "\x05value\x18\x03 \x01(\fR\x05value:\x1c\x82\xf9+\x18\n" + - "\bsessions\x12\async_id\x12\x03key\"\xd9\x01\n" + + "\bsessions\x12\async_id\x12\x03key\"\xeb\x01\n" + "\x16SourceCacheEntryRecord\x12\x19\n" + - "\brow_kind\x18\x01 \x01(\tR\arowKind\x12\x1d\n" + - "\n" + - "scope_hash\x18\x02 \x01(\tR\tscopeHash\x12\x12\n" + - "\x04etag\x18\x03 \x01(\tR\x04etag\x12?\n" + - "\rdiscovered_at\x18\x04 \x01(\v2\x1a.google.protobuf.TimestampR\fdiscoveredAt:0\x82\xf9+,\n" + - "\x14source_cache_entries\x12\brow_kind\x12\n" + - "scope_hash*\xae\x01\n" + + "\brow_kind\x18\x01 \x01(\tR\arowKind\x12\x1b\n" + + "\tscope_key\x18\x02 \x01(\tR\bscopeKey\x12'\n" + + "\x0fcache_validator\x18\x03 \x01(\tR\x0ecacheValidator\x12?\n" + + "\rdiscovered_at\x18\x04 \x01(\v2\x1a.google.protobuf.TimestampR\fdiscoveredAt:/\x82\xf9++\n" + + "\x14source_cache_entries\x12\brow_kind\x12\tscope_key*\xae\x01\n" + "\bSyncType\x12\x19\n" + "\x15SYNC_TYPE_UNSPECIFIED\x10\x00\x12\x12\n" + "\x0eSYNC_TYPE_FULL\x10\x01\x12\x15\n" + diff --git a/pkg/connectorbuilder/continuation_test.go b/pkg/connectorbuilder/continuation_test.go index ad871cc2f..cf679d151 100644 --- a/pkg/connectorbuilder/continuation_test.go +++ b/pkg/connectorbuilder/continuation_test.go @@ -36,9 +36,9 @@ func (s *continuationTestSyncer) Grants(ctx context.Context, r *v2.Resource, opt s.lookupCalls++ if s.batch { queries := []sourcecache.Query{ - {RowKind: sourcecache.RowKindGrants, ScopeHash: s.scope + "/1"}, - {RowKind: sourcecache.RowKindGrants, ScopeHash: s.scope + "/2"}, - {RowKind: sourcecache.RowKindGrants, ScopeHash: s.scope + "/3"}, + {RowKind: sourcecache.RowKindGrants, ScopeKey: s.scope + "/1"}, + {RowKind: sourcecache.RowKindGrants, ScopeKey: s.scope + "/2"}, + {RowKind: sourcecache.RowKindGrants, ScopeKey: s.scope + "/3"}, } answers, err := sourcecache.LookupMany(ctx, opts.SourceCache, queries) if err != nil { @@ -46,26 +46,26 @@ func (s *continuationTestSyncer) Grants(ctx context.Context, r *v2.Resource, opt } ret := &resource.SyncOpResults{Annotations: annotations.Annotations{}} for _, a := range answers { - if !a.Found || a.ETag != "current" { + if !a.Found || a.CacheValidator != "current" { return nil, nil, fmt.Errorf("test wants all hits, got %+v", a) } } - ret.Annotations.Update(v2.SourceCacheReplay_builder{ScopeHash: s.scope + "/1", Etag: "current"}.Build()) + ret.Annotations.Update(v2.SourceCacheReplay_builder{ScopeKey: s.scope + "/1", CacheValidator: "current"}.Build()) return nil, ret, nil } - entry, found, err := opts.SourceCache.LookupPreviousSourceCache(ctx, sourcecache.RowKindGrants, s.scope) + entry, found, err := opts.SourceCache.Lookup(ctx, sourcecache.RowKindGrants, s.scope) if err != nil && !s.swallow { // Idiomatic wrapping — the builder must match via errors.Is. return nil, nil, fmt.Errorf("listing grants for %s: %w", r.GetId().GetResource(), err) } ret := &resource.SyncOpResults{Annotations: annotations.Annotations{}} - if err == nil && found && entry.ETag == "current" { - ret.Annotations.Update(v2.SourceCacheReplay_builder{ScopeHash: s.scope, Etag: entry.ETag}.Build()) + if err == nil && found && entry.CacheValidator == "current" { + ret.Annotations.Update(v2.SourceCacheReplay_builder{ScopeKey: s.scope, CacheValidator: entry.CacheValidator}.Build()) return nil, ret, nil } // Miss (or swallowed deferral): cold row. - ret.Annotations.Update(v2.SourceCacheScope_builder{ScopeHash: s.scope, Etag: "current"}.Build()) + ret.Annotations.Update(v2.SourceCacheRecord_builder{ScopeKey: s.scope, CacheValidator: "current"}.Build()) return []*v2.Grant{mkTestGrant(r)}, ret, nil } @@ -117,13 +117,13 @@ func TestContinuation_DeferThenAnswer(t *testing.T) { require.NoError(t, err) require.True(t, hasAsk) require.Len(t, ask.GetQueries(), 1) - require.Equal(t, "groups/g1/members", ask.GetQueries()[0].GetScopeHash()) + require.Equal(t, "groups/g1/members", ask.GetQueries()[0].GetScopeKey()) require.Equal(t, string(sourcecache.RowKindGrants), ask.GetQueries()[0].GetRowKind()) // Phase 2: same request + answers → the SAME connector code replays. answers := annotations.New(&v2.SourceCacheLookupOffer{}) answers.Update(sourcecache.AnswersProto([]sourcecache.Answer{ - {Query: sourcecache.Query{RowKind: sourcecache.RowKindGrants, ScopeHash: "groups/g1/members"}, Found: true, ETag: "current"}, + {Query: sourcecache.Query{RowKind: sourcecache.RowKindGrants, ScopeKey: "groups/g1/members"}, Found: true, CacheValidator: "current"}, })) resp2, err := b.ListGrants(ctx, continuationGrantsRequest(answers)) require.NoError(t, err) @@ -137,7 +137,7 @@ func TestContinuation_DeferThenAnswer(t *testing.T) { // A not-found answer is a MISS, not a deferral: cold fetch. miss := annotations.New(&v2.SourceCacheLookupOffer{}) miss.Update(sourcecache.AnswersProto([]sourcecache.Answer{ - {Query: sourcecache.Query{RowKind: sourcecache.RowKindGrants, ScopeHash: "groups/g1/members"}, Found: false}, + {Query: sourcecache.Query{RowKind: sourcecache.RowKindGrants, ScopeKey: "groups/g1/members"}, Found: false}, })) resp3, err := b.ListGrants(ctx, continuationGrantsRequest(miss)) require.NoError(t, err) @@ -161,8 +161,8 @@ func TestContinuation_LookupManyBatchesOneAsk(t *testing.T) { var answers []sourcecache.Answer for _, q := range ask.GetQueries() { answers = append(answers, sourcecache.Answer{ - Query: sourcecache.Query{RowKind: sourcecache.RowKind(q.GetRowKind()), ScopeHash: q.GetScopeHash()}, - Found: true, ETag: "current", + Query: sourcecache.Query{RowKind: sourcecache.RowKind(q.GetRowKind()), ScopeKey: q.GetScopeKey()}, + Found: true, CacheValidator: "current", }) } req := annotations.New(&v2.SourceCacheLookupOffer{}) @@ -217,6 +217,6 @@ func TestContinuation_DirectLookupWins(t *testing.T) { type staticTestLookup struct{ etag string } -func (s staticTestLookup) LookupPreviousSourceCache(context.Context, sourcecache.RowKind, string) (sourcecache.Entry, bool, error) { - return sourcecache.Entry{ETag: s.etag}, true, nil +func (s staticTestLookup) Lookup(context.Context, sourcecache.RowKind, string) (sourcecache.Entry, bool, error) { + return sourcecache.Entry{CacheValidator: s.etag}, true, nil } diff --git a/pkg/connectorbuilder/resource_syncer.go b/pkg/connectorbuilder/resource_syncer.go index c5aa24f83..e48052cf7 100644 --- a/pkg/connectorbuilder/resource_syncer.go +++ b/pkg/connectorbuilder/resource_syncer.go @@ -76,7 +76,7 @@ type StaticEntitlementSyncerV2 interface { // // The first call of a sync arrives with an empty page token (the planning // call); the connector may answer with rows directly and/or spawn -// additional independent cursors by attaching a v2.SpawnCursors annotation +// additional independent cursors by attaching a v2.EnqueuePageTokens annotation // whose page tokens are delivered back through opts.PageToken, one action // each. Source-cache scope/replay/tombstone annotations work exactly as on // per-resource grants pages. @@ -91,7 +91,7 @@ type TypeScopedGrantsSyncer interface { // annotation; the syncer then issues ListEntitlements calls with an empty // resource id, which the builder routes here. // -// Planning / SpawnCursors / source-cache behavior matches TypeScopedGrantsSyncer. +// Planning / EnqueuePageTokens / source-cache behavior matches TypeScopedGrantsSyncer. type TypeScopedEntitlementsSyncer interface { EntitlementsForResourceType(ctx context.Context, resourceTypeID string, opts resource.SyncOpAttrs) ([]*v2.Entitlement, *resource.SyncOpResults, error) } diff --git a/pkg/connectorbuilder/type_scoped_entitlements_test.go b/pkg/connectorbuilder/type_scoped_entitlements_test.go index 654f39ddb..b0bb66826 100644 --- a/pkg/connectorbuilder/type_scoped_entitlements_test.go +++ b/pkg/connectorbuilder/type_scoped_entitlements_test.go @@ -29,7 +29,7 @@ func (s *typeScopedEntitlementsSyncer) Entitlements(ctx context.Context, r *v2.R func (s *typeScopedEntitlementsSyncer) EntitlementsForResourceType(ctx context.Context, resourceTypeID string, opts resource.SyncOpAttrs) ([]*v2.Entitlement, *resource.SyncOpResults, error) { s.calls++ if s.deferredLookup { - _, _, err := opts.SourceCache.LookupPreviousSourceCache(ctx, sourcecache.RowKindEntitlements, "ents/type") + _, _, err := opts.SourceCache.Lookup(ctx, sourcecache.RowKindEntitlements, "ents/type") if err != nil { return nil, nil, err } @@ -112,7 +112,7 @@ func TestListEntitlements_TypeScopedContinuationPreservesMarker(t *testing.T) { answers := annotations.New(&v2.SourceCacheLookupOffer{}, &v2.TypeScopedEntitlements{}) answers.Update(sourcecache.AnswersProto([]sourcecache.Answer{ - {Query: sourcecache.Query{RowKind: sourcecache.RowKindEntitlements, ScopeHash: "ents/type"}, Found: false}, + {Query: sourcecache.Query{RowKind: sourcecache.RowKindEntitlements, ScopeKey: "ents/type"}, Found: false}, })) resp2, err := conn.ListEntitlements(ctx, typeScopedEntitlementsRequest(answers)) require.NoError(t, err) diff --git a/pkg/dotc1z/assets.go b/pkg/dotc1z/assets.go index 618f36318..863127cb4 100644 --- a/pkg/dotc1z/assets.go +++ b/pkg/dotc1z/assets.go @@ -104,7 +104,7 @@ func (c *C1File) PutAsset(ctx context.Context, assetRef *v2.AssetRef, contentTyp return err } - c.dbUpdated = true + c.dbUpdated.Store(true) return nil } diff --git a/pkg/dotc1z/c1file.go b/pkg/dotc1z/c1file.go index a9feeec68..7cf1d5ba1 100644 --- a/pkg/dotc1z/c1file.go +++ b/pkg/dotc1z/c1file.go @@ -10,6 +10,7 @@ import ( "path/filepath" "strings" "sync" + "sync/atomic" "time" "github.com/doug-martin/goqu/v9" @@ -49,7 +50,10 @@ type C1File struct { viewSyncID string outputFilePath string dbFilePath string - dbUpdated bool + // dbUpdated records that any mutation ran; parallel sync workers set + // it concurrently, so it is atomic (it was a plain bool data race + // under -race). Only ever set true during a session; read at Close. + dbUpdated atomic.Bool tempDir string pragmas []pragma readOnly bool @@ -624,7 +628,7 @@ func (c *C1File) Close(ctx context.Context) (retErr error) { span.SetAttributes( attribute.Bool("read_only", c.readOnly), - attribute.Bool("db_updated", c.dbUpdated), + attribute.Bool("db_updated", c.dbUpdated.Load()), attribute.String("db_path", c.dbFilePath), ) @@ -635,13 +639,13 @@ func (c *C1File) Close(ctx context.Context) (retErr error) { // open) before returning so a misuse like opening read-only and // then dirtying via an attached-db mutation still releases the // SQLite handle and any FDs/goroutines it owns. - if !c.dbUpdated || c.readOnly { + if !c.dbUpdated.Load() || c.readOnly { if c.rawDb != nil { if err := c.closeRawDB(ctx); err != nil { return cleanupDbDir(c.dbFilePath, err) } } - if c.dbUpdated && c.readOnly { + if c.dbUpdated.Load() && c.readOnly { c.closed = true return cleanupDbDir(c.dbFilePath, ErrReadOnly) } @@ -1331,7 +1335,7 @@ func (c *C1File) stats(ctx context.Context, syncType connectorstore.SyncType, sy if err != nil { return nil, nil, fmt.Errorf("c1file-stats: error saving stats: %w", err) } - c.dbUpdated = true + c.dbUpdated.Store(true) } return &reader_v2.SyncRun{ diff --git a/pkg/dotc1z/c1file_attached.go b/pkg/dotc1z/c1file_attached.go index fa6a4c96b..6ae5a76f4 100644 --- a/pkg/dotc1z/c1file_attached.go +++ b/pkg/dotc1z/c1file_attached.go @@ -346,7 +346,7 @@ func (c *C1FileAttached) GenerateSyncDiffFromFile(ctx context.Context, oldSyncID return "", "", fmt.Errorf("failed to commit transaction: %w", err) } committed = true - c.file.dbUpdated = true + c.file.dbUpdated.Store(true) return upsertsSyncID, deletionsSyncID, nil } diff --git a/pkg/dotc1z/c1file_close_cancel_test.go b/pkg/dotc1z/c1file_close_cancel_test.go index 58eab5d5e..dcd2126d2 100644 --- a/pkg/dotc1z/c1file_close_cancel_test.go +++ b/pkg/dotc1z/c1file_close_cancel_test.go @@ -138,7 +138,7 @@ func TestC1FileCloseReadOnlyButDirtyClosesRawDb(t *testing.T) { require.NoError(t, err) require.NotNil(t, f2.rawDb) // Force the readonly+dbUpdated cheap-path branch. - f2.dbUpdated = true + f2.dbUpdated.Store(true) err = f2.Close(openCtx) require.ErrorIs(t, err, ErrReadOnly) diff --git a/pkg/dotc1z/c1zstore/grant_store.go b/pkg/dotc1z/c1zstore/grant_store.go index 1e8dd0c03..c0caccc91 100644 --- a/pkg/dotc1z/c1zstore/grant_store.go +++ b/pkg/dotc1z/c1zstore/grant_store.go @@ -146,12 +146,12 @@ type GrantAnnotation struct { PrincipalResourceID string NeedsExpansion bool - // SourceScopeHash is the grant row's source-cache scope stamp ("" for + // SourceScopeKey is the grant row's source-cache scope stamp ("" for // unstamped rows, and always "" on engines without source-cache // support). The external-principal post-processing step writes its // transformed grants under the SOURCE grant's scope so a future // sync's replay of that scope carries them forward — the source // grant itself is deleted by that step, and a scope left holding // neither source nor transformed rows would silently replay empty. - SourceScopeHash string + SourceScopeKey string } diff --git a/pkg/dotc1z/clone_sync.go b/pkg/dotc1z/clone_sync.go index 9c34ed97f..e56c7250a 100644 --- a/pkg/dotc1z/clone_sync.go +++ b/pkg/dotc1z/clone_sync.go @@ -357,7 +357,7 @@ func (c *C1File) cloneCopy(ctx context.Context, outPath string, syncID string, s if err != nil { return err } - outFile.dbUpdated = true + outFile.dbUpdated.Store(true) outFile.outputFilePath = outPath err = outFile.Close(ctx) if err != nil { diff --git a/pkg/dotc1z/copy_isolate_sync.go b/pkg/dotc1z/copy_isolate_sync.go index 0872b5396..28eef769f 100644 --- a/pkg/dotc1z/copy_isolate_sync.go +++ b/pkg/dotc1z/copy_isolate_sync.go @@ -233,7 +233,7 @@ func (c *C1File) CopyIsolateSync(ctx context.Context, outPath string, syncID str // Step 4 — recompress the isolated copy to outPath. Mirrors cloneCopy's // finalize: dbUpdated + outputFilePath + Close() runs the // WAL-checkpoint -> close-raw-db -> saveC1z sequence. - copyFile.dbUpdated = true + copyFile.dbUpdated.Store(true) copyFile.outputFilePath = outPath finalized = true if err = copyFile.Close(ctx); err != nil { diff --git a/pkg/dotc1z/diff.go b/pkg/dotc1z/diff.go index 162001e40..c991b585a 100644 --- a/pkg/dotc1z/diff.go +++ b/pkg/dotc1z/diff.go @@ -55,7 +55,7 @@ func (c *C1File) GenerateSyncDiff(ctx context.Context, baseSyncID string, applie if err != nil { return "", err } - c.dbUpdated = true + c.dbUpdated.Store(true) } if err := c.endSyncRun(ctx, diffSyncID); err != nil { diff --git a/pkg/dotc1z/engine/pebble/adapter.go b/pkg/dotc1z/engine/pebble/adapter.go index a5cfd3665..3fb22f48d 100644 --- a/pkg/dotc1z/engine/pebble/adapter.go +++ b/pkg/dotc1z/engine/pebble/adapter.go @@ -417,17 +417,17 @@ func (a *Adapter) PutGrants(ctx context.Context, grants ...*v2.Grant) error { return ErrNoCurrentSync } records := translateGrants(syncID, grants) - stampSourceScope(ctx, records, func(r *v3.GrantRecord, s string) { r.SetSourceScopeHash(s) }) + stampSourceScope(ctx, records, func(r *v3.GrantRecord, s string) { r.SetSourceScopeKey(s) }) if err := a.engine.PutGrantRecords(ctx, records...); err != nil { return fmt.Errorf("PutGrants: %w", err) } return nil } -// stampSourceScope stamps the source-cache scope hash carried by ctx +// stampSourceScope stamps the source-cache scope key carried by ctx // (sourcecache.WithScope) onto freshly translated records. The syncer // sets the scope around a page's store writes when the page carried a -// SourceCacheScope annotation; everything else writes unstamped rows. +// SourceCacheRecord annotation; everything else writes unstamped rows. func stampSourceScope[T any](ctx context.Context, records []T, set func(T, string)) { scope := sourcecache.ScopeFromContext(ctx) if scope == "" { @@ -595,7 +595,7 @@ func (a *Adapter) PutResources(ctx context.Context, resources ...*v2.Resource) e } records = append(records, rec) } - stampSourceScope(ctx, records, func(r *v3.ResourceRecord, s string) { r.SetSourceScopeHash(s) }) + stampSourceScope(ctx, records, func(r *v3.ResourceRecord, s string) { r.SetSourceScopeKey(s) }) if err := a.engine.PutResourceRecords(ctx, records...); err != nil { return fmt.Errorf("PutResources: %w", err) } @@ -623,7 +623,7 @@ func (a *Adapter) PutEntitlements(ctx context.Context, entitlements ...*v2.Entit } records = append(records, rec) } - stampSourceScope(ctx, records, func(r *v3.EntitlementRecord, s string) { r.SetSourceScopeHash(s) }) + stampSourceScope(ctx, records, func(r *v3.EntitlementRecord, s string) { r.SetSourceScopeKey(s) }) if err := a.engine.PutEntitlementRecords(ctx, records...); err != nil { return fmt.Errorf("PutEntitlements: %w", err) } diff --git a/pkg/dotc1z/engine/pebble/adapter_grants_store.go b/pkg/dotc1z/engine/pebble/adapter_grants_store.go index 6cff1955b..f88453a58 100644 --- a/pkg/dotc1z/engine/pebble/adapter_grants_store.go +++ b/pkg/dotc1z/engine/pebble/adapter_grants_store.go @@ -223,7 +223,7 @@ func (g pebbleGrantStore) translateExpanded(syncID string, grants []*v2.Grant) [ // Same shape for the source-cache scope stamp: existing records get // their prior stamp restored in PutExpandedGrantRecords; brand-new // expander-derived rows are never part of a source scope. - newRec.SetSourceScopeHash("") + newRec.SetSourceScopeKey("") merged = append(merged, newRec) } return merged @@ -331,7 +331,7 @@ func (g pebbleGrantStore) ListWithAnnotationsPage(ctx context.Context, pageToken PrincipalResourceTypeID: princ.GetResourceTypeId(), PrincipalResourceID: princ.GetResourceId(), NeedsExpansion: rec.GetNeedsExpansion(), - SourceScopeHash: rec.GetSourceScopeHash(), + SourceScopeKey: rec.GetSourceScopeKey(), }) } return rows, next, nil @@ -377,7 +377,7 @@ func (g pebbleGrantStore) ListWithAnnotationsForResourcePage( PrincipalResourceTypeID: rec.GetPrincipal().GetResourceTypeId(), PrincipalResourceID: rec.GetPrincipal().GetResourceId(), NeedsExpansion: rec.GetNeedsExpansion(), - SourceScopeHash: rec.GetSourceScopeHash(), + SourceScopeKey: rec.GetSourceScopeKey(), }) } return rows, next, nil diff --git a/pkg/dotc1z/engine/pebble/entitlements.go b/pkg/dotc1z/engine/pebble/entitlements.go index 3cc37a557..401175571 100644 --- a/pkg/dotc1z/engine/pebble/entitlements.go +++ b/pkg/dotc1z/engine/pebble/entitlements.go @@ -80,7 +80,7 @@ func (e *Engine) PutEntitlementRecords(ctx context.Context, records ...*v3.Entit if err != nil { return err } - newScope := r.GetSourceScopeHash() + newScope := r.GetSourceScopeKey() if !skipGet { oldVal, closer, getErr := e.db.Get(key) switch { diff --git a/pkg/dotc1z/engine/pebble/grants.go b/pkg/dotc1z/engine/pebble/grants.go index 7d1430745..cacf44ef8 100644 --- a/pkg/dotc1z/engine/pebble/grants.go +++ b/pkg/dotc1z/engine/pebble/grants.go @@ -273,7 +273,7 @@ func (e *Engine) PutExpandedGrantRecords(ctx context.Context, records []*v3.Gran // direct grants to bake in Sources, and clobbering the // stamp here would silently drop every expander-touched // grant from the next sync's replay of its scope. - r.SetSourceScopeHash(prior.GetSourceScopeHash()) + r.SetSourceScopeKey(prior.GetSourceScopeKey()) var err error idxScratch, err = e.deleteGrantIndexesScratch(idxBatch, ext, oldVal, idxScratch) if err != nil { @@ -759,7 +759,7 @@ func (e *Engine) putSynthesizedGrantContributionsBatch(ctx context.Context, reco } // sources (field 9) is the highest field this record carries, so // appending it after the base marshal matches the deterministic - // byte order. (source_scope_hash is field 10, but synthesized + // byte order. (source_scope_key is field 10, but synthesized // grants never carry a scope stamp — fillSynthGrantRecord leaves // it unset — so field 9 stays last on the wire.) val, srcScratch = appendGrantSourcesWire(val, srcScratch, rec.sources) @@ -1017,7 +1017,7 @@ func grantIndexKeys(r *v3.GrantRecord) [][]byte { if r.GetNeedsExpansion() { keys = append(keys, encodeGrantByNeedsExpansionIdentityIndexKey(id)) } - if sh := r.GetSourceScopeHash(); sh != "" { + if sh := r.GetSourceScopeKey(); sh != "" { keys = append(keys, encodeGrantBySourceScopeIndexKey(sh, id)) } return keys @@ -1067,7 +1067,7 @@ func (e *Engine) writeGrantIndexesScratch(batch *pebble.Batch, r *v3.GrantRecord if err != nil { return scratch, err } - return e.writeGrantIndexesForIdentityScratch(batch, id, r.GetNeedsExpansion(), r.GetSourceScopeHash(), scratch) + return e.writeGrantIndexesForIdentityScratch(batch, id, r.GetNeedsExpansion(), r.GetSourceScopeKey(), scratch) } // markDeferredIdxPending arms the deferred by_principal rebuild, durably. @@ -1113,7 +1113,7 @@ func (e *Engine) clearDeferredIdxPending() error { // by_needs_expansion and by_source_scope are written inline: both share the // entitlement-first ordering of the primary keyspace, so their writes stay // sorted. -func (e *Engine) writeGrantIndexesForIdentityScratch(batch *pebble.Batch, id grantIdentity, needsExpansion bool, sourceScopeHash string, scratch []byte) ([]byte, error) { +func (e *Engine) writeGrantIndexesForIdentityScratch(batch *pebble.Batch, id grantIdentity, needsExpansion bool, sourceScopeKey string, scratch []byte) ([]byte, error) { if err := e.markDeferredIdxPending(); err != nil { return scratch, err } @@ -1128,8 +1128,8 @@ func (e *Engine) writeGrantIndexesForIdentityScratch(batch *pebble.Batch, id gra if err := e.stageGrantDigestInvalidation(batch, id.entitlement); err != nil { return scratch, err } - if sourceScopeHash != "" { - scratch = appendGrantBySourceScopeIndexKey(scratch[:0], sourceScopeHash, id) + if sourceScopeKey != "" { + scratch = appendGrantBySourceScopeIndexKey(scratch[:0], sourceScopeKey, id) if err := batch.Set(scratch, nil, nil); err != nil { return scratch, err } @@ -1146,7 +1146,7 @@ func (e *Engine) writeGrantIndexesForIdentityScratch(batch *pebble.Batch, id gra // which also clears any stale entries an overwrite would have left. Returns // the (possibly grown) scratch buffer. func (e *Engine) deleteGrantIndexesScratch(batch *pebble.Batch, externalID string, value, scratch []byte) ([]byte, error) { - entRT, entRID, entID, principalRT, principalID, _, sourceScopeHash, err := scanGrantIndexFieldsRaw(value) + entRT, entRID, entID, principalRT, principalID, _, sourceScopeKey, err := scanGrantIndexFieldsRaw(value) if err != nil { return scratch, err } @@ -1167,8 +1167,8 @@ func (e *Engine) deleteGrantIndexesScratch(batch *pebble.Batch, externalID strin if err := e.stageGrantDigestInvalidation(batch, id.entitlement); err != nil { return scratch, err } - if sourceScopeHash != "" { - scratch = appendGrantBySourceScopeIndexKey(scratch[:0], sourceScopeHash, id) + if sourceScopeKey != "" { + scratch = appendGrantBySourceScopeIndexKey(scratch[:0], sourceScopeKey, id) if err := batch.Delete(scratch, nil); err != nil { return scratch, err } diff --git a/pkg/dotc1z/engine/pebble/grants_synth_encode.go b/pkg/dotc1z/engine/pebble/grants_synth_encode.go index a818dcf7e..19c67eec5 100644 --- a/pkg/dotc1z/engine/pebble/grants_synth_encode.go +++ b/pkg/dotc1z/engine/pebble/grants_synth_encode.go @@ -31,7 +31,7 @@ var expandedGrantImmutableAnnotations = []*anypb.Any{expandedGrantImmutableAnnot // appendGrantSourcesWire). Expansion/NeedsExpansion stay zero for // synthesized grants. // -// SourceScopeHash is cleared unconditionally, for two reasons: synthesized +// SourceScopeKey is cleared unconditionally, for two reasons: synthesized // (expander-derived) grants are never part of a source-cache scope, and — // load-bearing for appendGrantSourcesWire — an empty proto3 scalar emits no // bytes, which keeps sources (field 9) the highest field on the wire so the @@ -42,7 +42,7 @@ func fillSynthGrantRecord(r *v3.GrantRecord, rec *synthesizedGrantRecord, now *t r.SetPrincipal(rec.principal) r.SetAnnotations(expandedGrantImmutableAnnotations) r.SetDiscoveredAt(now) - r.SetSourceScopeHash("") + r.SetSourceScopeKey("") } // grantExternalIDFieldTag is the wire tag for GrantRecord.external_id (2). diff --git a/pkg/dotc1z/engine/pebble/grants_synth_encode_test.go b/pkg/dotc1z/engine/pebble/grants_synth_encode_test.go index bcb99caba..3f4f0b9a3 100644 --- a/pkg/dotc1z/engine/pebble/grants_synth_encode_test.go +++ b/pkg/dotc1z/engine/pebble/grants_synth_encode_test.go @@ -45,19 +45,19 @@ func TestGrantSourcesWireSchemaPin(t *testing.T) { maxNum = n } } - // source_scope_hash (10) sorts above sources (9), which would break the + // source_scope_key (10) sorts above sources (9), which would break the // append-after-base-marshal canonical order IF it were ever populated on // a synthesized record. It never is: fillSynthGrantRecord clears it // unconditionally, and an empty proto3 scalar emits no bytes, so sources // stays the highest field on the wire. Any field beyond 10 (or a // non-scalar 10) requires revisiting appendGrantSourcesWire. require.Equal(t, protoreflect.FieldNumber(10), maxNum, - "GrantRecord gained a field beyond source_scope_hash(10): decide whether appendGrantSourcesWire's append-last invariant still holds") + "GrantRecord gained a field beyond source_scope_key(10): decide whether appendGrantSourcesWire's append-last invariant still holds") ssh := fields.ByNumber(10) require.NotNil(t, ssh) - require.Equal(t, protoreflect.Name("source_scope_hash"), ssh.Name(), "field 10 is no longer source_scope_hash") + require.Equal(t, protoreflect.Name("source_scope_key"), ssh.Name(), "field 10 is no longer source_scope_key") require.Equal(t, protoreflect.StringKind, ssh.Kind(), - "source_scope_hash is no longer a string scalar; the empty-value-emits-nothing guarantee appendGrantSourcesWire relies on may not hold") + "source_scope_key is no longer a string scalar; the empty-value-emits-nothing guarantee appendGrantSourcesWire relies on may not hold") src := fields.ByName("sources") require.NotNil(t, src, "GrantRecord.sources missing") @@ -249,12 +249,12 @@ func TestFillSynthGrantRecordSchemaPin(t *testing.T) { "7:needs_expansion:bool", "8:annotations:message", "9:sources:message", - // source_scope_hash is part of the synthesized write path only as an + // source_scope_key is part of the synthesized write path only as an // unconditional CLEAR (expander-derived grants are never part of a // source-cache scope); fillSynthGrantRecord sets it to "" to honor // the reused-struct invariant and appendGrantSourcesWire's // highest-populated-field assumption. - "10:source_scope_hash:string", + "10:source_scope_key:string", }, protoFields, "GrantRecord shape changed: decide whether fillSynthGrantRecord must set the new/changed field") recType := reflect.TypeOf(synthesizedGrantRecord{}) diff --git a/pkg/dotc1z/engine/pebble/if_newer.go b/pkg/dotc1z/engine/pebble/if_newer.go index b4bfc9ce0..37466e376 100644 --- a/pkg/dotc1z/engine/pebble/if_newer.go +++ b/pkg/dotc1z/engine/pebble/if_newer.go @@ -198,7 +198,7 @@ func (e *Engine) PutEntitlementRecordsIfNewer(ctx context.Context, records ...*v if scanErr != nil { return scanErr } - if oldScope != "" && oldScope != r.GetSourceScopeHash() { + if oldScope != "" && oldScope != r.GetSourceScopeKey() { if err := batch.Delete(encodeEntitlementBySourceScopeIndexKey(oldScope, id), nil); err != nil { return err } @@ -214,7 +214,7 @@ func (e *Engine) PutEntitlementRecordsIfNewer(ctx context.Context, records ...*v if err := batch.Set(key, val, nil); err != nil { return err } - if sh := r.GetSourceScopeHash(); sh != "" { + if sh := r.GetSourceScopeKey(); sh != "" { if err := batch.Set(encodeEntitlementBySourceScopeIndexKey(sh, id), nil, nil); err != nil { return err } diff --git a/pkg/dotc1z/engine/pebble/keys.go b/pkg/dotc1z/engine/pebble/keys.go index c71800ef8..f9d1e92a9 100644 --- a/pkg/dotc1z/engine/pebble/keys.go +++ b/pkg/dotc1z/engine/pebble/keys.go @@ -65,7 +65,7 @@ const ( typeSession byte = 0x09 typeDigest byte = 0x0A // typeSourceCache holds one SourceCacheEntryRecord per - // (row_kind, scope_hash) — the source-cache replay manifest (see + // (row_kind, scope_key) — the source-cache replay manifest (see // proto/c1/connector/v2/annotation_source_cache.proto). Placed // after typeSession so the clone path's counter/session excise // span ([typeCounter, upperBoundOf(typeSession))) leaves it in the @@ -96,7 +96,7 @@ const ( // pass, never maintained inline. See digest.go and grant_digest.go. idxGrantByEntitlementPrincipalHash byte = 0x08 // by_source_scope families: partial indexes over records whose - // source_scope_hash is non-empty (source-cache replay). Tails are + // source_scope_key is non-empty (source-cache replay). Tails are // identity tuples per the keys.go convention, so replay derives // each primary key from the index key without touching the record. idxGrantBySourceScope byte = 0x09 @@ -418,27 +418,27 @@ func DigestUpperBound() []byte { // --- Source-cache (by_source_scope indexes + entry keyspace) --- // encodeGrantBySourceScopeIndexKey is the partial by_source_scope index -// over grants whose SourceScopeHash is non-empty: +// over grants whose SourceScopeKey is non-empty: // // v3 | typeIndex | idxGrantBySourceScope | 0x00 | -// scope_hash | 0x00 | +// scope_key | 0x00 | // ent_rt | 0x00 | ent_rid | 0x00 | ent_flag | 0x00 | ent_tail | 0x00 | // principal_rt | 0x00 | principal_id // -// The tail after scope_hash is the grant identity tuple — byte-identical +// The tail after scope_key is the grant identity tuple — byte-identical // to the grant primary key's tail — so replay derives the primary key from // the index key alone. Paired with encodeGrantBySourceScopePrefix // (by-value prefix, with trailing sep). -func encodeGrantBySourceScopeIndexKey(scopeHash string, id grantIdentity) []byte { - return appendGrantBySourceScopeIndexKey(make([]byte, 0, 128), scopeHash, id) +func encodeGrantBySourceScopeIndexKey(scopeKey string, id grantIdentity) []byte { + return appendGrantBySourceScopeIndexKey(make([]byte, 0, 128), scopeKey, id) } -func appendGrantBySourceScopeIndexKey(dst []byte, scopeHash string, id grantIdentity) []byte { +func appendGrantBySourceScopeIndexKey(dst []byte, scopeKey string, id grantIdentity) []byte { dst = append(dst, versionV3, typeIndex, idxGrantBySourceScope) dst = codec.AppendTupleSeparator(dst) return codec.AppendTupleStrings( dst, - scopeHash, + scopeKey, id.entitlement.resourceTypeID, id.entitlement.resourceID, id.entitlement.flagComponent(), @@ -449,78 +449,78 @@ func appendGrantBySourceScopeIndexKey(dst []byte, scopeHash string, id grantIden } // encodeGrantBySourceScopePrefix is the by-value prefix for "all grants -// stamped with this scope hash". Trailing sep is load-bearing — see the +// stamped with this scope key". Trailing sep is load-bearing — see the // keys.go convention doc. -func encodeGrantBySourceScopePrefix(scopeHash string) []byte { - buf := make([]byte, 0, 32+len(scopeHash)) +func encodeGrantBySourceScopePrefix(scopeKey string) []byte { + buf := make([]byte, 0, 32+len(scopeKey)) buf = append(buf, versionV3, typeIndex, idxGrantBySourceScope) buf = codec.AppendTupleSeparator(buf) - buf = codec.AppendTupleStrings(buf, scopeHash) + buf = codec.AppendTupleStrings(buf, scopeKey) return codec.AppendTupleSeparator(buf) } // encodeEntitlementBySourceScopeIndexKey: // // v3 | typeIndex | idxEntitlementBySourceScope | 0x00 | -// scope_hash | 0x00 | rt | 0x00 | rid | 0x00 | flag | 0x00 | tail +// scope_key | 0x00 | rt | 0x00 | rid | 0x00 | flag | 0x00 | tail // // Tail is the entitlement identity tuple, byte-identical to the // entitlement primary key's tail. Paired with // encodeEntitlementBySourceScopePrefix. -func encodeEntitlementBySourceScopeIndexKey(scopeHash string, id entitlementIdentity) []byte { - return appendEntitlementBySourceScopeIndexKey(make([]byte, 0, 128), scopeHash, id) +func encodeEntitlementBySourceScopeIndexKey(scopeKey string, id entitlementIdentity) []byte { + return appendEntitlementBySourceScopeIndexKey(make([]byte, 0, 128), scopeKey, id) } -func appendEntitlementBySourceScopeIndexKey(dst []byte, scopeHash string, id entitlementIdentity) []byte { +func appendEntitlementBySourceScopeIndexKey(dst []byte, scopeKey string, id entitlementIdentity) []byte { dst = append(dst, versionV3, typeIndex, idxEntitlementBySourceScope) dst = codec.AppendTupleSeparator(dst) - return codec.AppendTupleStrings(dst, scopeHash, id.resourceTypeID, id.resourceID, id.flagComponent(), id.tail) + return codec.AppendTupleStrings(dst, scopeKey, id.resourceTypeID, id.resourceID, id.flagComponent(), id.tail) } -func encodeEntitlementBySourceScopePrefix(scopeHash string) []byte { - buf := make([]byte, 0, 32+len(scopeHash)) +func encodeEntitlementBySourceScopePrefix(scopeKey string) []byte { + buf := make([]byte, 0, 32+len(scopeKey)) buf = append(buf, versionV3, typeIndex, idxEntitlementBySourceScope) buf = codec.AppendTupleSeparator(buf) - buf = codec.AppendTupleStrings(buf, scopeHash) + buf = codec.AppendTupleStrings(buf, scopeKey) return codec.AppendTupleSeparator(buf) } // encodeResourceBySourceScopeIndexKey: // // v3 | typeIndex | idxResourceBySourceScope | 0x00 | -// scope_hash | 0x00 | resource_type_id | 0x00 | resource_id +// scope_key | 0x00 | resource_type_id | 0x00 | resource_id // // Tail is the resource primary tuple. Paired with // encodeResourceBySourceScopePrefix. -func encodeResourceBySourceScopeIndexKey(scopeHash, resourceTypeID, resourceID string) []byte { - return appendResourceBySourceScopeIndexKey(make([]byte, 0, 96), scopeHash, resourceTypeID, resourceID) +func encodeResourceBySourceScopeIndexKey(scopeKey, resourceTypeID, resourceID string) []byte { + return appendResourceBySourceScopeIndexKey(make([]byte, 0, 96), scopeKey, resourceTypeID, resourceID) } -func appendResourceBySourceScopeIndexKey(dst []byte, scopeHash, resourceTypeID, resourceID string) []byte { +func appendResourceBySourceScopeIndexKey(dst []byte, scopeKey, resourceTypeID, resourceID string) []byte { dst = append(dst, versionV3, typeIndex, idxResourceBySourceScope) dst = codec.AppendTupleSeparator(dst) - return codec.AppendTupleStrings(dst, scopeHash, resourceTypeID, resourceID) + return codec.AppendTupleStrings(dst, scopeKey, resourceTypeID, resourceID) } -func encodeResourceBySourceScopePrefix(scopeHash string) []byte { - buf := make([]byte, 0, 32+len(scopeHash)) +func encodeResourceBySourceScopePrefix(scopeKey string) []byte { + buf := make([]byte, 0, 32+len(scopeKey)) buf = append(buf, versionV3, typeIndex, idxResourceBySourceScope) buf = codec.AppendTupleSeparator(buf) - buf = codec.AppendTupleStrings(buf, scopeHash) + buf = codec.AppendTupleStrings(buf, scopeKey) return codec.AppendTupleSeparator(buf) } // encodeSourceCacheEntryKey is the primary key for one source-cache // manifest entry: // -// v3 | typeSourceCache | 0x00 | row_kind | 0x00 | scope_hash +// v3 | typeSourceCache | 0x00 | row_kind | 0x00 | scope_key // // Paired with encodeSourceCachePrefix (by-type prefix). -func encodeSourceCacheEntryKey(rowKind, scopeHash string) []byte { - buf := make([]byte, 0, 32+len(rowKind)+len(scopeHash)) +func encodeSourceCacheEntryKey(rowKind, scopeKey string) []byte { + buf := make([]byte, 0, 32+len(rowKind)+len(scopeKey)) buf = append(buf, versionV3, typeSourceCache) buf = codec.AppendTupleSeparator(buf) - return codec.AppendTupleStrings(buf, rowKind, scopeHash) + return codec.AppendTupleStrings(buf, rowKind, scopeKey) } // encodeSourceCachePrefix is the by-type prefix for all source-cache diff --git a/pkg/dotc1z/engine/pebble/raw_records.go b/pkg/dotc1z/engine/pebble/raw_records.go index 1881a4a9f..307172cd8 100644 --- a/pkg/dotc1z/engine/pebble/raw_records.go +++ b/pkg/dotc1z/engine/pebble/raw_records.go @@ -114,7 +114,7 @@ func rawTimestampNanos(value []byte) (int64, error) { } func (e *Engine) deleteResourceIndexesRaw(batch *pebble.Batch, resourceTypeID string, resourceID string, value []byte) error { - parentRT, parentID, sourceScopeHash, err := scanResourceIndexFieldsRaw(value) + parentRT, parentID, sourceScopeKey, err := scanResourceIndexFieldsRaw(value) if err != nil { return err } @@ -123,8 +123,8 @@ func (e *Engine) deleteResourceIndexesRaw(batch *pebble.Batch, resourceTypeID st return err } } - if sourceScopeHash != "" { - if err := batch.Delete(encodeResourceBySourceScopeIndexKey(sourceScopeHash, resourceTypeID, resourceID), nil); err != nil { + if sourceScopeKey != "" { + if err := batch.Delete(encodeResourceBySourceScopeIndexKey(sourceScopeKey, resourceTypeID, resourceID), nil); err != nil { return err } } @@ -132,7 +132,7 @@ func (e *Engine) deleteResourceIndexesRaw(batch *pebble.Batch, resourceTypeID st } func (e *Engine) deleteGrantIndexesRaw(batch *pebble.Batch, externalID string, value []byte) error { - entRT, entRID, entID, principalRT, principalID, _, sourceScopeHash, err := scanGrantIndexFieldsRaw(value) + entRT, entRID, entID, principalRT, principalID, _, sourceScopeKey, err := scanGrantIndexFieldsRaw(value) if err != nil { return err } @@ -153,8 +153,8 @@ func (e *Engine) deleteGrantIndexesRaw(batch *pebble.Batch, externalID string, v if err := e.stageGrantDigestInvalidation(batch, id.entitlement); err != nil { return err } - if sourceScopeHash != "" { - if err := batch.Delete(encodeGrantBySourceScopeIndexKey(sourceScopeHash, id), nil); err != nil { + if sourceScopeKey != "" { + if err := batch.Delete(encodeGrantBySourceScopeIndexKey(sourceScopeKey, id), nil); err != nil { return err } } @@ -336,10 +336,10 @@ func scanEntitlementResourceTypeRaw(value []byte) ([]byte, error) { // approximating proto merge semantics. Values written by this SDK carry // at most one occurrence, so this only matters for foreign writers. // scanResourceIndexFieldsRaw extracts the parent ref (field 6) and -// source_scope_hash (field 9) from a marshaled ResourceRecord — the +// source_scope_key (field 9) from a marshaled ResourceRecord — the // fields that key resource secondary indexes. func scanResourceIndexFieldsRaw(value []byte) (string, string, string, error) { - var rt, id, sourceScopeHash string + var rt, id, sourceScopeKey string for len(value) > 0 { num, typ, n := protowire.ConsumeTag(value) if n < 0 { @@ -363,13 +363,13 @@ func scanResourceIndexFieldsRaw(value []byte) (string, string, string, error) { value = value[n:] case 9: if typ != protowire.BytesType { - return "", "", "", fmt.Errorf("raw record: resource source_scope_hash has wire type %v", typ) + return "", "", "", fmt.Errorf("raw record: resource source_scope_key has wire type %v", typ) } s, n := protowire.ConsumeBytes(value) if n < 0 { return "", "", "", protowire.ParseError(n) } - sourceScopeHash = string(s) + sourceScopeKey = string(s) value = value[n:] default: n = protowire.ConsumeFieldValue(num, typ, value) @@ -379,14 +379,14 @@ func scanResourceIndexFieldsRaw(value []byte) (string, string, string, error) { value = value[n:] } } - return rt, id, sourceScopeHash, nil + return rt, id, sourceScopeKey, nil } -// scanEntitlementSourceScopeRaw extracts source_scope_hash (field 11) +// scanEntitlementSourceScopeRaw extracts source_scope_key (field 11) // from a marshaled EntitlementRecord. Keys the entitlement // by_source_scope index — the only entitlement secondary index. func scanEntitlementSourceScopeRaw(value []byte) (string, error) { - var sourceScopeHash string + var sourceScopeKey string for len(value) > 0 { num, typ, n := protowire.ConsumeTag(value) if n < 0 { @@ -402,16 +402,16 @@ func scanEntitlementSourceScopeRaw(value []byte) (string, error) { continue } if typ != protowire.BytesType { - return "", fmt.Errorf("raw record: entitlement source_scope_hash has wire type %v", typ) + return "", fmt.Errorf("raw record: entitlement source_scope_key has wire type %v", typ) } s, n := protowire.ConsumeBytes(value) if n < 0 { return "", protowire.ParseError(n) } - sourceScopeHash = string(s) + sourceScopeKey = string(s) value = value[n:] } - return sourceScopeHash, nil + return sourceScopeKey, nil } func scanEntitlementResourceRaw(value []byte) (string, string, error) { @@ -492,7 +492,7 @@ func scanEntitlementIdentityFieldsRaw(value []byte) (string, string, string, err } func scanGrantIndexFieldsRaw(value []byte) (string, string, string, string, string, bool, string, error) { - var entRT, entRID, entID, principalRT, principalID, sourceScopeHash string + var entRT, entRID, entID, principalRT, principalID, sourceScopeKey string var needsExpansion bool for len(value) > 0 { num, typ, n := protowire.ConsumeTag(value) @@ -541,13 +541,13 @@ func scanGrantIndexFieldsRaw(value []byte) (string, string, string, string, stri value = value[n:] case 10: if typ != protowire.BytesType { - return "", "", "", "", "", false, "", fmt.Errorf("raw record: grant source_scope_hash has wire type %v", typ) + return "", "", "", "", "", false, "", fmt.Errorf("raw record: grant source_scope_key has wire type %v", typ) } s, n := protowire.ConsumeBytes(value) if n < 0 { return "", "", "", "", "", false, "", protowire.ParseError(n) } - sourceScopeHash = string(s) + sourceScopeKey = string(s) value = value[n:] default: n = protowire.ConsumeFieldValue(num, typ, value) @@ -557,7 +557,7 @@ func scanGrantIndexFieldsRaw(value []byte) (string, string, string, string, stri value = value[n:] } } - return entRT, entRID, entID, principalRT, principalID, needsExpansion, sourceScopeHash, nil + return entRT, entRID, entID, principalRT, principalID, needsExpansion, sourceScopeKey, nil } // scanGrantNeedsExpansionRaw extracts only the needs_expansion flag diff --git a/pkg/dotc1z/engine/pebble/resources.go b/pkg/dotc1z/engine/pebble/resources.go index 0f0d820cb..582e5ce35 100644 --- a/pkg/dotc1z/engine/pebble/resources.go +++ b/pkg/dotc1z/engine/pebble/resources.go @@ -160,7 +160,7 @@ func (e *Engine) writeResourceIndexes(batch *pebble.Batch, r *v3.ResourceRecord) return err } } - if sh := r.GetSourceScopeHash(); sh != "" { + if sh := r.GetSourceScopeKey(); sh != "" { k := encodeResourceBySourceScopeIndexKey(sh, r.GetResourceTypeId(), r.GetResourceId()) if err := batch.Set(k, nil, nil); err != nil { return err diff --git a/pkg/dotc1z/engine/pebble/source_cache.go b/pkg/dotc1z/engine/pebble/source_cache.go index 961562dc5..e33d1ef5c 100644 --- a/pkg/dotc1z/engine/pebble/source_cache.go +++ b/pkg/dotc1z/engine/pebble/source_cache.go @@ -16,9 +16,9 @@ import ( // Source-cache replay, engine side. // // The typeSourceCache keyspace holds one SourceCacheEntryRecord per -// (row_kind, scope_hash): the opaque upstream validator (etag / delta +// (row_kind, scope_key): the opaque upstream validator (etag / delta // token) the sync recorded for that scope. Rows produced under a scope -// are stamped with source_scope_hash and indexed under the +// are stamped with source_scope_key and indexed under the // by_source_scope families, whose tails are identity tuples — so a // replay derives every primary key from the index key alone and copies // raw values across files without a proto unmarshal. @@ -76,7 +76,7 @@ type SourceCacheReplayResult struct { } // SourceCacheManifestSnapshot returns every manifest entry as -// "row_kind\x00scope_hash" -> etag. Inspection surface: the +// "row_kind\x00scope_key" -> etag. Inspection surface: the // replay-equivalence harness compares a warm sync's manifest against a // cold sync's (they must be identical — same scopes, same validators), // catching phantom or missing entries that are invisible through the v2 @@ -99,17 +99,17 @@ func (e *Engine) SourceCacheManifestSnapshot(ctx context.Context) (map[string]st if err := unmarshalRecord(iter.Value(), rec); err != nil { return nil, fmt.Errorf("SourceCacheManifestSnapshot: unmarshal: %w", err) } - out[rec.GetRowKind()+"\x00"+rec.GetScopeHash()] = rec.GetEtag() + out[rec.GetRowKind()+"\x00"+rec.GetScopeKey()] = rec.GetCacheValidator() } return out, iter.Error() } -// SourceCacheScopeIndexSnapshot returns, per row kind, scope_hash -> the +// SourceScopeIndexSnapshot returns, per row kind, scope_key -> the // number of by_source_scope index entries. Same inspection rationale as // SourceCacheManifestSnapshot: stamp/index divergence between a warm and // a cold sync is invisible to v2 reads until a later sync replays from // the damaged file. -func (e *Engine) SourceCacheScopeIndexSnapshot(ctx context.Context) (map[string]map[string]int, error) { +func (e *Engine) SourceScopeIndexSnapshot(ctx context.Context) (map[string]map[string]int, error) { out := map[string]map[string]int{} families := []struct { kind string @@ -138,7 +138,7 @@ func (e *Engine) SourceCacheScopeIndexSnapshot(ctx context.Context) (map[string] scope, _, ok := codec.DecodeTupleStringAlias(tail, 0) if !ok { iter.Close() - return nil, fmt.Errorf("SourceCacheScopeIndexSnapshot: malformed %s index key %x", fam.kind, iter.Key()) + return nil, fmt.Errorf("SourceScopeIndexSnapshot: malformed %s index key %x", fam.kind, iter.Key()) } counts[string(scope)]++ } @@ -169,18 +169,18 @@ func (e *Engine) ClearSourceCacheEntries(ctx context.Context) error { }) } -// PutSourceCacheEntry writes the manifest entry for (rowKind, scopeHash). +// PutSourceCacheEntry writes the manifest entry for (rowKind, scopeKey). // Zero-row scopes still get entries — the validator must survive to the // next sync even when the scope produced no rows. -func (e *Engine) PutSourceCacheEntry(ctx context.Context, rowKind, scopeHash, etag string) error { +func (e *Engine) PutSourceCacheEntry(ctx context.Context, rowKind, scopeKey, etag string) error { return e.withWrite(func() error { if err := e.requireCurrentSync(); err != nil { return err } rec := &v3.SourceCacheEntryRecord{} rec.SetRowKind(rowKind) - rec.SetScopeHash(scopeHash) - rec.SetEtag(etag) + rec.SetScopeKey(scopeKey) + rec.SetCacheValidator(etag) rec.SetDiscoveredAt(timestamppb.Now()) val, err := marshalRecord(rec) if err != nil { @@ -190,14 +190,14 @@ func (e *Engine) PutSourceCacheEntry(ctx context.Context, rowKind, scopeHash, et if e.IsFreshSync() { opts = pebble.NoSync } - return e.db.Set(encodeSourceCacheEntryKey(rowKind, scopeHash), val, opts) + return e.db.Set(encodeSourceCacheEntryKey(rowKind, scopeKey), val, opts) }) } -// GetSourceCacheEntry returns the manifest entry for (rowKind, scopeHash), +// GetSourceCacheEntry returns the manifest entry for (rowKind, scopeKey), // or pebble.ErrNotFound. -func (e *Engine) GetSourceCacheEntry(ctx context.Context, rowKind, scopeHash string) (*v3.SourceCacheEntryRecord, error) { - val, closer, err := e.db.Get(encodeSourceCacheEntryKey(rowKind, scopeHash)) +func (e *Engine) GetSourceCacheEntry(ctx context.Context, rowKind, scopeKey string) (*v3.SourceCacheEntryRecord, error) { + val, closer, err := e.db.Get(encodeSourceCacheEntryKey(rowKind, scopeKey)) if err != nil { return nil, err } @@ -218,7 +218,7 @@ func (e *Engine) GetSourceCacheEntry(ctx context.Context, rowKind, scopeHash str // Consequence, by design: a grant stored under a connector-CUSTOM id (one // that isn't the SDK concat shape) is unreachable here and the delete // no-ops. Connectors with custom grant ids must use principal-scoped -// tombstones (SourceCacheScope.deleted_principal_ids) instead — documented +// tombstones (SourceCacheRecord.deleted_principal_ids) instead — documented // in the annotation proto. func (e *Engine) DeleteGrantRecordBounded(ctx context.Context, externalID string) error { return e.withWrite(func() error { @@ -234,9 +234,9 @@ func (e *Engine) DeleteGrantRecordBounded(ctx context.Context, externalID string } // DeleteGrantsByPrincipalsInScope deletes every grant row in the CURRENT -// store stamped with scopeHash whose principal id is in principalIDs — +// store stamped with scopeKey whose principal id is in principalIDs — // the engine side of principal-scoped delta tombstones -// (SourceCacheScope.deleted_principal_ids). +// (SourceCacheRecord.deleted_principal_ids). // // One prefix scan of the scope's by_source_scope index resolves // everything: the index tail IS the grant identity, so the primary key @@ -249,11 +249,11 @@ func (e *Engine) DeleteGrantRecordBounded(ctx context.Context, externalID string // // Complexity: O(scope size) tuple-walks per call regardless of tombstone // count — callers batch a page's tombstones into one call. -func (e *Engine) DeleteGrantsByPrincipalsInScope(ctx context.Context, scopeHash string, principalIDs map[string]struct{}) (int64, error) { +func (e *Engine) DeleteGrantsByPrincipalsInScope(ctx context.Context, scopeKey string, principalIDs map[string]struct{}) (int64, error) { if len(principalIDs) == 0 { return 0, nil } - prefix := encodeGrantBySourceScopePrefix(scopeHash) + prefix := encodeGrantBySourceScopePrefix(scopeKey) var deleted int64 err := e.withWrite(func() error { @@ -328,17 +328,17 @@ func (e *Engine) DeleteGrantsByPrincipalsInScope(ctx context.Context, scopeHash } // DeleteGrantsByExternalIDsInScope deletes every grant row in the CURRENT -// store stamped with scopeHash whose STORED grant id (external id, which +// store stamped with scopeKey whose STORED grant id (external id, which // may be a connector-custom shape) is in ids. One scan of the scope's // index, loading each candidate's primary row to compare the stored id — // bounded by the scope's row count, never the whole keyspace. This is the // tombstone path for connectors with custom grant ids whose scopes span // multiple resources (so principal-scoped deletes would over-delete). -func (e *Engine) DeleteGrantsByExternalIDsInScope(ctx context.Context, scopeHash string, ids map[string]struct{}) (int64, error) { +func (e *Engine) DeleteGrantsByExternalIDsInScope(ctx context.Context, scopeKey string, ids map[string]struct{}) (int64, error) { if len(ids) == 0 { return 0, nil } - prefix := encodeGrantBySourceScopePrefix(scopeHash) + prefix := encodeGrantBySourceScopePrefix(scopeKey) var deleted int64 err := e.withWrite(func() error { @@ -422,13 +422,13 @@ func (e *Engine) DeleteGrantsByExternalIDsInScope(ctx context.Context, scopeHash } // DeleteResourcesByIDsInScope deletes every resource row in the CURRENT -// store stamped with scopeHash whose resource id is in resourceIDs (any +// store stamped with scopeKey whose resource id is in resourceIDs (any // resource type) — principal-scoped tombstones for RowKindResources. -func (e *Engine) DeleteResourcesByIDsInScope(ctx context.Context, scopeHash string, resourceIDs map[string]struct{}) (int64, error) { +func (e *Engine) DeleteResourcesByIDsInScope(ctx context.Context, scopeKey string, resourceIDs map[string]struct{}) (int64, error) { if len(resourceIDs) == 0 { return 0, nil } - prefix := encodeResourceBySourceScopePrefix(scopeHash) + prefix := encodeResourceBySourceScopePrefix(scopeKey) var deleted int64 err := e.withWrite(func() error { @@ -488,7 +488,7 @@ func (e *Engine) DeleteResourcesByIDsInScope(ctx context.Context, scopeHash stri return err } // deleteResourceIndexesRaw already covered the scope entry - // (source_scope_hash is in the value), but delete the iterated + // (source_scope_key is in the value), but delete the iterated // key too in case the value read missed (orphan entry). if err := batch.Delete(key, nil); err != nil { return err @@ -593,14 +593,14 @@ func replayPrimaryFromIndexKey(indexKey, indexPrefix []byte, primaryHeader [2]by return append(key, tail...), nil } -// ReplaySourceCacheGrants copies every grant stamped with scopeHash from +// ReplaySourceCacheGrants copies every grant stamped with scopeKey from // prev into the receiver: raw primary copy plus index synthesis from the // raw value (principal, needs_expansion, source-scope families). Mirrors // PutGrantRecords' read-before-write index cleanup when the receiver // already holds a record at the same identity. -func (e *Engine) ReplaySourceCacheGrants(ctx context.Context, prev *Engine, scopeHash string) (SourceCacheReplayResult, error) { +func (e *Engine) ReplaySourceCacheGrants(ctx context.Context, prev *Engine, scopeKey string) (SourceCacheReplayResult, error) { var res SourceCacheReplayResult - prefix := encodeGrantBySourceScopePrefix(scopeHash) + prefix := encodeGrantBySourceScopePrefix(scopeKey) primaryHeader := [2]byte{versionV3, typeGrant} err := e.withWrite(func() error { @@ -683,7 +683,7 @@ func (e *Engine) ReplaySourceCacheGrants(ctx context.Context, prev *Engine, scop // compaction predating the source-cache bucket plans, or an // in-sync same-identity rewrite under a different scope. Copying // it would inject rows upstream never returned for this scope. - if srcScope != scopeHash { + if srcScope != scopeKey { closer.Close() res.StaleSkipped++ continue @@ -859,10 +859,10 @@ func (e *Engine) replayRelatedResourceRaw(batch *pebble.Batch, prev *Engine, pri } // ReplaySourceCacheEntitlements copies every entitlement stamped with -// scopeHash from prev into the receiver. -func (e *Engine) ReplaySourceCacheEntitlements(ctx context.Context, prev *Engine, scopeHash string) (SourceCacheReplayResult, error) { +// scopeKey from prev into the receiver. +func (e *Engine) ReplaySourceCacheEntitlements(ctx context.Context, prev *Engine, scopeKey string) (SourceCacheReplayResult, error) { var res SourceCacheReplayResult - prefix := encodeEntitlementBySourceScopePrefix(scopeHash) + prefix := encodeEntitlementBySourceScopePrefix(scopeKey) primaryHeader := [2]byte{versionV3, typeEntitlement} err := e.withWrite(func() error { @@ -923,7 +923,7 @@ func (e *Engine) ReplaySourceCacheEntitlements(ctx context.Context, prev *Engine closer.Close() return prevScanErr } - if prevScope != scopeHash { + if prevScope != scopeKey { closer.Close() res.StaleSkipped++ continue @@ -939,7 +939,7 @@ func (e *Engine) ReplaySourceCacheEntitlements(ctx context.Context, prev *Engine closer.Close() return scanErr } - if oldScope != "" && oldScope != scopeHash { + if oldScope != "" && oldScope != scopeKey { // The old index key's tail equals the primary key's // tail (identity tuple), so rebuild it byte-wise. oldIdxKey := make([]byte, 0, 8+len(oldScope)+len(priKey)) @@ -1000,12 +1000,12 @@ func (e *Engine) ReplaySourceCacheEntitlements(ctx context.Context, prev *Engine return res, nil } -// ReplaySourceCacheResources copies every resource stamped with scopeHash +// ReplaySourceCacheResources copies every resource stamped with scopeKey // from prev into the receiver, synthesizing by_parent and by_source_scope // index entries from the raw value. -func (e *Engine) ReplaySourceCacheResources(ctx context.Context, prev *Engine, scopeHash string) (SourceCacheReplayResult, error) { +func (e *Engine) ReplaySourceCacheResources(ctx context.Context, prev *Engine, scopeKey string) (SourceCacheReplayResult, error) { var res SourceCacheReplayResult - prefix := encodeResourceBySourceScopePrefix(scopeHash) + prefix := encodeResourceBySourceScopePrefix(scopeKey) primaryHeader := [2]byte{versionV3, typeResource} err := e.withWrite(func() error { @@ -1067,7 +1067,7 @@ func (e *Engine) ReplaySourceCacheResources(ctx context.Context, prev *Engine, s } // Stale-index defense: only copy rows whose VALUE stamp matches // the queried scope (see the grants replay for rationale). - if srcScope != scopeHash { + if srcScope != scopeKey { closer.Close() res.StaleSkipped++ continue diff --git a/pkg/dotc1z/engine/pebble/source_cache_engine_test.go b/pkg/dotc1z/engine/pebble/source_cache_engine_test.go index 7e829e170..2dd3be6f7 100644 --- a/pkg/dotc1z/engine/pebble/source_cache_engine_test.go +++ b/pkg/dotc1z/engine/pebble/source_cache_engine_test.go @@ -58,10 +58,10 @@ func TestSourceCacheEntryCRUD(t *testing.T) { require.NoError(t, e.PutSourceCacheEntry(ctx, "grants", scopeA, `W/"etag-1"`)) rec, err := e.GetSourceCacheEntry(ctx, "grants", scopeA) require.NoError(t, err) - require.Equal(t, `W/"etag-1"`, rec.GetEtag()) + require.Equal(t, `W/"etag-1"`, rec.GetCacheValidator()) require.NotNil(t, rec.GetDiscoveredAt()) - // Row-kind partition: the same scope hash under a different kind misses. + // Row-kind partition: the same scope key under a different kind misses. _, err = e.GetSourceCacheEntry(ctx, "resources", scopeA) require.ErrorIs(t, err, pebble.ErrNotFound) @@ -69,7 +69,7 @@ func TestSourceCacheEntryCRUD(t *testing.T) { require.NoError(t, e.PutSourceCacheEntry(ctx, "grants", scopeA, "delta-token-2")) rec, err = e.GetSourceCacheEntry(ctx, "grants", scopeA) require.NoError(t, err) - require.Equal(t, "delta-token-2", rec.GetEtag()) + require.Equal(t, "delta-token-2", rec.GetCacheValidator()) } // TestSourceCacheReplayGrantsAcrossEngines is the core replay contract: @@ -108,7 +108,7 @@ func TestSourceCacheReplayGrantsAcrossEngines(t *testing.T) { got, err := cur.PebbleEngine().GetGrantRecord(ctx, gExpandable.GetId()) require.NoError(t, err) require.True(t, got.GetNeedsExpansion()) - require.Equal(t, scopeA, got.GetSourceScopeHash()) + require.Equal(t, scopeA, got.GetSourceScopeKey()) _, err = cur.PebbleEngine().GetGrantRecord(ctx, gPlain.GetId()) require.NoError(t, err) @@ -251,18 +251,18 @@ func TestEntitlementRestampCleansOldScopeIndex(t *testing.T) { require.Zero(t, resB.StaleSkipped) got, err := cur.PebbleEngine().GetEntitlementRecord(ctx, entID) require.NoError(t, err) - require.Equal(t, scopeB, got.GetSourceScopeHash()) + require.Equal(t, scopeB, got.GetSourceScopeKey()) require.Equal(t, "member (renamed)", got.GetDisplayName()) } // replayTestGrantScopeIndexKey builds a by_source_scope index key for the // given scope pointing at rec's identity, bypassing the write path — the // test plants it as a stale entry. -func replayTestGrantScopeIndexKey(t *testing.T, scopeHash string, rec *v3.GrantRecord) []byte { +func replayTestGrantScopeIndexKey(t *testing.T, scopeKey string, rec *v3.GrantRecord) []byte { t.Helper() id, err := grantIdentityFromRecord(rec) require.NoError(t, err) - return encodeGrantBySourceScopeIndexKey(scopeHash, id) + return encodeGrantBySourceScopeIndexKey(scopeKey, id) } // TestDeleteGrantsByPrincipalsInScope pins the principal-scoped tombstone @@ -391,11 +391,11 @@ func TestSourceCacheReplayResourcesAndEntitlements(t *testing.T) { gotRes, err := cur.PebbleEngine().GetResourceRecord(ctx, "group", "g1") require.NoError(t, err) require.Equal(t, "Group One", gotRes.GetDisplayName()) - require.Equal(t, scopeA, gotRes.GetSourceScopeHash()) + require.Equal(t, scopeA, gotRes.GetSourceScopeKey()) gotEnt, err := cur.PebbleEngine().GetEntitlementRecord(ctx, entID) require.NoError(t, err) - require.Equal(t, scopeA, gotEnt.GetSourceScopeHash()) + require.Equal(t, scopeA, gotEnt.GetSourceScopeKey()) } // TestReplayInvalidatesEntitlementIDLookup pins the keyspace-generation @@ -437,7 +437,7 @@ func TestReplayInvalidatesEntitlementIDLookup(t *testing.T) { // map built above is stale and must have been invalidated) ... gotEnt, err := cur.PebbleEngine().GetEntitlementRecord(ctx, entID) require.NoError(t, err, "replayed entitlement must resolve by external id after a pre-replay map build") - require.Equal(t, scopeA, gotEnt.GetSourceScopeHash()) + require.Equal(t, scopeA, gotEnt.GetSourceScopeKey()) // ... and, the actual production stake: a delta tombstone against the // replayed row must DELETE it, not silently no-op. @@ -472,7 +472,7 @@ func TestStoreExpandedGrantsPreservesSourceScope(t *testing.T) { got, err := a.PebbleEngine().GetGrantRecord(ctx, g.GetId()) require.NoError(t, err) - require.Equal(t, scopeA, got.GetSourceScopeHash(), "expander rewrite clobbered the source scope stamp") + require.Equal(t, scopeA, got.GetSourceScopeKey(), "expander rewrite clobbered the source scope stamp") require.True(t, got.GetNeedsExpansion(), "expansion side-state must be preserved too") // And the index survives: a replay from this store still finds the row. diff --git a/pkg/dotc1z/entitlements.go b/pkg/dotc1z/entitlements.go index 58049ec2c..6ca4764c3 100644 --- a/pkg/dotc1z/entitlements.go +++ b/pkg/dotc1z/entitlements.go @@ -127,6 +127,6 @@ func (c *C1File) putEntitlementsInternal(ctx context.Context, f entitlementPutFu if err != nil { return err } - c.dbUpdated = true + c.dbUpdated.Store(true) return nil } diff --git a/pkg/dotc1z/grants.go b/pkg/dotc1z/grants.go index 44faaf418..abf3b39a2 100644 --- a/pkg/dotc1z/grants.go +++ b/pkg/dotc1z/grants.go @@ -514,7 +514,7 @@ func (c *C1File) upsertGrants(ctx context.Context, opts grantUpsertOptions, bulk return err } - c.dbUpdated = true + c.dbUpdated.Store(true) return nil } diff --git a/pkg/dotc1z/resouce_types.go b/pkg/dotc1z/resouce_types.go index 7af8a2e29..e1c1f70c3 100644 --- a/pkg/dotc1z/resouce_types.go +++ b/pkg/dotc1z/resouce_types.go @@ -115,6 +115,6 @@ func (c *C1File) putResourceTypesInternal(ctx context.Context, f resourceTypePut if err != nil { return err } - c.dbUpdated = true + c.dbUpdated.Store(true) return nil } diff --git a/pkg/dotc1z/resources.go b/pkg/dotc1z/resources.go index 89186c2e5..75a02e64c 100644 --- a/pkg/dotc1z/resources.go +++ b/pkg/dotc1z/resources.go @@ -131,6 +131,6 @@ func (c *C1File) putResourcesInternal(ctx context.Context, f resourcePutFunc, re if err != nil { return err } - c.dbUpdated = true + c.dbUpdated.Store(true) return nil } diff --git a/pkg/dotc1z/rollback_expansion.go b/pkg/dotc1z/rollback_expansion.go index 7fafd69b5..0fc80f6c8 100644 --- a/pkg/dotc1z/rollback_expansion.go +++ b/pkg/dotc1z/rollback_expansion.go @@ -231,7 +231,7 @@ func (c *C1File) RollbackExpansion(ctx context.Context, syncID string, dryRun bo return nil, err } - c.dbUpdated = true + c.dbUpdated.Store(true) // The cached view sync run carries the now-stale stats; drop it so the // next Stats()/GetSync recomputes against the rolled-back rows. c.invalidateCachedViewSyncRun() diff --git a/pkg/dotc1z/source_cache.go b/pkg/dotc1z/source_cache.go index f89d38f7f..707feab42 100644 --- a/pkg/dotc1z/source_cache.go +++ b/pkg/dotc1z/source_cache.go @@ -23,20 +23,20 @@ type SourceCacheReplayResult = pebble.SourceCacheReplayResult // no replay). It is deliberately NOT part of c1zstore.Store. type SourceCacheStore interface { // LookupSourceCacheEntry returns this store's manifest entry for - // (kind, scopeHash). Backs the connector-facing lookup when this + // (kind, scopeKey). Backs the connector-facing lookup when this // store is the previous sync. - LookupSourceCacheEntry(ctx context.Context, kind sourcecache.RowKind, scopeHash string) (sourcecache.Entry, bool, error) + LookupSourceCacheEntry(ctx context.Context, kind sourcecache.RowKind, scopeKey string) (sourcecache.Entry, bool, error) // PutSourceCacheEntry writes the current sync's manifest entry for - // (kind, scopeHash). Zero-row scopes still get entries. - PutSourceCacheEntry(ctx context.Context, kind sourcecache.RowKind, scopeHash string, etag string) error + // (kind, scopeKey). Zero-row scopes still get entries. + PutSourceCacheEntry(ctx context.Context, kind sourcecache.RowKind, scopeKey string, etag string) error - // ReplaySourceCache copies every row stamped with scopeHash from prev + // ReplaySourceCache copies every row stamped with scopeKey from prev // (the previous sync's store, opened read-only) into this store. prev // must be a Pebble store. Does NOT write the manifest entry — the // caller writes it after the scope's overlay/deletes complete, so a // failed replay can't leave a phantom hit for the next sync. - ReplaySourceCache(ctx context.Context, prev connectorstore.Reader, kind sourcecache.RowKind, scopeHash string) (SourceCacheReplayResult, error) + ReplaySourceCache(ctx context.Context, prev connectorstore.Reader, kind sourcecache.RowKind, scopeKey string) (SourceCacheReplayResult, error) // DeleteSourceCacheRows removes rows by public canonical ID from the // current sync, after replay + overlay (delta-query tombstones). @@ -49,20 +49,20 @@ type SourceCacheStore interface { // connectors use DeleteSourceCacheRowsInScope instead. DeleteSourceCacheRows(ctx context.Context, kind sourcecache.RowKind, ids []string) error - // DeleteSourceCacheRowsInScope removes rows stamped with scopeHash by + // DeleteSourceCacheRowsInScope removes rows stamped with scopeKey by // bare object id — grants by principal id (no principal type, no // canonical-id reconstruction), resources by resource id (any type). // One index scan of the scope per call; a page's tombstones are // batched into one call. Ids with no matching rows are no-ops. // Not supported for entitlements. - DeleteSourceCacheRowsInScope(ctx context.Context, kind sourcecache.RowKind, scopeHash string, ids []string) (int64, error) + DeleteSourceCacheRowsInScope(ctx context.Context, kind sourcecache.RowKind, scopeKey string, ids []string) (int64, error) // DeleteSourceCacheGrantsByIDInScope removes grant rows stamped with - // scopeHash whose STORED grant id is in ids — works for + // scopeKey whose STORED grant id is in ids — works for // connector-custom grant-id shapes that the global bounded delete // cannot resolve, and stays bounded by the scope's row count. Ids with // no matching rows are no-ops. - DeleteSourceCacheGrantsByIDInScope(ctx context.Context, scopeHash string, ids []string) (int64, error) + DeleteSourceCacheGrantsByIDInScope(ctx context.Context, scopeKey string, ids []string) (int64, error) } var _ SourceCacheStore = (*pebbleStore)(nil) @@ -122,11 +122,11 @@ func sourceCacheEngine(store any) (*pebble.Engine, bool) { return e, e != nil } -func (s *pebbleStore) LookupSourceCacheEntry(ctx context.Context, kind sourcecache.RowKind, scopeHash string) (sourcecache.Entry, bool, error) { +func (s *pebbleStore) LookupSourceCacheEntry(ctx context.Context, kind sourcecache.RowKind, scopeKey string) (sourcecache.Entry, bool, error) { if err := sourcecache.ValidateRowKind(kind); err != nil { return sourcecache.Entry{}, false, err } - rec, err := s.engine.GetSourceCacheEntry(ctx, string(kind), scopeHash) + rec, err := s.engine.GetSourceCacheEntry(ctx, string(kind), scopeKey) if err != nil { if errors.Is(err, cdbpebble.ErrNotFound) { return sourcecache.Entry{}, false, nil @@ -134,19 +134,19 @@ func (s *pebbleStore) LookupSourceCacheEntry(ctx context.Context, kind sourcecac return sourcecache.Entry{}, false, err } return sourcecache.Entry{ - ETag: rec.GetEtag(), - DiscoveredAt: rec.GetDiscoveredAt().AsTime(), + CacheValidator: rec.GetCacheValidator(), + DiscoveredAt: rec.GetDiscoveredAt().AsTime(), }, true, nil } -func (s *pebbleStore) PutSourceCacheEntry(ctx context.Context, kind sourcecache.RowKind, scopeHash string, etag string) error { +func (s *pebbleStore) PutSourceCacheEntry(ctx context.Context, kind sourcecache.RowKind, scopeKey string, etag string) error { if err := sourcecache.ValidateRowKind(kind); err != nil { return err } - return s.markDirty(s.engine.PutSourceCacheEntry(ctx, string(kind), scopeHash, etag)) + return s.markDirty(s.engine.PutSourceCacheEntry(ctx, string(kind), scopeKey, etag)) } -func (s *pebbleStore) ReplaySourceCache(ctx context.Context, prev connectorstore.Reader, kind sourcecache.RowKind, scopeHash string) (SourceCacheReplayResult, error) { +func (s *pebbleStore) ReplaySourceCache(ctx context.Context, prev connectorstore.Reader, kind sourcecache.RowKind, scopeKey string) (SourceCacheReplayResult, error) { prevEngine, ok := sourceCacheEngine(prev) if !ok { return SourceCacheReplayResult{}, errors.New("source cache replay: previous sync store is not a pebble store") @@ -155,11 +155,11 @@ func (s *pebbleStore) ReplaySourceCache(ctx context.Context, prev connectorstore var err error switch kind { case sourcecache.RowKindResources: - res, err = s.engine.ReplaySourceCacheResources(ctx, prevEngine, scopeHash) + res, err = s.engine.ReplaySourceCacheResources(ctx, prevEngine, scopeKey) case sourcecache.RowKindEntitlements: - res, err = s.engine.ReplaySourceCacheEntitlements(ctx, prevEngine, scopeHash) + res, err = s.engine.ReplaySourceCacheEntitlements(ctx, prevEngine, scopeKey) case sourcecache.RowKindGrants: - res, err = s.engine.ReplaySourceCacheGrants(ctx, prevEngine, scopeHash) + res, err = s.engine.ReplaySourceCacheGrants(ctx, prevEngine, scopeKey) default: return SourceCacheReplayResult{}, fmt.Errorf("source cache replay: invalid row kind %q", kind) } @@ -208,7 +208,7 @@ func (s *pebbleStore) DeleteSourceCacheRows(ctx context.Context, kind sourcecach return nil } -func (s *pebbleStore) DeleteSourceCacheGrantsByIDInScope(ctx context.Context, scopeHash string, ids []string) (int64, error) { +func (s *pebbleStore) DeleteSourceCacheGrantsByIDInScope(ctx context.Context, scopeKey string, ids []string) (int64, error) { if len(ids) == 0 { return 0, nil } @@ -216,9 +216,9 @@ func (s *pebbleStore) DeleteSourceCacheGrantsByIDInScope(ctx context.Context, sc for _, id := range ids { idSet[id] = struct{}{} } - deleted, err := s.engine.DeleteGrantsByExternalIDsInScope(ctx, scopeHash, idSet) + deleted, err := s.engine.DeleteGrantsByExternalIDsInScope(ctx, scopeKey, idSet) if err != nil { - return 0, fmt.Errorf("source cache grant-id delete for scope %q: %w", scopeHash, err) + return 0, fmt.Errorf("source cache grant-id delete for scope %q: %w", scopeKey, err) } if deleted > 0 { s.MarkDirty() @@ -226,7 +226,7 @@ func (s *pebbleStore) DeleteSourceCacheGrantsByIDInScope(ctx context.Context, sc return deleted, nil } -func (s *pebbleStore) DeleteSourceCacheRowsInScope(ctx context.Context, kind sourcecache.RowKind, scopeHash string, ids []string) (int64, error) { +func (s *pebbleStore) DeleteSourceCacheRowsInScope(ctx context.Context, kind sourcecache.RowKind, scopeKey string, ids []string) (int64, error) { if len(ids) == 0 { return 0, nil } @@ -238,16 +238,16 @@ func (s *pebbleStore) DeleteSourceCacheRowsInScope(ctx context.Context, kind sou var err error switch kind { case sourcecache.RowKindGrants: - deleted, err = s.engine.DeleteGrantsByPrincipalsInScope(ctx, scopeHash, idSet) + deleted, err = s.engine.DeleteGrantsByPrincipalsInScope(ctx, scopeKey, idSet) case sourcecache.RowKindResources: - deleted, err = s.engine.DeleteResourcesByIDsInScope(ctx, scopeHash, idSet) + deleted, err = s.engine.DeleteResourcesByIDsInScope(ctx, scopeKey, idSet) case sourcecache.RowKindEntitlements: return 0, fmt.Errorf("source cache scoped delete: not supported for entitlements") default: return 0, fmt.Errorf("source cache scoped delete: invalid row kind %q", kind) } if err != nil { - return 0, fmt.Errorf("source cache scoped delete for scope %q: %w", scopeHash, err) + return 0, fmt.Errorf("source cache scoped delete for scope %q: %w", scopeKey, err) } if deleted > 0 { s.MarkDirty() diff --git a/pkg/dotc1z/sync_runs.go b/pkg/dotc1z/sync_runs.go index 9171ac446..ee53c2f39 100644 --- a/pkg/dotc1z/sync_runs.go +++ b/pkg/dotc1z/sync_runs.go @@ -518,7 +518,7 @@ func (c *C1File) CheckpointSync(ctx context.Context, syncToken string) error { return err } - c.dbUpdated = true + c.dbUpdated.Store(true) return nil } @@ -707,7 +707,7 @@ func (c *C1File) insertSyncRunWithLink(ctx context.Context, syncID string, syncT if err != nil { return err } - c.dbUpdated = true + c.dbUpdated.Store(true) return nil } @@ -762,7 +762,7 @@ func (c *C1File) endSyncRun(ctx context.Context, syncID string) error { if err != nil { return err } - c.dbUpdated = true + c.dbUpdated.Store(true) // Run stats to generate and save the cached stats. _, _, statsErr := c.stats(ctx, connectorstore.SyncTypeAny, syncID, true) @@ -803,7 +803,7 @@ func (c *C1File) SetSupportsDiff(ctx context.Context, syncID string) error { if err != nil { return err } - c.dbUpdated = true + c.dbUpdated.Store(true) return nil } @@ -840,7 +840,7 @@ func (c *C1File) SetSyncLink(ctx context.Context, syncID string, linkedSyncID st if err != nil { return err } - c.dbUpdated = true + c.dbUpdated.Store(true) return nil } @@ -930,7 +930,7 @@ func (c *C1File) Cleanup(ctx context.Context) error { l.Debug("vacuum complete") } - c.dbUpdated = true + c.dbUpdated.Store(true) // If DB is open in WAL mode, truncate the WAL. var journalMode string @@ -1025,7 +1025,7 @@ func (c *C1File) DeleteSyncRun(ctx context.Context, syncID string) error { deleted += rowsDeleted l.Debug("deleted sync run", zap.String("sync_id", syncID), zap.Int64("rows", deleted)) - c.dbUpdated = true + c.dbUpdated.Store(true) return nil } @@ -1063,7 +1063,7 @@ func (c *C1File) Vacuum(ctx context.Context) error { } } - c.dbUpdated = true + c.dbUpdated.Store(true) return nil } diff --git a/pkg/lambda/grpc/continuation_wire_test.go b/pkg/lambda/grpc/continuation_wire_test.go index e1a7754ad..a6029283d 100644 --- a/pkg/lambda/grpc/continuation_wire_test.go +++ b/pkg/lambda/grpc/continuation_wire_test.go @@ -33,15 +33,15 @@ func continuationRequestPayload(t *testing.T) []byte { annos.Update(v2.SourceCacheLookupAnswers_builder{ Answers: []*v2.SourceCacheLookupAnswers_Answer{ v2.SourceCacheLookupAnswers_Answer_builder{ - RowKind: "grants", - ScopeHash: "groups/g1/members", - Found: true, - Etag: `W/"abc123"`, + RowKind: "grants", + ScopeKey: "groups/g1/members", + Found: true, + CacheValidator: `W/"abc123"`, }.Build(), v2.SourceCacheLookupAnswers_Answer_builder{ - RowKind: "grants", - ScopeHash: "groups/g2/members", - Found: false, + RowKind: "grants", + ScopeKey: "groups/g2/members", + Found: false, }.Build(), }, }.Build()) @@ -71,7 +71,7 @@ func assertContinuationRequest(t *testing.T, req *Request) { require.True(t, ok, "answers must survive the wire") require.Len(t, answers.GetAnswers(), 2) require.True(t, answers.GetAnswers()[0].GetFound()) - require.Equal(t, `W/"abc123"`, answers.GetAnswers()[0].GetEtag()) + require.Equal(t, `W/"abc123"`, answers.GetAnswers()[0].GetCacheValidator()) require.False(t, answers.GetAnswers()[1].GetFound(), "explicit not-found must survive (distinct from absent)") } @@ -107,7 +107,7 @@ func TestContinuationAnnotations_AskResponseRoundTrip(t *testing.T) { inner := v2.GrantsServiceListGrantsResponse_builder{ Annotations: annotations.New(v2.SourceCacheLookupAsk_builder{ Queries: []*v2.SourceCacheLookupAsk_Query{ - v2.SourceCacheLookupAsk_Query_builder{RowKind: "grants", ScopeHash: "groups/g1/members"}.Build(), + v2.SourceCacheLookupAsk_Query_builder{RowKind: "grants", ScopeKey: "groups/g1/members"}.Build(), }, }.Build()), }.Build() @@ -139,6 +139,6 @@ func TestContinuationAnnotations_AskResponseRoundTrip(t *testing.T) { require.NoError(t, err) require.True(t, ok, "ask must survive the wire (wireV2=%v)", wireV2) require.Len(t, ask.GetQueries(), 1) - require.Equal(t, "groups/g1/members", ask.GetQueries()[0].GetScopeHash()) + require.Equal(t, "groups/g1/members", ask.GetQueries()[0].GetScopeKey()) } } diff --git a/pkg/sourcecache/context.go b/pkg/sourcecache/context.go index 266dd5c04..da56f938b 100644 --- a/pkg/sourcecache/context.go +++ b/pkg/sourcecache/context.go @@ -4,15 +4,15 @@ import "context" type scopeContextKey struct{} -// WithScope returns a context carrying the source-cache scope hash for +// WithScope returns a context carrying the source-cache scope key for // rows written under it. The syncer wraps a page's store writes in this -// context when the page carried a SourceCacheScope annotation; the Pebble -// write path stamps the record's source_scope_hash from it. -func WithScope(ctx context.Context, scopeHash string) context.Context { - return context.WithValue(ctx, scopeContextKey{}, scopeHash) +// context when the page carried a SourceCacheRecord annotation; the Pebble +// write path stamps the record's source_scope_key from it. +func WithScope(ctx context.Context, scopeKey string) context.Context { + return context.WithValue(ctx, scopeContextKey{}, scopeKey) } -// ScopeFromContext returns the scope hash set by WithScope, or "" when +// ScopeFromContext returns the scope key set by WithScope, or "" when // the context carries none (the common, unstamped case). func ScopeFromContext(ctx context.Context) string { s, _ := ctx.Value(scopeContextKey{}).(string) diff --git a/pkg/sourcecache/continuation.go b/pkg/sourcecache/continuation.go index ea647bc8a..9ef68c040 100644 --- a/pkg/sourcecache/continuation.go +++ b/pkg/sourcecache/continuation.go @@ -33,8 +33,8 @@ var ErrLookupDeferred = errors.New("source cache lookup deferred: answer arrives // Query identifies one scope to resolve against the previous sync. type Query struct { - RowKind RowKind - ScopeHash string + RowKind RowKind + ScopeKey string } // Answer resolves one Query. Found=false means the previous sync has no @@ -42,15 +42,15 @@ type Query struct { // Answer at all, which means unresolved: ask again.) type Answer struct { Query - Found bool - ETag string + Found bool + CacheValidator string } // BatchLookup is optionally implemented by Lookup implementations that can // resolve many scopes in one round trip. Connectors should not type-assert // for it directly; call LookupMany, which falls back to per-query lookups. type BatchLookup interface { - LookupPreviousSourceCacheMany(ctx context.Context, queries []Query) ([]Answer, error) + LookupMany(ctx context.Context, queries []Query) ([]Answer, error) } // LookupMany resolves a batch of queries through lookup, using one round @@ -67,17 +67,17 @@ type BatchLookup interface { // is correct, just slower — never as a silent omission.) func LookupMany(ctx context.Context, lookup Lookup, queries []Query) ([]Answer, error) { if bl, ok := lookup.(BatchLookup); ok { - return bl.LookupPreviousSourceCacheMany(ctx, queries) + return bl.LookupMany(ctx, queries) } answers := make([]Answer, 0, len(queries)) for _, q := range queries { - entry, found, err := lookup.LookupPreviousSourceCache(ctx, q.RowKind, q.ScopeHash) + entry, found, err := lookup.Lookup(ctx, q.RowKind, q.ScopeKey) if err != nil { return nil, err } a := Answer{Query: q, Found: found} if found { - a.ETag = entry.ETag + a.CacheValidator = entry.CacheValidator } answers = append(answers, a) } @@ -117,21 +117,21 @@ func NewContinuationLookup(answers []Answer) *ContinuationLookup { } } -func (c *ContinuationLookup) LookupPreviousSourceCache(_ context.Context, rowKind RowKind, scopeHash string) (Entry, bool, error) { - q := Query{RowKind: rowKind, ScopeHash: scopeHash} +func (c *ContinuationLookup) Lookup(_ context.Context, rowKind RowKind, scopeKey string) (Entry, bool, error) { + q := Query{RowKind: rowKind, ScopeKey: scopeKey} c.mu.Lock() defer c.mu.Unlock() if a, ok := c.answers[q]; ok { if !a.Found { return Entry{}, false, nil } - return Entry{ETag: a.ETag}, true, nil + return Entry{CacheValidator: a.CacheValidator}, true, nil } c.recordLocked(q) - return Entry{}, false, fmt.Errorf("lookup %s/%s: %w", rowKind, scopeHash, ErrLookupDeferred) + return Entry{}, false, fmt.Errorf("lookup %s/%s: %w", rowKind, scopeKey, ErrLookupDeferred) } -func (c *ContinuationLookup) LookupPreviousSourceCacheMany(_ context.Context, queries []Query) ([]Answer, error) { +func (c *ContinuationLookup) LookupMany(_ context.Context, queries []Query) ([]Answer, error) { c.mu.Lock() defer c.mu.Unlock() answers := make([]Answer, 0, len(queries)) @@ -181,8 +181,8 @@ func AskProto(queries []Query) *v2.SourceCacheLookupAsk { qs := make([]*v2.SourceCacheLookupAsk_Query, 0, len(queries)) for _, q := range queries { qs = append(qs, v2.SourceCacheLookupAsk_Query_builder{ - RowKind: string(q.RowKind), - ScopeHash: q.ScopeHash, + RowKind: string(q.RowKind), + ScopeKey: q.ScopeKey, }.Build()) } return v2.SourceCacheLookupAsk_builder{Queries: qs}.Build() @@ -219,10 +219,10 @@ func QueriesFromProto(ask *v2.SourceCacheLookupAsk) ([]Query, error) { if err := ValidateRowKind(kind); err != nil { return nil, err } - if err := ValidateScopeHash(q.GetScopeHash()); err != nil { + if err := ValidateScopeKey(q.GetScopeKey()); err != nil { return nil, err } - out = append(out, Query{RowKind: kind, ScopeHash: q.GetScopeHash()}) + out = append(out, Query{RowKind: kind, ScopeKey: q.GetScopeKey()}) } return out, nil } @@ -232,10 +232,10 @@ func AnswersProto(answers []Answer) *v2.SourceCacheLookupAnswers { as := make([]*v2.SourceCacheLookupAnswers_Answer, 0, len(answers)) for _, a := range answers { as = append(as, v2.SourceCacheLookupAnswers_Answer_builder{ - RowKind: string(a.RowKind), - ScopeHash: a.ScopeHash, - Found: a.Found, - Etag: a.ETag, + RowKind: string(a.RowKind), + ScopeKey: a.ScopeKey, + Found: a.Found, + CacheValidator: a.CacheValidator, }.Build()) } return v2.SourceCacheLookupAnswers_builder{Answers: as}.Build() @@ -253,8 +253,8 @@ func AnswersProto(answers []Answer) *v2.SourceCacheLookupAnswers { // one ask's size would hard-fail a legal 4096-scope first ask followed by // a single late scope on a later bounce. const ( - maxAnswersPerMessage = maxQueriesPerAsk * MaxLookupBouncesPerRequest - maxAnswerETagLen = 65536 + maxAnswersPerMessage = maxQueriesPerAsk * MaxLookupBouncesPerRequest + maxAnswerValidatorLen = 65536 ) // AnswersFromProto extracts and validates the answers delivered on a @@ -269,16 +269,16 @@ func AnswersFromProto(msg *v2.SourceCacheLookupAnswers) ([]Answer, error) { if err := ValidateRowKind(kind); err != nil { return nil, fmt.Errorf("source cache lookup answers: %w", err) } - if err := ValidateScopeHash(a.GetScopeHash()); err != nil { + if err := ValidateScopeKey(a.GetScopeKey()); err != nil { return nil, fmt.Errorf("source cache lookup answers: %w", err) } - if len(a.GetEtag()) > maxAnswerETagLen { - return nil, fmt.Errorf("source cache lookup answers: etag for scope %q is %d bytes (max %d)", a.GetScopeHash(), len(a.GetEtag()), maxAnswerETagLen) + if len(a.GetCacheValidator()) > maxAnswerValidatorLen { + return nil, fmt.Errorf("source cache lookup answers: cache validator for scope %q is %d bytes (max %d)", a.GetScopeKey(), len(a.GetCacheValidator()), maxAnswerValidatorLen) } out = append(out, Answer{ - Query: Query{RowKind: kind, ScopeHash: a.GetScopeHash()}, - Found: a.GetFound(), - ETag: a.GetEtag(), + Query: Query{RowKind: kind, ScopeKey: a.GetScopeKey()}, + Found: a.GetFound(), + CacheValidator: a.GetCacheValidator(), }) } return out, nil diff --git a/pkg/sourcecache/grpc_lookup.go b/pkg/sourcecache/grpc_lookup.go index 4d81eba9f..b69206adf 100644 --- a/pkg/sourcecache/grpc_lookup.go +++ b/pkg/sourcecache/grpc_lookup.go @@ -35,7 +35,7 @@ import ( // scope), and not at all when the caller's context is already done — a // teardown-time Canceled is noise, and logging it would eat the rate // window a real failure needs. Validation failures stay loud: a malformed -// row kind or scope hash is a connector bug, and degrading it to a miss +// row kind or scope key is a connector bug, and degrading it to a miss // would let the connector silently cold-fetch forever with no signal. type GRPCLookup struct { client v1.BatonSourceCacheServiceClient @@ -58,16 +58,16 @@ func NewGRPCLookup(client v1.BatonSourceCacheServiceClient) Lookup { return &GRPCLookup{client: client} } -func (g *GRPCLookup) LookupPreviousSourceCache(ctx context.Context, rowKind RowKind, scopeHash string) (Entry, bool, error) { +func (g *GRPCLookup) Lookup(ctx context.Context, rowKind RowKind, scopeKey string) (Entry, bool, error) { if err := ValidateRowKind(rowKind); err != nil { return Entry{}, false, err } - if err := ValidateScopeHash(scopeHash); err != nil { + if err := ValidateScopeKey(scopeKey); err != nil { return Entry{}, false, err } resp, err := g.client.Lookup(ctx, v1.LookupRequest_builder{ - RowKind: string(rowKind), - ScopeHash: scopeHash, + RowKind: string(rowKind), + ScopeKey: scopeKey, }.Build()) if err != nil { count := g.failures.Add(1) @@ -87,5 +87,5 @@ func (g *GRPCLookup) LookupPreviousSourceCache(ctx context.Context, rowKind RowK if !resp.GetFound() { return Entry{}, false, nil } - return Entry{ETag: resp.GetEtag()}, true, nil + return Entry{CacheValidator: resp.GetCacheValidator()}, true, nil } diff --git a/pkg/sourcecache/grpc_lookup_test.go b/pkg/sourcecache/grpc_lookup_test.go index 3d8bcfcd1..ba9d6112e 100644 --- a/pkg/sourcecache/grpc_lookup_test.go +++ b/pkg/sourcecache/grpc_lookup_test.go @@ -34,7 +34,7 @@ func TestGRPCLookupDegradesRPCErrorToMiss(t *testing.T) { lookup := NewGRPCLookup(&fakeLookupClient{err: errors.New("connection refused")}) scope := HashScope("https://example.test/teams/1/members?page=1") - _, found, err := lookup.LookupPreviousSourceCache(ctx, RowKindGrants, scope) + _, found, err := lookup.Lookup(ctx, RowKindGrants, scope) require.NoError(t, err, "an RPC failure must degrade to a miss, not fail the connector call") require.False(t, found) } @@ -90,14 +90,14 @@ func TestGRPCLookupFailureLogRateLimited(t *testing.T) { // window a real failure needs. canceledCtx, cancel := context.WithCancel(ctx) cancel() - _, found, err := lookup.LookupPreviousSourceCache(canceledCtx, RowKindGrants, scope) + _, found, err := lookup.Lookup(canceledCtx, RowKindGrants, scope) require.NoError(t, err) require.False(t, found) require.Empty(t, *logs, "a failure on a done context must not log") // The first live-context failure logs immediately, with the running // count including the silent canceled-context failure above. - _, found, err = lookup.LookupPreviousSourceCache(ctx, RowKindGrants, scope) + _, found, err = lookup.Lookup(ctx, RowKindGrants, scope) require.NoError(t, err) require.False(t, found) require.Len(t, *logs, 1) @@ -107,7 +107,7 @@ func TestGRPCLookupFailureLogRateLimited(t *testing.T) { // Immediate subsequent failures still degrade to misses but stay // within the rate window and do not log again. for range 10 { - _, found, err = lookup.LookupPreviousSourceCache(ctx, RowKindGrants, scope) + _, found, err = lookup.Lookup(ctx, RowKindGrants, scope) require.NoError(t, err) require.False(t, found) } @@ -119,27 +119,27 @@ func TestGRPCLookupFailureLogRateLimited(t *testing.T) { // would let a broken connector silently cold-fetch forever. func TestGRPCLookupValidationStaysLoud(t *testing.T) { ctx := context.Background() - lookup := NewGRPCLookup(&fakeLookupClient{resp: v1.LookupResponse_builder{Found: true, Etag: "e"}.Build()}) + lookup := NewGRPCLookup(&fakeLookupClient{resp: v1.LookupResponse_builder{Found: true, CacheValidator: "e"}.Build()}) - _, _, err := lookup.LookupPreviousSourceCache(ctx, RowKind("bogus"), HashScope("s")) + _, _, err := lookup.Lookup(ctx, RowKind("bogus"), HashScope("s")) require.Error(t, err, "invalid row kind must fail loudly") - _, _, err = lookup.LookupPreviousSourceCache(ctx, RowKindGrants, "") - require.Error(t, err, "empty scope hash must fail loudly") + _, _, err = lookup.Lookup(ctx, RowKindGrants, "") + require.Error(t, err, "empty scope key must fail loudly") } func TestGRPCLookupHitAndMiss(t *testing.T) { ctx := context.Background() scope := HashScope("https://example.test/teams/1/members?page=1") - hit := NewGRPCLookup(&fakeLookupClient{resp: v1.LookupResponse_builder{Found: true, Etag: `W/"etag-1"`}.Build()}) - entry, found, err := hit.LookupPreviousSourceCache(ctx, RowKindGrants, scope) + hit := NewGRPCLookup(&fakeLookupClient{resp: v1.LookupResponse_builder{Found: true, CacheValidator: `W/"etag-1"`}.Build()}) + entry, found, err := hit.Lookup(ctx, RowKindGrants, scope) require.NoError(t, err) require.True(t, found) - require.Equal(t, `W/"etag-1"`, entry.ETag) + require.Equal(t, `W/"etag-1"`, entry.CacheValidator) miss := NewGRPCLookup(&fakeLookupClient{resp: v1.LookupResponse_builder{Found: false}.Build()}) - _, found, err = miss.LookupPreviousSourceCache(ctx, RowKindGrants, scope) + _, found, err = miss.Lookup(ctx, RowKindGrants, scope) require.NoError(t, err) require.False(t, found) } diff --git a/pkg/sourcecache/grpc_server.go b/pkg/sourcecache/grpc_server.go index 088a5fbeb..e6d5d0206 100644 --- a/pkg/sourcecache/grpc_server.go +++ b/pkg/sourcecache/grpc_server.go @@ -60,8 +60,8 @@ func (s *GRPCServer) Lookup(ctx context.Context, req *v1.LookupRequest) (*v1.Loo if err := ValidateRowKind(rowKind); err != nil { return nil, err } - scopeHash := req.GetScopeHash() - if err := ValidateScopeHash(scopeHash); err != nil { + scopeKey := req.GetScopeKey() + if err := ValidateScopeKey(scopeKey); err != nil { return nil, err } @@ -69,7 +69,7 @@ func (s *GRPCServer) Lookup(ctx context.Context, req *v1.LookupRequest) (*v1.Loo if lookupPtr == nil { return v1.LookupResponse_builder{Found: false}.Build(), nil } - entry, found, err := (*lookupPtr).LookupPreviousSourceCache(ctx, rowKind, scopeHash) + entry, found, err := (*lookupPtr).Lookup(ctx, rowKind, scopeKey) if err != nil { return nil, fmt.Errorf("source cache lookup: %w", err) } @@ -77,7 +77,7 @@ func (s *GRPCServer) Lookup(ctx context.Context, req *v1.LookupRequest) (*v1.Loo return v1.LookupResponse_builder{Found: false}.Build(), nil } return v1.LookupResponse_builder{ - Found: true, - Etag: entry.ETag, + Found: true, + CacheValidator: entry.CacheValidator, }.Build(), nil } diff --git a/pkg/sourcecache/sourcecache.go b/pkg/sourcecache/sourcecache.go index 1892cda35..bc378eaf6 100644 --- a/pkg/sourcecache/sourcecache.go +++ b/pkg/sourcecache/sourcecache.go @@ -6,18 +6,18 @@ // SourceCacheCapability MODE_READ_WRITE to its Validate response. During a // sync it looks up the previous validator for a scope via the Lookup the SDK // provides on SyncOpAttrs, revalidates upstream, and either emits fresh rows -// tagged with SourceCacheScope or asks the SDK to replay the previous rows +// tagged with SourceCacheRecord or asks the SDK to replay the previous rows // with SourceCacheReplay. // // The connector owns scope computation; the SDK only keys storage by the -// connector-supplied scope hash. The validator (etag, delta token) is opaque +// connector-supplied scope key. The validator (etag, delta token) is opaque // to the SDK. // // Invariant that keeps replay safe: a connector must only emit // SourceCacheReplay for a scope whose validator it received from THIS sync's // Lookup. The lookup need not happen in the same call that emits the // replay: a planning call may batch-resolve many scopes and pass the -// verdicts to sibling cursors through SpawnCursors page tokens — that +// verdicts to sibling cursors through EnqueuePageTokens page tokens — that // satisfies the invariant, because the validator still originates from the // consuming sync. What's forbidden is a validator that outlives a sync // (connector-side caches, config, upstream echoes). When source cache is @@ -72,31 +72,31 @@ func ValidateRowKind(rowKind RowKind) error { return nil } -// maxScopeHashLen bounds scope identifiers on the wire and in storage +// maxScopeKeyLen bounds scope identifiers on the wire and in storage // keys. Deliberately generous: the shape is a connector convention // (HashScope produces 64 hex chars) and is not enforced beyond // non-emptiness and this cap while the model is being proven out against // real providers. -const maxScopeHashLen = 256 +const maxScopeKeyLen = 256 -// ValidateScopeHash returns an error when scopeHash is empty or +// ValidateScopeKey returns an error when scopeKey is empty or // unreasonably long. Connectors conventionally use HashScope, but any // stable identifier is accepted. -func ValidateScopeHash(scopeHash string) error { - if scopeHash == "" { - return fmt.Errorf("source cache scope hash is required") +func ValidateScopeKey(scopeKey string) error { + if scopeKey == "" { + return fmt.Errorf("source cache scope key is required") } - if len(scopeHash) > maxScopeHashLen { - return fmt.Errorf("source cache scope hash too long: %d bytes (max %d)", len(scopeHash), maxScopeHashLen) + if len(scopeKey) > maxScopeKeyLen { + return fmt.Errorf("source cache scope key too long: %d bytes (max %d)", len(scopeKey), maxScopeKeyLen) } return nil } // Entry is a previous sync's persisted validator for one scope. type Entry struct { - // ETag is the opaque upstream validator: a literal HTTP ETag, a delta - // token, etc. Never interpreted by the SDK. - ETag string + // CacheValidator is the opaque upstream validator: a literal HTTP + // ETag, a delta token, etc. Never interpreted by the SDK. + CacheValidator string // DiscoveredAt is when the entry was written. DiscoveredAt time.Time @@ -106,11 +106,11 @@ type Entry struct { // implementation on SyncOpAttrs; connectors call it before revalidating // upstream. type Lookup interface { - // LookupPreviousSourceCache returns the previous sync's entry for - // (rowKind, scopeHash). found=false means no entry: fetch fresh. + // Lookup returns the previous sync's entry for + // (rowKind, scopeKey). found=false means no entry: fetch fresh. // Implementations must treat internal read errors that leave fresh // fetch available as misses rather than failing the connector call. - LookupPreviousSourceCache(ctx context.Context, rowKind RowKind, scopeHash string) (entry Entry, found bool, err error) + Lookup(ctx context.Context, rowKind RowKind, scopeKey string) (entry Entry, found bool, err error) } // NoopLookup is the Lookup installed when source cache is disabled or @@ -119,7 +119,7 @@ type NoopLookup struct{} var _ Lookup = NoopLookup{} -func (NoopLookup) LookupPreviousSourceCache(context.Context, RowKind, string) (Entry, bool, error) { +func (NoopLookup) Lookup(context.Context, RowKind, string) (Entry, bool, error) { return Entry{}, false, nil } @@ -133,7 +133,7 @@ type SetLookup interface { // HashScope returns the lowercase-hex sha256 of a canonical scope string. // Convenience for connectors; any stable identifier is acceptable as a -// scope hash (only non-emptiness and a length cap are enforced). +// scope key (only non-emptiness and a length cap are enforced). func HashScope(canonicalScope string) string { sum := sha256.Sum256([]byte(canonicalScope)) return hex.EncodeToString(sum[:]) diff --git a/pkg/sync/spawn_cursors_test.go b/pkg/sync/enqueue_page_tokens_test.go similarity index 95% rename from pkg/sync/spawn_cursors_test.go rename to pkg/sync/enqueue_page_tokens_test.go index 82dc7147a..655d5325b 100644 --- a/pkg/sync/spawn_cursors_test.go +++ b/pkg/sync/enqueue_page_tokens_test.go @@ -1,13 +1,13 @@ package sync //nolint:revive,nolintlint // we can't change the package name for backwards compatibility -// Per-resource SpawnCursors: parallel warm revalidation for page-numbered +// Per-resource EnqueuePageTokens: parallel warm revalidation for page-numbered // APIs (the GitHub shape). On page one of a resource's grant collection the // connector already knows every other page's URL and stored validator, so // it answers page one and spawns pages 2..N as sibling actions; each // spawned page independently hits or misses its source-cache lookup. // // Under test: -// - SpawnCursors on a per-resource grants response fans out sibling +// - EnqueuePageTokens on a per-resource grants response fans out sibling // actions carrying the resource identity (no warning, no drops); // - spawned pages are ordinary pages: replay on hit, cold fetch on miss // (page boundary shifted), and a cold spawned page may CHAIN to a @@ -41,7 +41,7 @@ const spawnPageSize = 4 // pagedSpawnMockConnector serves each group's member grants through // numbered pages of size spawnPageSize. Page 1 carries its own rows plus -// SpawnCursors for every other page the previous sync recorded (warm) or +// EnqueuePageTokens for every other page the previous sync recorded (warm) or // that exist upstream (cold). Every page is its own source-cache scope // (page-URL style). Membership mutations shift page contents, so mutated // pages miss their lookup and refetch cold — including discovering fresh @@ -202,18 +202,18 @@ func (mc *pagedSpawnMockConnector) servePage(ctx context.Context, gid string, pa } scope := pageScopeFor(gid, page) - entry, found, err := lookup.LookupPreviousSourceCache(ctx, sourcecache.RowKindGrants, scope) + entry, found, err := lookup.Lookup(ctx, sourcecache.RowKindGrants, scope) if err != nil { return nil, err } - if found && entry.ETag == etag { + if found && entry.CacheValidator == etag { mc.mu.Lock() mc.replayPages++ mc.mu.Unlock() return v2.GrantsServiceListGrantsResponse_builder{ Annotations: annotations.New(v2.SourceCacheReplay_builder{ - ScopeHash: scope, - Etag: etag, + ScopeKey: scope, + CacheValidator: etag, }.Build()), }.Build(), nil } @@ -232,9 +232,9 @@ func (mc *pagedSpawnMockConnector) servePage(ctx context.Context, gid string, pa mc.mu.Unlock() return v2.GrantsServiceListGrantsResponse_builder{ List: mc.grantsFor(gid, rows), - Annotations: annotations.New(v2.SourceCacheScope_builder{ - ScopeHash: scope, - Etag: etag, + Annotations: annotations.New(v2.SourceCacheRecord_builder{ + ScopeKey: scope, + CacheValidator: etag, }.Build()), NextPageToken: next, }.Build(), nil @@ -274,7 +274,7 @@ func (mc *pagedSpawnMockConnector) ListGrants(ctx context.Context, in *v2.Grants } if len(tokens) > 0 { annos := annotations.Annotations(resp.GetAnnotations()) - annos.Update(v2.SpawnCursors_builder{PageTokens: tokens}.Build()) + annos.Update(v2.EnqueuePageTokens_builder{PageTokens: tokens}.Build()) resp.SetAnnotations(annos) // Page one's own chain ends here; siblings carry the rest. resp.SetNextPageToken("") @@ -322,7 +322,7 @@ func runSpawnSync(ctx context.Context, t *testing.T, mc *pagedSpawnMockConnector return covered } -func TestPerResourceSpawnCursorsWarmRevalidation(t *testing.T) { +func TestPerResourceEnqueuePageTokensWarmRevalidation(t *testing.T) { for _, workers := range []int{0, 4} { t.Run(fmt.Sprintf("workers=%d", workers), func(t *testing.T) { ctx, err := logging.Init(context.Background()) @@ -377,7 +377,7 @@ func TestPerResourceSpawnCursorsWarmRevalidation(t *testing.T) { } } -// TestPerResourceSpawnCursorsResumeWithPendingSiblings pins suspend/resume +// TestPerResourceEnqueuePageTokensResumeWithPendingSiblings pins suspend/resume // across a spawn fan-out. A warm sync crashes while serving one of a // group's spawned sibling pages and is then resumed on the same c1z. // Checkpoints are rate-limited (minCheckpointInterval), so resume @@ -395,7 +395,7 @@ func TestPerResourceSpawnCursorsWarmRevalidation(t *testing.T) { // (The complementary serialization half — a checkpoint can never hold // "page finished but siblings missing", and the Spawned marker survives // the round trip — is pinned by TestSpawnedActionsSurviveCheckpoint.) -func TestPerResourceSpawnCursorsResumeWithPendingSiblings(t *testing.T) { +func TestPerResourceEnqueuePageTokensResumeWithPendingSiblings(t *testing.T) { for _, workers := range []int{0, 4} { t.Run(fmt.Sprintf("workers=%d", workers), func(t *testing.T) { ctx, err := logging.Init(context.Background()) @@ -516,7 +516,7 @@ func TestSpawnedActionsSurviveCheckpoint(t *testing.T) { require.Equal(t, map[string]bool{"page=1": true, "page=2": true}, tokens) } -// TestPerResourceSpawnCursorsChurnSoak is the randomized soak (the shape +// TestPerResourceEnqueuePageTokensChurnSoak is the randomized soak (the shape // that caught real bugs in both the Entra and GitHub POCs): seeded rounds // of membership churn against the paged-spawn connector, each round // running a warm chained sync at workers=4 and asserting reader-surface @@ -524,7 +524,7 @@ func TestSpawnedActionsSurviveCheckpoint(t *testing.T) { // shifts page boundaries arbitrarily, so every round mixes replayed pages, // cold refetches of shifted pages, vanished tails (zero-row probes), and // fresh tails discovered by chaining — concurrently. -func TestPerResourceSpawnCursorsChurnSoak(t *testing.T) { +func TestPerResourceEnqueuePageTokensChurnSoak(t *testing.T) { if testing.Short() { t.Skip("soak test") } diff --git a/pkg/sync/progresslog/progresslog.go b/pkg/sync/progresslog/progresslog.go index 9286ddc98..e30773825 100644 --- a/pkg/sync/progresslog/progresslog.go +++ b/pkg/sync/progresslog/progresslog.go @@ -300,7 +300,7 @@ func (p *ProgressLog) SetGrantsCountOnly(resourceType string) { // GrantsProgress returns the current grant-coverage counter for a resource // type. For per-resource types this is "resources covered" and must never // exceed the type's resource total — spawned sibling cursors -// (v2.SpawnCursors) don't increment it, only the origin action's chain end +// (v2.EnqueuePageTokens) don't increment it, only the origin action's chain end // does. Exposed for tests pinning that accounting. func (p *ProgressLog) GrantsProgress(resourceType string) int { p.mu.RLock() diff --git a/pkg/sync/source_cache.go b/pkg/sync/source_cache.go index 375e3eb5f..eb12c4a9b 100644 --- a/pkg/sync/source_cache.go +++ b/pkg/sync/source_cache.go @@ -69,7 +69,7 @@ type replayedScopeSet struct { m map[string]struct{} } -func (c *syncerSourceCache) markScopeReplayed(kind sourcecache.RowKind, scopeHash string) { +func (c *syncerSourceCache) markScopeReplayed(kind sourcecache.RowKind, scopeKey string) { if c.replayedScopes == nil { return } @@ -78,16 +78,16 @@ func (c *syncerSourceCache) markScopeReplayed(kind sourcecache.RowKind, scopeHas if c.replayedScopes.m == nil { c.replayedScopes.m = make(map[string]struct{}) } - c.replayedScopes.m[string(kind)+"\x00"+scopeHash] = struct{}{} + c.replayedScopes.m[string(kind)+"\x00"+scopeKey] = struct{}{} } -func (c *syncerSourceCache) scopeReplayedThisSync(kind sourcecache.RowKind, scopeHash string) bool { +func (c *syncerSourceCache) scopeReplayedThisSync(kind sourcecache.RowKind, scopeKey string) bool { if c.replayedScopes == nil { return false } c.replayedScopes.mu.Lock() defer c.replayedScopes.mu.Unlock() - _, ok := c.replayedScopes.m[string(kind)+"\x00"+scopeHash] + _, ok := c.replayedScopes.m[string(kind)+"\x00"+scopeKey] return ok } @@ -102,8 +102,8 @@ type prevStoreLookup struct { var _ sourcecache.Lookup = prevStoreLookup{} -func (p prevStoreLookup) LookupPreviousSourceCache(ctx context.Context, kind sourcecache.RowKind, scopeHash string) (sourcecache.Entry, bool, error) { - entry, found, err := p.prev.LookupSourceCacheEntry(ctx, kind, scopeHash) +func (p prevStoreLookup) Lookup(ctx context.Context, kind sourcecache.RowKind, scopeKey string) (sourcecache.Entry, bool, error) { + entry, found, err := p.prev.LookupSourceCacheEntry(ctx, kind, scopeKey) if err != nil { p.logOnce.Do(func() { ctxzap.Extract(ctx).Warn("source cache lookup failed; treating as miss", zap.Error(err)) @@ -244,9 +244,9 @@ func (s *syncer) clearSourceCacheLookup(ctx context.Context) { // from beginSourceCachePage (before rows are written) to // finishSourceCachePage (after rows are written). type sourceCachePage struct { - kind sourcecache.RowKind - scopeHash string - etag string + kind sourcecache.RowKind + scopeKey string + etag string // replayed reports that beginSourceCachePage copied the previous // sync's rows for this scope into the current sync BEFORE the page's // own rows commit. Consumers that dedupe against "already synced this @@ -287,7 +287,7 @@ func (s *syncer) beginSourceCachePage( if err != nil { return ctx, nil, fmt.Errorf("source cache: error parsing replay annotation: %w", err) } - scope := &v2.SourceCacheScope{} + scope := &v2.SourceCacheRecord{} hasScope, err := respAnnos.Pick(scope) if err != nil { return ctx, nil, fmt.Errorf("source cache: error parsing scope annotation: %w", err) @@ -302,7 +302,7 @@ func (s *syncer) beginSourceCachePage( // source cache disabled there is nothing to replay from. This is // a connector bug (replay for a scope it never got a lookup hit // on), not a degradable condition. - return ctx, nil, fmt.Errorf("source cache: connector requested replay for scope %q but source cache is disabled", replay.GetScopeHash()) + return ctx, nil, fmt.Errorf("source cache: connector requested replay for scope %q but source cache is disabled", replay.GetScopeKey()) } // Scope annotations without the capability handshake are ignored. return ctx, nil, nil @@ -311,23 +311,23 @@ func (s *syncer) beginSourceCachePage( page := &sourceCachePage{kind: kind} switch { case hasReplay && hasScope: - if replay.GetScopeHash() != scope.GetScopeHash() { - return ctx, nil, fmt.Errorf("source cache: replay scope %q and page scope %q disagree", replay.GetScopeHash(), scope.GetScopeHash()) + if replay.GetScopeKey() != scope.GetScopeKey() { + return ctx, nil, fmt.Errorf("source cache: replay scope %q and page scope %q disagree", replay.GetScopeKey(), scope.GetScopeKey()) } - page.scopeHash = replay.GetScopeHash() + page.scopeKey = replay.GetScopeKey() case hasReplay: - page.scopeHash = replay.GetScopeHash() + page.scopeKey = replay.GetScopeKey() default: - page.scopeHash = scope.GetScopeHash() + page.scopeKey = scope.GetScopeKey() } - if err := sourcecache.ValidateScopeHash(page.scopeHash); err != nil { + if err := sourcecache.ValidateScopeKey(page.scopeKey); err != nil { return ctx, nil, fmt.Errorf("source cache: %w", err) } // Prefer the scope annotation's etag (the freshest validator on // overlay pages); fall back to the replay's. - page.etag = scope.GetEtag() + page.etag = scope.GetCacheValidator() if page.etag == "" { - page.etag = replay.GetEtag() + page.etag = replay.GetCacheValidator() } // Tombstones may ride either annotation — the replay annotation on a // round's first page, the scope annotation on every page (so a @@ -341,7 +341,7 @@ func (s *syncer) beginSourceCachePage( if s.sourceCache.prev == nil { // Same invariant violation as the disabled case: the connector // can only have gotten a lookup hit if a previous source exists. - return ctx, nil, fmt.Errorf("source cache: connector requested replay for scope %q but no previous sync is available", replay.GetScopeHash()) + return ctx, nil, fmt.Errorf("source cache: connector requested replay for scope %q but no previous sync is available", replay.GetScopeKey()) } page.replayed = true if !replay.GetOverlay() && rowCount > 0 { @@ -351,7 +351,7 @@ func (s *syncer) beginSourceCachePage( // sync. Kept lenient while the model is proven against real // providers. ctxzap.Extract(ctx).Warn("source cache: non-overlay replay returned rows; treating them as an overlay", - zap.String("scope_hash", page.scopeHash), + zap.String("scope_key", page.scopeKey), zap.Int("rows", rowCount), ) } @@ -361,19 +361,19 @@ func (s *syncer) beginSourceCachePage( // loss (the stamped rows may still exist, e.g. a partially carried // file). The hard error below is reserved for a replay that // produces nothing. - _, entryFound, err := s.sourceCache.prev.LookupSourceCacheEntry(ctx, kind, page.scopeHash) + _, entryFound, err := s.sourceCache.prev.LookupSourceCacheEntry(ctx, kind, page.scopeKey) if err != nil { - return ctx, nil, fmt.Errorf("source cache: error reading previous manifest for scope %q: %w", page.scopeHash, err) + return ctx, nil, fmt.Errorf("source cache: error reading previous manifest for scope %q: %w", page.scopeKey, err) } if !entryFound { ctxzap.Extract(ctx).Warn("source cache: replay requested for scope with no previous manifest entry", - zap.String("scope_hash", page.scopeHash)) + zap.String("scope_key", page.scopeKey)) } replayStart := time.Now() - res, err := s.sourceCache.current.ReplaySourceCache(ctx, s.sourceCache.prevReader, kind, page.scopeHash) + res, err := s.sourceCache.current.ReplaySourceCache(ctx, s.sourceCache.prevReader, kind, page.scopeKey) s.state.AddStepDuration("source_cache_replay", time.Since(replayStart)) if err != nil { - return ctx, nil, fmt.Errorf("source cache: replay for scope %q failed: %w", page.scopeHash, err) + return ctx, nil, fmt.Errorf("source cache: replay for scope %q failed: %w", page.scopeKey, err) } // Stats are stashed on the page and recorded in // finishSourceCachePage so a page re-run after failing between @@ -385,13 +385,13 @@ func (s *syncer) beginSourceCachePage( // a LATER page's scope annotation (delta rounds), and that // page must count toward ScopesReplayed's scope, not as a // freshly stamped one — see finishSourceCachePage. - s.sourceCache.markScopeReplayed(kind, page.scopeHash) + s.sourceCache.markScopeReplayed(kind, page.scopeKey) } if res.Rows == 0 && !entryFound { // The connector skipped row generation expecting a base that // does not exist anywhere in the previous file — this sync // would silently drop the scope's rows. - return ctx, nil, fmt.Errorf("source cache: replay for scope %q found no previous rows and no manifest entry; the connector replayed a scope it never looked up", page.scopeHash) + return ctx, nil, fmt.Errorf("source cache: replay for scope %q found no previous rows and no manifest entry; the connector replayed a scope it never looked up", page.scopeKey) } // Empty-vs-dropped discrimination via the previous file's scope // index: a legitimately empty scope has NO index entries @@ -403,14 +403,14 @@ func (s *syncer) beginSourceCachePage( return ctx, nil, fmt.Errorf( "source cache: replay for scope %q copied no rows but the previous file's scope index holds %d entries "+ "whose rows no longer carry this scope's stamp; refusing to silently drop the scope", - page.scopeHash, res.StaleSkipped) + page.scopeKey, res.StaleSkipped) } if res.StaleSkipped > 0 { // Partial staleness: some rows copied, some index entries were // dead. Not data loss by itself (the live rows carried their // true stamp), but a signal a writer skipped index cleanup. ctxzap.Extract(ctx).Warn("source cache: replay skipped stale scope-index entries", - zap.String("scope_hash", page.scopeHash), + zap.String("scope_key", page.scopeKey), zap.Int64("rows", res.Rows), zap.Int64("stale_skipped", res.StaleSkipped), ) @@ -434,14 +434,14 @@ func (s *syncer) beginSourceCachePage( } ctxzap.Extract(ctx).Debug("source cache replayed scope", zap.String("row_kind", string(kind)), - zap.String("scope_hash", page.scopeHash), + zap.String("scope_key", page.scopeKey), zap.Int64("rows", res.Rows), zap.Int("deleted_ids", len(page.deletedIDs)), zap.Int("deleted_principal_ids", len(page.deletedPrincipalIDs)), ) } - return sourcecache.WithScope(ctx, page.scopeHash), page, nil + return sourcecache.WithScope(ctx, page.scopeKey), page, nil } // finishSourceCachePage runs after the page's rows committed: applies @@ -465,36 +465,36 @@ func (s *syncer) finishSourceCachePage(ctx context.Context, page *sourceCachePag // connector-custom grant-id shapes (unreachable by the global // bounded delete) work, and the cost stays bounded by the // scope's size. - deleted, err := s.sourceCache.current.DeleteSourceCacheGrantsByIDInScope(ctx, page.scopeHash, page.deletedIDs) + deleted, err := s.sourceCache.current.DeleteSourceCacheGrantsByIDInScope(ctx, page.scopeKey, page.deletedIDs) if err != nil { - return fmt.Errorf("source cache: error applying grant deletions for scope %q: %w", page.scopeHash, err) + return fmt.Errorf("source cache: error applying grant deletions for scope %q: %w", page.scopeKey, err) } rowsDeleted += deleted ctxzap.Extract(ctx).Debug("source cache applied grant-id deletions", - zap.String("scope_hash", page.scopeHash), + zap.String("scope_key", page.scopeKey), zap.Int("tombstones", len(page.deletedIDs)), zap.Int64("rows_deleted", deleted), ) } else if err := s.sourceCache.current.DeleteSourceCacheRows(ctx, page.kind, page.deletedIDs); err != nil { - return fmt.Errorf("source cache: error applying deletions for scope %q: %w", page.scopeHash, err) + return fmt.Errorf("source cache: error applying deletions for scope %q: %w", page.scopeKey, err) } } if len(page.deletedPrincipalIDs) > 0 { - deleted, err := s.sourceCache.current.DeleteSourceCacheRowsInScope(ctx, page.kind, page.scopeHash, page.deletedPrincipalIDs) + deleted, err := s.sourceCache.current.DeleteSourceCacheRowsInScope(ctx, page.kind, page.scopeKey, page.deletedPrincipalIDs) if err != nil { - return fmt.Errorf("source cache: error applying scoped deletions for scope %q: %w", page.scopeHash, err) + return fmt.Errorf("source cache: error applying scoped deletions for scope %q: %w", page.scopeKey, err) } rowsDeleted += deleted ctxzap.Extract(ctx).Debug("source cache applied scoped deletions", zap.String("row_kind", string(page.kind)), - zap.String("scope_hash", page.scopeHash), + zap.String("scope_key", page.scopeKey), zap.Int("tombstones", len(page.deletedPrincipalIDs)), zap.Int64("rows_deleted", deleted), ) } if page.etag != "" { - if err := s.sourceCache.current.PutSourceCacheEntry(ctx, page.kind, page.scopeHash, page.etag); err != nil { - return fmt.Errorf("source cache: error writing manifest entry for scope %q: %w", page.scopeHash, err) + if err := s.sourceCache.current.PutSourceCacheEntry(ctx, page.kind, page.scopeKey, page.etag); err != nil { + return fmt.Errorf("source cache: error writing manifest entry for scope %q: %w", page.scopeKey, err) } } @@ -511,7 +511,7 @@ func (s *syncer) finishSourceCachePage(ctx context.Context, page *sourceCachePag stats.RowsReplayed = map[string]int64{string(page.kind): page.replayRows} stats.OverlayRows = int64(page.overlayRows) } - if page.etag != "" && !page.replayed && !s.sourceCache.scopeReplayedThisSync(page.kind, page.scopeHash) { + if page.etag != "" && !page.replayed && !s.sourceCache.scopeReplayedThisSync(page.kind, page.scopeKey) { // A cold/fresh page that persisted a validator: the denominator // of the replay hit ratio. The replayed-scope check keeps a // multi-page overlay round — whose new validator legally arrives diff --git a/pkg/sync/source_cache_continuation.go b/pkg/sync/source_cache_continuation.go index 35dd69cdd..a99516c7a 100644 --- a/pkg/sync/source_cache_continuation.go +++ b/pkg/sync/source_cache_continuation.go @@ -39,13 +39,13 @@ const ( // (bounces × ask size) stays coherent with this loop's enforcement. sourceCacheBounceCap = sourcecache.MaxLookupBouncesPerRequest - // maxSpawnCursorsPerResponse bounds SpawnCursors fan-out per response + // maxEnqueuePageTokensPerResponse bounds EnqueuePageTokens fan-out per response // (each spawned action persists in the checkpointed state token, so // unbounded fan-out bloats every checkpoint). Matches the proto - // max_items validate rule on SpawnCursors.page_tokens. - maxSpawnCursorsPerResponse = 1024 + // max_items validate rule on EnqueuePageTokens.page_tokens. + maxEnqueuePageTokensPerResponse = 1024 // maxSpawnCursorTokenBytes matches the page-token request field and - // the SpawnCursors item validation. Enforced before creating Actions: + // the EnqueuePageTokens item validation. Enforced before creating Actions: // action tokens are checkpointed before they are sent through the // later request field's validator. maxSpawnCursorTokenBytes = 1 << 20 @@ -148,8 +148,8 @@ func (s *syncer) withSourceCacheContinuation(ctx context.Context, op string, iss return fmt.Errorf("%s: connector sent a source-cache lookup ask on a request that carried no offer (connector must gate asks on SourceCacheLookupOffer)", op) } if attempt.rows > 0 || attempt.nextToken != "" || - attempt.annos.Contains(&v2.SourceCacheScope{}) || attempt.annos.Contains(&v2.SourceCacheReplay{}) || - attempt.annos.Contains(&v2.SpawnCursors{}) { + attempt.annos.Contains(&v2.SourceCacheRecord{}) || attempt.annos.Contains(&v2.SourceCacheReplay{}) || + attempt.annos.Contains(&v2.EnqueuePageTokens{}) { return fmt.Errorf("%s: source-cache lookup ask response must carry ONLY the ask: "+ "no rows, no next page token, no scope/replay annotations, no spawned cursors "+ "(spawn on the re-invoked request's real response instead)", op) @@ -169,7 +169,7 @@ func (s *syncer) withSourceCacheContinuation(ctx context.Context, op string, iss budget := sourceCacheAnswerBudget for _, a := range ordered { - budget -= len(a.ETag) + budget -= len(a.CacheValidator) } newAsked, newFound, newNotFound, newTruncated := 0, 0, 0, 0 for _, q := range queries { @@ -177,7 +177,7 @@ func (s *syncer) withSourceCacheContinuation(ctx context.Context, op string, iss continue } newAsked++ - entry, ok, err := s.sourceCache.lookup.LookupPreviousSourceCache(ctx, q.RowKind, q.ScopeHash) + entry, ok, err := s.sourceCache.lookup.Lookup(ctx, q.RowKind, q.ScopeKey) if err != nil { return fmt.Errorf("%s: resolving source-cache lookup ask: %w", op, err) } @@ -187,7 +187,7 @@ func (s *syncer) withSourceCacheContinuation(ctx context.Context, op string, iss newNotFound++ continue } - if len(entry.ETag) > budget { + if len(entry.CacheValidator) > budget { // Found, but the etag does not fit the remaining budget. // Degrade to an explicit not-found so the connector // fetches this scope fresh (cold-correct). See the @@ -199,9 +199,9 @@ func (s *syncer) withSourceCacheContinuation(ctx context.Context, op string, iss newTruncated++ continue } - budget -= len(entry.ETag) + budget -= len(entry.CacheValidator) seen[q] = true - ordered = append(ordered, sourcecache.Answer{Query: q, Found: true, ETag: entry.ETag}) + ordered = append(ordered, sourcecache.Answer{Query: q, Found: true, CacheValidator: entry.CacheValidator}) newFound++ } asked += newAsked diff --git a/pkg/sync/source_cache_continuation_e2e_test.go b/pkg/sync/source_cache_continuation_e2e_test.go index 5cc066cc4..5a50a7fc4 100644 --- a/pkg/sync/source_cache_continuation_e2e_test.go +++ b/pkg/sync/source_cache_continuation_e2e_test.go @@ -164,16 +164,16 @@ func (s *lambdaE2ESyncer) Grants(ctx context.Context, r *v2.Resource, opts rs.Sy uids := append([]string(nil), s.members[gid]...) s.mu.Unlock() - entry, found, err := opts.SourceCache.LookupPreviousSourceCache(ctx, sourcecache.RowKindGrants, scope) + entry, found, err := opts.SourceCache.Lookup(ctx, sourcecache.RowKindGrants, scope) if err != nil { return nil, nil, fmt.Errorf("revalidating %s: %w", gid, err) } ret := &rs.SyncOpResults{Annotations: annotations.Annotations{}} - if found && entry.ETag == etag { + if found && entry.CacheValidator == etag { s.mu.Lock() s.replays++ s.mu.Unlock() - ret.Annotations.Update(v2.SourceCacheReplay_builder{ScopeHash: scope, Etag: etag}.Build()) + ret.Annotations.Update(v2.SourceCacheReplay_builder{ScopeKey: scope, CacheValidator: etag}.Build()) return nil, ret, nil } @@ -188,7 +188,7 @@ func (s *lambdaE2ESyncer) Grants(ctx context.Context, r *v2.Resource, opts rs.Sy } grants = append(grants, gt.NewGrant(r, "member", u.GetId())) } - ret.Annotations.Update(v2.SourceCacheScope_builder{ScopeHash: scope, Etag: etag}.Build()) + ret.Annotations.Update(v2.SourceCacheRecord_builder{ScopeKey: scope, CacheValidator: etag}.Build()) return grants, ret, nil } diff --git a/pkg/sync/source_cache_continuation_test.go b/pkg/sync/source_cache_continuation_test.go index 4777e088d..66caddc3e 100644 --- a/pkg/sync/source_cache_continuation_test.go +++ b/pkg/sync/source_cache_continuation_test.go @@ -71,7 +71,7 @@ type continuationMockConnector struct { askAlways bool // never satisfied: bounce-cap test askWithRows bool // protocol violation: ask + rows askNoOffer bool // protocol violation: ask on offerless request - askWithSpawn bool // protocol violation: ask + SpawnCursors + askWithSpawn bool // protocol violation: ask + EnqueuePageTokens asks int replayPages int @@ -96,7 +96,7 @@ func (mc *continuationMockConnector) askResponse(scope string, rows []*v2.Grant) return v2.GrantsServiceListGrantsResponse_builder{ List: rows, Annotations: annotations.New(sourcecache.AskProto([]sourcecache.Query{ - {RowKind: sourcecache.RowKindGrants, ScopeHash: scope}, + {RowKind: sourcecache.RowKindGrants, ScopeKey: scope}, })), }.Build() } @@ -134,7 +134,7 @@ func (mc *continuationMockConnector) ListGrants(ctx context.Context, in *v2.Gran if mc.askWithSpawn && !hasAnswers && hasOffer { resp := mc.askResponse(scope, nil) annos := annotations.Annotations(resp.GetAnnotations()) - annos.Update(v2.SpawnCursors_builder{PageTokens: []string{"bogus"}}.Build()) + annos.Update(v2.EnqueuePageTokens_builder{PageTokens: []string{"bogus"}}.Build()) resp.SetAnnotations(annos) return resp, nil } @@ -145,17 +145,17 @@ func (mc *continuationMockConnector) ListGrants(ctx context.Context, in *v2.Gran return nil, answersErr } for _, a := range answers { - if a.ScopeHash != scope || a.RowKind != sourcecache.RowKindGrants { + if a.ScopeKey != scope || a.RowKind != sourcecache.RowKindGrants { continue } - if a.Found && a.ETag == etag { + if a.Found && a.CacheValidator == etag { mc.mu.Lock() mc.replayPages++ mc.mu.Unlock() return v2.GrantsServiceListGrantsResponse_builder{ Annotations: annotations.New(v2.SourceCacheReplay_builder{ - ScopeHash: scope, - Etag: etag, + ScopeKey: scope, + CacheValidator: etag, }.Build()), }.Build(), nil } @@ -171,9 +171,9 @@ func (mc *continuationMockConnector) ListGrants(ctx context.Context, in *v2.Gran mc.mu.Unlock() return v2.GrantsServiceListGrantsResponse_builder{ List: rows, - Annotations: annotations.New(v2.SourceCacheScope_builder{ - ScopeHash: scope, - Etag: etag, + Annotations: annotations.New(v2.SourceCacheRecord_builder{ + ScopeKey: scope, + CacheValidator: etag, }.Build()), }.Build(), nil } @@ -267,7 +267,7 @@ func TestSourceCacheContinuation_WarmReplay(t *testing.T) { // spawns sibling cursors whose tokens CARRY the verdicts. Siblings serve // replay/cold from their token alone — zero asks, zero bounces. This pins // the two contract commitments from the review briefs: the ask/answer loop -// composes with a phase-2 SpawnCursors response, and a validator resolved +// composes with a phase-2 EnqueuePageTokens response, and a validator resolved // in an EARLIER action of the same sync legally rides page tokens. type verdictSpawnMock struct { *mockConnector @@ -323,9 +323,9 @@ func (mc *verdictSpawnMock) servePage(gid string, page int, chain bool) *v2.Gran mc.colds++ return v2.GrantsServiceListGrantsResponse_builder{ List: grants, - Annotations: annotations.New(v2.SourceCacheScope_builder{ - ScopeHash: vsPageScope(gid, page), - Etag: vsPageEtag(gid, page, rows), + Annotations: annotations.New(v2.SourceCacheRecord_builder{ + ScopeKey: vsPageScope(gid, page), + CacheValidator: vsPageEtag(gid, page, rows), }.Build()), NextPageToken: next, }.Build() @@ -355,8 +355,8 @@ func (mc *verdictSpawnMock) ListGrants(_ context.Context, in *v2.GrantsServiceLi mc.replays++ return v2.GrantsServiceListGrantsResponse_builder{ Annotations: annotations.New(v2.SourceCacheReplay_builder{ - ScopeHash: vsPageScope(gid, page), - Etag: etag, + ScopeKey: vsPageScope(gid, page), + CacheValidator: etag, }.Build()), }.Build(), nil } @@ -385,7 +385,7 @@ func (mc *verdictSpawnMock) ListGrants(_ context.Context, in *v2.GrantsServiceLi mc.originAsks++ queries := make([]sourcecache.Query, 0, prev) for p := 0; p < prev; p++ { - queries = append(queries, sourcecache.Query{RowKind: sourcecache.RowKindGrants, ScopeHash: vsPageScope(gid, p)}) + queries = append(queries, sourcecache.Query{RowKind: sourcecache.RowKindGrants, ScopeKey: vsPageScope(gid, p)}) } return v2.GrantsServiceListGrantsResponse_builder{ Annotations: annotations.New(sourcecache.AskProto(queries)), @@ -402,15 +402,15 @@ func (mc *verdictSpawnMock) ListGrants(_ context.Context, in *v2.GrantsServiceLi return nil, answersErr } for _, a := range answers { - byScope[a.ScopeHash] = a + byScope[a.ScopeKey] = a } var resp *v2.GrantsServiceListGrantsResponse - if a, ok := byScope[vsPageScope(gid, 0)]; ok && a.Found && a.ETag == vsPageEtag(gid, 0, mc.pageRows(gid, 0)) { + if a, ok := byScope[vsPageScope(gid, 0)]; ok && a.Found && a.CacheValidator == vsPageEtag(gid, 0, mc.pageRows(gid, 0)) { mc.replays++ resp = v2.GrantsServiceListGrantsResponse_builder{ Annotations: annotations.New(v2.SourceCacheReplay_builder{ - ScopeHash: vsPageScope(gid, 0), - Etag: a.ETag, + ScopeKey: vsPageScope(gid, 0), + CacheValidator: a.CacheValidator, }.Build()), }.Build() } else { @@ -423,18 +423,18 @@ func (mc *verdictSpawnMock) ListGrants(_ context.Context, in *v2.GrantsServiceLi if a.Found { found = 1 } - tokens = append(tokens, fmt.Sprintf("page=%d|%d|%s", p, found, a.ETag)) + tokens = append(tokens, fmt.Sprintf("page=%d|%d|%s", p, found, a.CacheValidator)) } if len(tokens) > 0 { annos := annotations.Annotations(resp.GetAnnotations()) - annos.Update(v2.SpawnCursors_builder{PageTokens: tokens}.Build()) + annos.Update(v2.EnqueuePageTokens_builder{PageTokens: tokens}.Build()) resp.SetAnnotations(annos) resp.SetNextPageToken("") } return resp, nil } -func TestSourceCacheContinuation_Phase2SpawnCursors(t *testing.T) { +func TestSourceCacheContinuation_Phase2EnqueuePageTokens(t *testing.T) { for _, workers := range []int{0, 4} { t.Run(fmt.Sprintf("workers=%d", workers), func(t *testing.T) { ctx, err := logging.Init(context.Background()) @@ -527,7 +527,7 @@ func TestSourceCacheContinuation_BounceCapFails(t *testing.T) { // complete, because the cap is per REQUEST (same page token) and every // NextPageToken advance is a fresh request. The final planning page spawns // the real cursor; it must spawn exactly once (the ask response never -// reaches SpawnCursors processing; the re-invoke serves the same page). +// reaches EnqueuePageTokens processing; the re-invoke serves the same page). type multiPagePlannerMock struct { *mockConnector @@ -590,7 +590,7 @@ func (mc *multiPagePlannerMock) ListGrants(_ context.Context, in *v2.GrantsServi mc.asks++ return v2.GrantsServiceListGrantsResponse_builder{ Annotations: annotations.New(sourcecache.AskProto([]sourcecache.Query{ - {RowKind: sourcecache.RowKindGrants, ScopeHash: scope}, + {RowKind: sourcecache.RowKindGrants, ScopeKey: scope}, })), }.Build(), nil } @@ -601,7 +601,7 @@ func (mc *multiPagePlannerMock) ListGrants(_ context.Context, in *v2.GrantsServi return nil, answersErr } for _, a := range answers { - if a.ScopeHash == scope { + if a.ScopeKey == scope { mc.answersScope = scope } } @@ -611,7 +611,7 @@ func (mc *multiPagePlannerMock) ListGrants(_ context.Context, in *v2.GrantsServi if page == plannerPages-1 { mc.spawns++ return v2.GrantsServiceListGrantsResponse_builder{ - Annotations: annotations.New(v2.SpawnCursors_builder{PageTokens: []string{"cursor"}}.Build()), + Annotations: annotations.New(v2.EnqueuePageTokens_builder{PageTokens: []string{"cursor"}}.Build()), }.Build(), nil } return v2.GrantsServiceListGrantsResponse_builder{ @@ -670,8 +670,8 @@ func TestSourceCacheContinuation_MultiPagePlannerBouncesPerRequest(t *testing.T) // fatEtagLookup finds every scope with the same (large) etag. type fatEtagLookup struct{ etag string } -func (f fatEtagLookup) LookupPreviousSourceCache(context.Context, sourcecache.RowKind, string) (sourcecache.Entry, bool, error) { - return sourcecache.Entry{ETag: f.etag}, true, nil +func (f fatEtagLookup) Lookup(context.Context, sourcecache.RowKind, string) (sourcecache.Entry, bool, error) { + return sourcecache.Entry{CacheValidator: f.etag}, true, nil } // TestSourceCacheContinuation_AnswerBudgetDegradesToCold pins the answer @@ -700,7 +700,7 @@ func TestSourceCacheContinuation_AnswerBudgetDegradesToCold(t *testing.T) { queries := make([]sourcecache.Query, 0, queryCount) for i := 0; i < queryCount; i++ { - queries = append(queries, sourcecache.Query{RowKind: sourcecache.RowKindGrants, ScopeHash: fmt.Sprintf("scope-%03d", i)}) + queries = append(queries, sourcecache.Query{RowKind: sourcecache.RowKindGrants, ScopeKey: fmt.Sprintf("scope-%03d", i)}) } turns := 0 @@ -735,7 +735,7 @@ func TestSourceCacheContinuation_AnswerBudgetDegradesToCold(t *testing.T) { require.Zero(t, stats.LookupAnsweredNotFound, "budget degradation counts as truncated, not as a genuine miss") } -func TestSourceCacheContinuation_AskWithSpawnCursorsFails(t *testing.T) { +func TestSourceCacheContinuation_AskWithEnqueuePageTokensFails(t *testing.T) { ctx, err := logging.Init(context.Background()) require.NoError(t, err) tmpDir := t.TempDir() @@ -748,7 +748,7 @@ func TestSourceCacheContinuation_AskWithSpawnCursorsFails(t *testing.T) { err = runContinuationSync(ctx, t, mc, filepath.Join(tmpDir, "sync2.c1z"), sync1, tmpDir, 0) require.Error(t, err) require.Contains(t, err.Error(), "no spawned cursors", - "SpawnCursors alongside an ask is a connector bug and must fail loudly") + "EnqueuePageTokens alongside an ask is a connector bug and must fail loudly") } func TestSourceCacheContinuation_AskWithRowsFails(t *testing.T) { @@ -846,7 +846,7 @@ func (mc *multiPagePlannerEntitlementsMock) ListEntitlements( mc.asks++ return v2.EntitlementsServiceListEntitlementsResponse_builder{ Annotations: annotations.New(sourcecache.AskProto([]sourcecache.Query{ - {RowKind: sourcecache.RowKindEntitlements, ScopeHash: scope}, + {RowKind: sourcecache.RowKindEntitlements, ScopeKey: scope}, })), }.Build(), nil } @@ -857,7 +857,7 @@ func (mc *multiPagePlannerEntitlementsMock) ListEntitlements( return nil, answersErr } for _, a := range answers { - if a.ScopeHash == scope { + if a.ScopeKey == scope { mc.answersScope = scope } } @@ -866,7 +866,7 @@ func (mc *multiPagePlannerEntitlementsMock) ListEntitlements( if page == plannerPages-1 { mc.spawns++ return v2.EntitlementsServiceListEntitlementsResponse_builder{ - Annotations: annotations.New(v2.SpawnCursors_builder{PageTokens: []string{"cursor"}}.Build()), + Annotations: annotations.New(v2.EnqueuePageTokens_builder{PageTokens: []string{"cursor"}}.Build()), }.Build(), nil } return v2.EntitlementsServiceListEntitlementsResponse_builder{ diff --git a/pkg/sync/source_cache_equivalence_soak_test.go b/pkg/sync/source_cache_equivalence_soak_test.go new file mode 100644 index 000000000..8d12931a2 --- /dev/null +++ b/pkg/sync/source_cache_equivalence_soak_test.go @@ -0,0 +1,31 @@ +package sync //nolint:revive,nolintlint // we can't change the package name for backwards compatibility + +// Soak sweep for the replay-equivalence harness: 20 additional seeds +// beyond TestSourceCache_ReplayEquivalence's fixed five, alternating +// worker counts and continuation topology. Gated behind BATON_SOAK so +// ordinary CI runs don't pay ~20x harness cost. +// +// TEMPORARY VERIFICATION FILE — safe to delete; nothing references it. + +import ( + "fmt" + "os" + "testing" +) + +func TestSourceCache_ReplayEquivalenceSoak(t *testing.T) { + if os.Getenv("BATON_SOAK") == "" { + t.Skip("set BATON_SOAK=1 to run the seed soak") + } + for seed := int64(6); seed <= 25; seed++ { + workers := 0 + if seed%2 == 0 { + workers = 4 + } + continuation := seed%3 == 0 + name := fmt.Sprintf("workers=%d/seed=%d/continuation=%v", workers, seed, continuation) + t.Run(name, func(t *testing.T) { + runReplayEquivalenceScenario(t, workers, seed, continuation) + }) + } +} diff --git a/pkg/sync/source_cache_equivalence_test.go b/pkg/sync/source_cache_equivalence_test.go index 2eb6b38ca..058ab7c23 100644 --- a/pkg/sync/source_cache_equivalence_test.go +++ b/pkg/sync/source_cache_equivalence_test.go @@ -59,7 +59,7 @@ package sync //nolint:revive,nolintlint // we can't change the package name for // parity test in source_cache_sideeffects_test.go); // - the ask/answer lookup continuation topology (covered by // source_cache_continuation_test.go / _e2e_test.go); -// - SpawnCursors / type-scoped grants and entitlements +// - EnqueuePageTokens / type-scoped grants and entitlements // (spawn_cursors_test.go, type_scoped_grants_test.go, // type_scoped_entitlements_test.go); the team topology below also // exercises both type-scoped shapes in the differential chain; @@ -122,7 +122,7 @@ var ( equivRepoRT = v2.ResourceType_builder{Id: "eq_repo", DisplayName: "Repo"}.Build() // equivTeamRT is TYPE-SCOPED for both grants and entitlements: no // per-resource pages; one planner call per phase spawns chunk - // cursors (SpawnCursors), each chunk a scope covering two teams — + // cursors (EnqueuePageTokens), each chunk a scope covering two teams — // the Entra/Okta planner shape. equivTeamRT = v2.ResourceType_builder{ Id: "eq_team", @@ -443,7 +443,7 @@ type equivCounters struct { tombsPrincipal int childPageCalls int injectedFailures int - plannerPages int // type-scoped planning calls (SpawnCursors emitters) + plannerPages int // type-scoped planning calls (EnqueuePageTokens emitters) chunkPages304 int // spawned chunk cursors that replayed chunkPagesFresh int // spawned chunk cursors fetched fresh askPages int // continuation phase-1 responses (lookup asks) @@ -583,14 +583,14 @@ func (c *equivConnector) revalidate(ctx context.Context, lookup sourcecache.Look if c.forceCold { return equivFresh, nil } - entry, found, err := lookup.LookupPreviousSourceCache(ctx, kind, scope) + entry, found, err := lookup.Lookup(ctx, kind, scope) if err != nil { return equivFresh, err } if !found { return equivFresh, nil } - switch entry.ETag { + switch entry.CacheValidator { case equivEtag(version): return equivCurrent, nil case equivEtag(version - 1): @@ -613,16 +613,16 @@ func (c *equivConnector) snapshotCounters() equivCounters { } func equivScopeAnnos(scope string, version int) annotations.Annotations { - return annotations.New(v2.SourceCacheScope_builder{ - ScopeHash: sourcecache.HashScope(scope), - Etag: equivEtag(version), + return annotations.New(v2.SourceCacheRecord_builder{ + ScopeKey: sourcecache.HashScope(scope), + CacheValidator: equivEtag(version), }.Build()) } func equivReplayAnnos(scope string, version int) annotations.Annotations { return annotations.New(v2.SourceCacheReplay_builder{ - ScopeHash: sourcecache.HashScope(scope), - Etag: equivEtag(version), + ScopeKey: sourcecache.HashScope(scope), + CacheValidator: equivEtag(version), }.Build()) } @@ -822,8 +822,8 @@ func (c *equivConnector) usersPageMulti(ctx context.Context, lookup sourcecache. c.count(func(ct *equivCounters) { ct.pagesFreshInterim++ }) return v2.ResourcesServiceListResourcesResponse_builder{ List: buildRows(userIDs[:half]), - Annotations: annotations.New(v2.SourceCacheScope_builder{ - ScopeHash: sourcecache.HashScope(scope), + Annotations: annotations.New(v2.SourceCacheRecord_builder{ + ScopeKey: sourcecache.HashScope(scope), // No etag: the validator arrives on the final page. }.Build()), NextPageToken: "users:2", @@ -972,7 +972,7 @@ func (c *equivConnector) ListGrants(ctx context.Context, in *v2.GrantsServiceLis } // teamEntitlementsPage mirrors teamGrantsPage for type-scoped -// entitlements: planning SpawnCursors, then one fresh-or-304 scope per +// entitlements: planning EnqueuePageTokens, then one fresh-or-304 scope per // chunk of two teams. Entitlement rows are stable (one "member" per // team); version stays 0 so warm rounds are pure 304s after the cold // fetch — exercising type-scoped entitlement replay in the differential @@ -988,7 +988,7 @@ func (c *equivConnector) teamEntitlementsPage(ctx context.Context, lookup source tokens = append(tokens, fmt.Sprintf("ent-chunk:%d", i)) } return v2.EntitlementsServiceListEntitlementsResponse_builder{ - Annotations: annotations.New(v2.SpawnCursors_builder{PageTokens: tokens}.Build()), + Annotations: annotations.New(v2.EnqueuePageTokens_builder{PageTokens: tokens}.Build()), }.Build(), nil } var chunk int @@ -1022,7 +1022,7 @@ func (c *equivConnector) teamEntitlementsPage(ctx context.Context, lookup source } // teamGrantsPage is the type-scoped planner shape: the planning call -// (empty page token) emits SpawnCursors — one token per chunk of two +// (empty page token) emits EnqueuePageTokens — one token per chunk of two // teams — and each spawned cursor serves its chunk as a fresh-or-304 // scope. func (c *equivConnector) teamGrantsPage(ctx context.Context, lookup sourcecache.Lookup, pageToken string) (*v2.GrantsServiceListGrantsResponse, error) { @@ -1036,7 +1036,7 @@ func (c *equivConnector) teamGrantsPage(ctx context.Context, lookup sourcecache. tokens = append(tokens, fmt.Sprintf("chunk:%d", i)) } return v2.GrantsServiceListGrantsResponse_builder{ - Annotations: annotations.New(v2.SpawnCursors_builder{PageTokens: tokens}.Build()), + Annotations: annotations.New(v2.EnqueuePageTokens_builder{PageTokens: tokens}.Build()), }.Build(), nil } var chunk int @@ -1168,8 +1168,8 @@ func (c *equivConnector) groupGrantsPage(ctx context.Context, lookup sourcecache return v2.GrantsServiceListGrantsResponse_builder{ List: firstAdds, Annotations: annotations.New(v2.SourceCacheReplay_builder{ - ScopeHash: sourcecache.HashScope(scope), - Overlay: true, + ScopeKey: sourcecache.HashScope(scope), + Overlay: true, }.Build()), NextPageToken: "ov:2", }.Build(), nil @@ -1181,9 +1181,9 @@ func (c *equivConnector) groupGrantsPage(ctx context.Context, lookup sourcecache if len(adds) > 1 { restAdds = adds[1:] } - finalAnno := v2.SourceCacheScope_builder{ - ScopeHash: sourcecache.HashScope(scope), - Etag: equivEtag(version), + finalAnno := v2.SourceCacheRecord_builder{ + ScopeKey: sourcecache.HashScope(scope), + CacheValidator: equivEtag(version), }.Build() if canonical { ids := make([]string, 0, len(deltaRemoves)) @@ -1203,12 +1203,12 @@ func (c *equivConnector) groupGrantsPage(ctx context.Context, lookup sourcecache } c.count(func(ct *equivCounters) { ct.pagesOverlay++ }) replay := v2.SourceCacheReplay_builder{ - ScopeHash: sourcecache.HashScope(scope), - Etag: equivEtag(version), - Overlay: true, + ScopeKey: sourcecache.HashScope(scope), + CacheValidator: equivEtag(version), + Overlay: true, }.Build() - scopeAnno := v2.SourceCacheScope_builder{ - ScopeHash: sourcecache.HashScope(scope), + scopeAnno := v2.SourceCacheRecord_builder{ + ScopeKey: sourcecache.HashScope(scope), }.Build() if canonical { ids := make([]string, 0, len(deltaRemoves)) @@ -1322,8 +1322,8 @@ type c1zSnapshot struct { // grantsByEntResource: entitlement-resource "rt/id" -> grant ids via // the by_entitlement_resource index read path. grantsByEntResource map[string][]string - // manifest ("row_kind\x00scope_hash" -> etag) and scopeIndexCounts - // (row kind -> scope hash -> index entries): the source-cache state + // manifest ("row_kind\x00scope_key" -> etag) and scopeIndexCounts + // (row kind -> scope key -> index entries): the source-cache state // invisible to v2 reads but load-bearing for the NEXT sync's replay. manifest map[string]string scopeIndexCounts map[string]map[string]int @@ -1420,7 +1420,7 @@ func snapshotC1z(ctx context.Context, t *testing.T, path string) *c1zSnapshot { require.True(t, ok, "store must expose its pebble engine for source-cache inspection") snap.manifest, err = engineHolder.PebbleEngine().SourceCacheManifestSnapshot(ctx) require.NoError(t, err) - snap.scopeIndexCounts, err = engineHolder.PebbleEngine().SourceCacheScopeIndexSnapshot(ctx) + snap.scopeIndexCounts, err = engineHolder.PebbleEngine().SourceScopeIndexSnapshot(ctx) require.NoError(t, err) runLister, ok := any(reopen).(interface { ListSyncRuns(ctx context.Context, pageToken string, pageSize uint32) ([]*c1zstore.SyncRun, string, error) diff --git a/pkg/sync/source_cache_sideeffects_test.go b/pkg/sync/source_cache_sideeffects_test.go index 6e11f5da7..32f373848 100644 --- a/pkg/sync/source_cache_sideeffects_test.go +++ b/pkg/sync/source_cache_sideeffects_test.go @@ -115,26 +115,26 @@ func (c *childReplayConnector) ListResources(ctx context.Context, in *v2.Resourc if lookup == nil { lookup = sourcecache.NoopLookup{} } - entry, found, err := lookup.LookupPreviousSourceCache(ctx, sourcecache.RowKindResources, scope) + entry, found, err := lookup.Lookup(ctx, sourcecache.RowKindResources, scope) if err != nil { return nil, err } - if found && entry.ETag == etag { + if found && entry.CacheValidator == etag { c.mu.Lock() c.resourceLookupHits++ c.mu.Unlock() return v2.ResourcesServiceListResourcesResponse_builder{ Annotations: annotations.New(v2.SourceCacheReplay_builder{ - ScopeHash: scope, - Etag: etag, + ScopeKey: scope, + CacheValidator: etag, }.Build()), }.Build(), nil } return v2.ResourcesServiceListResourcesResponse_builder{ List: c.parents, - Annotations: annotations.New(v2.SourceCacheScope_builder{ - ScopeHash: scope, - Etag: etag, + Annotations: annotations.New(v2.SourceCacheRecord_builder{ + ScopeKey: scope, + CacheValidator: etag, }.Build()), }.Build(), nil case c.childType.GetId(): @@ -337,18 +337,18 @@ func (c *insertResourceGrantsConnector) ListGrants(ctx context.Context, in *v2.G if lookup == nil { lookup = sourcecache.NoopLookup{} } - entry, found, err := lookup.LookupPreviousSourceCache(ctx, sourcecache.RowKindGrants, scope) + entry, found, err := lookup.Lookup(ctx, sourcecache.RowKindGrants, scope) if err != nil { return nil, err } - if found && entry.ETag == currentEtag { + if found && entry.CacheValidator == currentEtag { c.mu.Lock() c.lookupHits++ c.mu.Unlock() return v2.GrantsServiceListGrantsResponse_builder{ Annotations: annotations.New(v2.SourceCacheReplay_builder{ - ScopeHash: scope, - Etag: currentEtag, + ScopeKey: scope, + CacheValidator: currentEtag, }.Build()), }.Build(), nil } @@ -358,7 +358,7 @@ func (c *insertResourceGrantsConnector) ListGrants(ctx context.Context, in *v2.G return v2.GrantsServiceListGrantsResponse_builder{ List: c.insertGrants, Annotations: annotations.New( - v2.SourceCacheScope_builder{ScopeHash: scope, Etag: currentEtag}.Build(), + v2.SourceCacheRecord_builder{ScopeKey: scope, CacheValidator: currentEtag}.Build(), &v2.InsertResourceGrants{}, ), }.Build(), nil @@ -464,18 +464,18 @@ func (c *exclusionGroupReplayConnector) ListEntitlements( if lookup == nil { lookup = sourcecache.NoopLookup{} } - entry, found, err := lookup.LookupPreviousSourceCache(ctx, sourcecache.RowKindEntitlements, scope) + entry, found, err := lookup.Lookup(ctx, sourcecache.RowKindEntitlements, scope) if err != nil { return nil, err } - if found && entry.ETag == etag { + if found && entry.CacheValidator == etag { c.mu.Lock() c.entLookupHits++ c.mu.Unlock() return v2.EntitlementsServiceListEntitlementsResponse_builder{ Annotations: annotations.New(v2.SourceCacheReplay_builder{ - ScopeHash: scope, - Etag: etag, + ScopeKey: scope, + CacheValidator: etag, }.Build()), }.Build(), nil } @@ -484,9 +484,9 @@ func (c *exclusionGroupReplayConnector) ListEntitlements( c.mu.Unlock() return v2.EntitlementsServiceListEntitlementsResponse_builder{ List: c.entDB[resourceID], - Annotations: annotations.New(v2.SourceCacheScope_builder{ - ScopeHash: scope, - Etag: etag, + Annotations: annotations.New(v2.SourceCacheRecord_builder{ + ScopeKey: scope, + CacheValidator: etag, }.Build()), }.Build(), nil } @@ -763,11 +763,11 @@ func TestSourceCacheContinuation_AnswerAccumulationBeyondOneAsk(t *testing.T) { bigAsk := make([]sourcecache.Query, 0, 4096) for i := 0; i < 4096; i++ { bigAsk = append(bigAsk, sourcecache.Query{ - RowKind: sourcecache.RowKindGrants, - ScopeHash: fmt.Sprintf("scope-%04d", i), + RowKind: sourcecache.RowKindGrants, + ScopeKey: fmt.Sprintf("scope-%04d", i), }) } - lateAsk := []sourcecache.Query{{RowKind: sourcecache.RowKindGrants, ScopeHash: "late-scope"}} + lateAsk := []sourcecache.Query{{RowKind: sourcecache.RowKindGrants, ScopeKey: "late-scope"}} err = s.withSourceCacheContinuation(ctx, "test-op", func(extra annotations.Annotations, attempt int) (listAttempt, error) { answersMsg := &v2.SourceCacheLookupAnswers{} @@ -802,7 +802,7 @@ func TestSourceCacheContinuation_AnswerAccumulationBeyondOneAsk(t *testing.T) { // TestSourceCacheContinuation_OversizedAskFailsLoudly pins the intended // hard boundary on ONE ask: more than 4096 queries in a single ask is a -// connector bug (shard via SpawnCursors instead) and must fail the sync +// connector bug (shard via EnqueuePageTokens instead) and must fail the sync // with a descriptive error, not silently truncate. func TestSourceCacheContinuation_OversizedAskFailsLoudly(t *testing.T) { ctx, err := logging.Init(t.Context()) @@ -823,8 +823,8 @@ func TestSourceCacheContinuation_OversizedAskFailsLoudly(t *testing.T) { oversized := make([]sourcecache.Query, 0, 4097) for i := 0; i < 4097; i++ { oversized = append(oversized, sourcecache.Query{ - RowKind: sourcecache.RowKindGrants, - ScopeHash: fmt.Sprintf("scope-%04d", i), + RowKind: sourcecache.RowKindGrants, + ScopeKey: fmt.Sprintf("scope-%04d", i), }) } err = s.withSourceCacheContinuation(ctx, "test-op", func(extra annotations.Annotations, attempt int) (listAttempt, error) { diff --git a/pkg/sync/source_cache_stats.go b/pkg/sync/source_cache_stats.go index 0e43daee2..bbd00d585 100644 --- a/pkg/sync/source_cache_stats.go +++ b/pkg/sync/source_cache_stats.go @@ -40,7 +40,7 @@ type SourceCacheStats struct { TombstonePrincipals int64 `json:"tombstone_principals,omitempty"` RowsDeleted int64 `json:"rows_deleted,omitempty"` - // SpawnedCursors counts sibling actions enqueued via SpawnCursors + // SpawnedCursors counts sibling actions enqueued via EnqueuePageTokens // (pairs with the per-response cap). SpawnedCursors int64 `json:"spawned_cursors,omitempty"` @@ -51,7 +51,7 @@ type SourceCacheStats struct { // fetch) because their etags did not fit the per-request answer size // budget; nonzero means a connector is batching more fat-token scopes // per ask than the transport can carry and should shard via - // SpawnCursors. + // EnqueuePageTokens. LookupBounces int64 `json:"lookup_bounces,omitempty"` LookupRequestsBounced int64 `json:"lookup_requests_bounced,omitempty"` LookupScopesAsked int64 `json:"lookup_scopes_asked,omitempty"` diff --git a/pkg/sync/source_cache_test.go b/pkg/sync/source_cache_test.go index 54db60409..8b713da01 100644 --- a/pkg/sync/source_cache_test.go +++ b/pkg/sync/source_cache_test.go @@ -28,7 +28,7 @@ import ( // syncer installs (SetSourceCache) and either: // // - miss, or validator != current upstream etag: return fresh rows + -// SourceCacheScope (a 200), +// SourceCacheRecord (a 200), // - hit with matching etag: return no rows + SourceCacheReplay (a 304), // - hit in delta mode: return only changed rows + overlay replay with // tombstones and a rotated token (a Graph-style delta round). @@ -46,7 +46,7 @@ type sourceCacheMockConnector struct { deltaAdds map[string][]*v2.Grant deltaDeletedIDs map[string][]string // deltaDeletedPrincipals are bare principal-id tombstones, emitted on - // the SourceCacheScope annotation (the per-page channel) rather than + // the SourceCacheRecord annotation (the per-page channel) rather than // the replay annotation — exercising both proposal A and B paths. deltaDeletedPrincipals map[string][]string @@ -103,7 +103,7 @@ func (mc *sourceCacheMockConnector) ListGrants(ctx context.Context, in *v2.Grant if lookup == nil { lookup = sourcecache.NoopLookup{} } - entry, found, err := lookup.LookupPreviousSourceCache(ctx, sourcecache.RowKindGrants, scope) + entry, found, err := lookup.Lookup(ctx, sourcecache.RowKindGrants, scope) if err != nil { return nil, err } @@ -120,20 +120,20 @@ func (mc *sourceCacheMockConnector) ListGrants(ctx context.Context, in *v2.Grant List: adds, Annotations: annotations.New( v2.SourceCacheReplay_builder{ - ScopeHash: scope, - Etag: currentEtag, // new token for this round - Overlay: true, - DeletedIds: deleted, + ScopeKey: scope, + CacheValidator: currentEtag, // new token for this round + Overlay: true, + DeletedIds: deleted, }.Build(), - v2.SourceCacheScope_builder{ - ScopeHash: scope, + v2.SourceCacheRecord_builder{ + ScopeKey: scope, DeletedPrincipalIds: deletedPrincipals, }.Build(), ), }.Build(), nil } - if found && entry.ETag == currentEtag { + if found && entry.CacheValidator == currentEtag { // Conditional-request 304: nothing changed upstream. mc.mu.Lock() mc.lookupHits++ @@ -141,8 +141,8 @@ func (mc *sourceCacheMockConnector) ListGrants(ctx context.Context, in *v2.Grant return v2.GrantsServiceListGrantsResponse_builder{ List: []*v2.Grant{}, Annotations: annotations.New(v2.SourceCacheReplay_builder{ - ScopeHash: scope, - Etag: currentEtag, + ScopeKey: scope, + CacheValidator: currentEtag, }.Build()), }.Build(), nil } @@ -153,9 +153,9 @@ func (mc *sourceCacheMockConnector) ListGrants(ctx context.Context, in *v2.Grant mc.mu.Unlock() return v2.GrantsServiceListGrantsResponse_builder{ List: mc.grantDB[resourceID], - Annotations: annotations.New(v2.SourceCacheScope_builder{ - ScopeHash: scope, - Etag: currentEtag, + Annotations: annotations.New(v2.SourceCacheRecord_builder{ + ScopeKey: scope, + CacheValidator: currentEtag, }.Build()), }.Build(), nil } @@ -473,15 +473,15 @@ type stubSourceCacheStore struct { } func (s *stubSourceCacheStore) LookupSourceCacheEntry(context.Context, sourcecache.RowKind, string) (sourcecache.Entry, bool, error) { - return sourcecache.Entry{ETag: "prev-token"}, true, nil + return sourcecache.Entry{CacheValidator: "prev-token"}, true, nil } func (s *stubSourceCacheStore) ReplaySourceCache(context.Context, connectorstore.Reader, sourcecache.RowKind, string) (dotc1z.SourceCacheReplayResult, error) { return dotc1z.SourceCacheReplayResult{Rows: s.replayRows}, nil } -func (s *stubSourceCacheStore) PutSourceCacheEntry(_ context.Context, _ sourcecache.RowKind, scopeHash, _ string) error { - s.stamped = append(s.stamped, scopeHash) +func (s *stubSourceCacheStore) PutSourceCacheEntry(_ context.Context, _ sourcecache.RowKind, scopeKey, _ string) error { + s.stamped = append(s.stamped, scopeKey) return nil } @@ -512,7 +512,7 @@ func TestSourceCacheStats_OverlayRoundBooksOneHit(t *testing.T) { // Page 1: overlay replay, no etag yet (the delta token rotates on the // final page). One overlay row rides along. _, page1, err := s.beginSourceCachePage(ctx, sourcecache.RowKindGrants, - annotations.New(v2.SourceCacheReplay_builder{ScopeHash: scope, Overlay: true}.Build()), 1) + annotations.New(v2.SourceCacheReplay_builder{ScopeKey: scope, Overlay: true}.Build()), 1) require.NoError(t, err) require.True(t, page1.replayed) @@ -526,7 +526,7 @@ func TestSourceCacheStats_OverlayRoundBooksOneHit(t *testing.T) { // replay annotation — this is a stamp write, but of the round's own // scope, not a cold miss. _, page2, err := s.beginSourceCachePage(ctx, sourcecache.RowKindGrants, - annotations.New(v2.SourceCacheScope_builder{ScopeHash: scope, Etag: "delta-token-2"}.Build()), 0) + annotations.New(v2.SourceCacheRecord_builder{ScopeKey: scope, CacheValidator: "delta-token-2"}.Build()), 0) require.NoError(t, err) require.False(t, page2.replayed) require.NoError(t, s.finishSourceCachePage(ctx, page2)) @@ -535,7 +535,7 @@ func TestSourceCacheStats_OverlayRoundBooksOneHit(t *testing.T) { // A genuinely cold scope still counts as stamped. coldScope := sourcecache.HashScope("delta://users") _, page3, err := s.beginSourceCachePage(ctx, sourcecache.RowKindGrants, - annotations.New(v2.SourceCacheScope_builder{ScopeHash: coldScope, Etag: "etag-1"}.Build()), 3) + annotations.New(v2.SourceCacheRecord_builder{ScopeKey: coldScope, CacheValidator: "etag-1"}.Build()), 3) require.NoError(t, err) require.NoError(t, s.finishSourceCachePage(ctx, page3)) @@ -551,8 +551,8 @@ func TestSourceCacheStats_OverlayRoundBooksOneHit(t *testing.T) { // connector that violates the only-replay-what-you-looked-up invariant. type staticLookup struct{ etag string } -func (s staticLookup) LookupPreviousSourceCache(context.Context, sourcecache.RowKind, string) (sourcecache.Entry, bool, error) { - return sourcecache.Entry{ETag: s.etag}, true, nil +func (s staticLookup) Lookup(context.Context, sourcecache.RowKind, string) (sourcecache.Entry, bool, error) { + return sourcecache.Entry{CacheValidator: s.etag}, true, nil } // degradedLookupConnector shadows SetSourceCache so the syncer cannot diff --git a/pkg/sync/state.go b/pkg/sync/state.go index 70c4e38f0..b309884d1 100644 --- a/pkg/sync/state.go +++ b/pkg/sync/state.go @@ -218,7 +218,7 @@ type Action struct { ParentResourceTypeID string `json:"parent_resource_type_id,omitempty"` ParentResourceID string `json:"parent_resource_id,omitempty"` - // Spawned marks an action enqueued by a connector's SpawnCursors + // Spawned marks an action enqueued by a connector's EnqueuePageTokens // annotation (a sibling cursor) rather than by the syncer's own // planners. Progress accounting uses it: per-resource grant / // entitlement coverage counts a resource once, when its ORIGIN diff --git a/pkg/sync/syncer.go b/pkg/sync/syncer.go index 328ccedf4..bbe5510c8 100644 --- a/pkg/sync/syncer.go +++ b/pkg/sync/syncer.go @@ -1527,7 +1527,7 @@ func (s *syncer) shouldSkipEntitlements(ctx context.Context, r *v2.Resource) (bo // and pushes an action to fetch the entitlements for each resource. // Resource types annotated with TypeScopedEntitlements are excluded from the per-resource fan-out and // get a single type-scoped action instead (empty ResourceID); the connector enumerates the whole -// type, optionally spawning additional cursors via the SpawnCursors annotation. +// type, optionally spawning additional cursors via the EnqueuePageTokens annotation. func (s *syncer) SyncEntitlements(ctx context.Context, action *Action) error { ctx, span := uotel.StartWithLink(ctx, tracer, "syncer.SyncEntitlements") uotel.SetSyncIdentityAttrs(ctx, span) @@ -1642,7 +1642,7 @@ func (s *syncer) typeScopedEntitlementsResourceTypes(ctx context.Context) ([]str // syncEntitlementsForResource fetches the entitlements for a specific resource from the connector. // An action with an empty ResourceID is a TYPE-SCOPED entitlements cursor: the connector // enumerates entitlements for the whole resource type (no single resource backs the call), -// and may spawn sibling cursors via the SpawnCursors response annotation. +// and may spawn sibling cursors via the EnqueuePageTokens response annotation. // No span here: only call site is SyncEntitlements, which already owns a span. func (s *syncer) syncEntitlementsForResource(ctx context.Context, action *Action) error { typeScoped := action.ResourceID == "" @@ -1731,19 +1731,19 @@ func (s *syncer) syncEntitlementsForResource(ctx context.Context, action *Action s.counts.LogEntitlementsProgress(ctx, resourceID.GetResourceType()) } - // SpawnCursors: enqueue sibling cursors (type-scoped shards or + // EnqueuePageTokens: enqueue sibling cursors (type-scoped shards or // per-resource parallel pages). Same contracts as syncGrantsForResource. - spawn := &v2.SpawnCursors{} + spawn := &v2.EnqueuePageTokens{} hasSpawn, err := respAnnos.Pick(spawn) if err != nil { return fmt.Errorf("sync-entitlements: error parsing spawn-cursors annotation: %w", err) } var spawned []Action if hasSpawn { - if len(spawn.GetPageTokens()) > maxSpawnCursorsPerResponse { + if len(spawn.GetPageTokens()) > maxEnqueuePageTokensPerResponse { return fmt.Errorf( - "sync-entitlements: SpawnCursors carried %d page tokens (max %d per response); chain additional spawns across pages instead", - len(spawn.GetPageTokens()), maxSpawnCursorsPerResponse) + "sync-entitlements: EnqueuePageTokens carried %d page tokens (max %d per response); chain additional spawns across pages instead", + len(spawn.GetPageTokens()), maxEnqueuePageTokensPerResponse) } for _, tok := range spawn.GetPageTokens() { if tok == "" { @@ -1751,7 +1751,7 @@ func (s *syncer) syncEntitlementsForResource(ctx context.Context, action *Action } if len(tok) > maxSpawnCursorTokenBytes { return fmt.Errorf( - "sync-entitlements: SpawnCursors page token is %d bytes (max %d)", + "sync-entitlements: EnqueuePageTokens page token is %d bytes (max %d)", len(tok), maxSpawnCursorTokenBytes) } spawned = append(spawned, Action{ @@ -2197,7 +2197,7 @@ func (s *syncer) fixEntitlementGraphCycles(ctx context.Context, graph *expand.En // from the datastore, and pushes a new action to sync the grants for each individual resource. // Resource types annotated with TypeScopedGrants are excluded from the per-resource fan-out and // get a single type-scoped action instead (empty ResourceID); the connector enumerates the whole -// type, optionally spawning additional cursors via the SpawnCursors annotation. +// type, optionally spawning additional cursors via the EnqueuePageTokens annotation. func (s *syncer) SyncGrants(ctx context.Context, action *Action) error { ctx, span := uotel.StartWithLink(ctx, tracer, "syncer.SyncGrants") uotel.SetSyncIdentityAttrs(ctx, span) @@ -2309,7 +2309,7 @@ func (s *syncer) typeScopedGrantsResourceTypes(ctx context.Context) ([]string, e // syncGrantsForResource fetches the grants for a specific resource from the connector. // An action with an empty ResourceID is a TYPE-SCOPED grants cursor: the connector // enumerates grants for the whole resource type (no single resource backs the call), -// and may spawn sibling cursors via the SpawnCursors response annotation. +// and may spawn sibling cursors via the EnqueuePageTokens response annotation. // No span here: only call site is SyncGrants, which already owns a span. func (s *syncer) syncGrantsForResource(ctx context.Context, action *Action) error { typeScoped := action.ResourceID == "" @@ -2497,7 +2497,7 @@ func (s *syncer) syncGrantsForResource(ctx context.Context, action *Action) erro s.counts.LogGrantsProgress(ctx, resourceID.GetResourceType()) } - // SpawnCursors: the response may enqueue sibling cursors — each runs + // EnqueuePageTokens: the response may enqueue sibling cursors — each runs // as its own action, scheduled by the worker pool, rate-limited, and // checkpointed like any other pagination. // @@ -2512,7 +2512,7 @@ func (s *syncer) syncGrantsForResource(ctx context.Context, action *Action) erro // eagerly: the SDK assumes nothing about replay — a spawned page may // hit its lookup and replay, miss and fetch cold (page boundary // shifted since last sync), and may chain further via NextPageToken. - spawn := &v2.SpawnCursors{} + spawn := &v2.EnqueuePageTokens{} hasSpawn, err := respAnnos.Pick(spawn) if err != nil { return fmt.Errorf("sync-grants-for-resource: error parsing spawn-cursors annotation: %w", err) @@ -2526,10 +2526,10 @@ func (s *syncer) syncGrantsForResource(ctx context.Context, action *Action) erro // per-response and fail loudly, mirroring the lookup bounce cap. // Legitimate patterns sit far below this: Entra spawns ~35 chunk // cursors per planning page, GitHub one sibling per stored page. - if len(spawn.GetPageTokens()) > maxSpawnCursorsPerResponse { + if len(spawn.GetPageTokens()) > maxEnqueuePageTokensPerResponse { return fmt.Errorf( - "sync-grants-for-resource: SpawnCursors carried %d page tokens (max %d per response); chain additional spawns across pages instead", - len(spawn.GetPageTokens()), maxSpawnCursorsPerResponse) + "sync-grants-for-resource: EnqueuePageTokens carried %d page tokens (max %d per response); chain additional spawns across pages instead", + len(spawn.GetPageTokens()), maxEnqueuePageTokensPerResponse) } for _, tok := range spawn.GetPageTokens() { if tok == "" { @@ -2537,7 +2537,7 @@ func (s *syncer) syncGrantsForResource(ctx context.Context, action *Action) erro } if len(tok) > maxSpawnCursorTokenBytes { return fmt.Errorf( - "sync-grants-for-resource: SpawnCursors page token is %d bytes (max %d)", + "sync-grants-for-resource: EnqueuePageTokens page token is %d bytes (max %d)", len(tok), maxSpawnCursorTokenBytes) } spawned = append(spawned, Action{ @@ -3017,7 +3017,7 @@ func (s *syncer) processGrantsWithExternalPrincipals(ctx context.Context, princi if !annos.ContainsAny(&v2.ExternalResourceMatchAll{}, &v2.ExternalResourceMatch{}, &v2.ExternalResourceMatchID{}) { continue } - sourceScope := ga.SourceScopeHash + sourceScope := ga.SourceScopeKey addExpanded := func(rank int, g *v2.Grant) { entry := &expandedEntry{grant: g, scope: sourceScope, rank: rank} prev, ok := expanded[g.GetId()] diff --git a/pkg/sync/type_scoped_entitlements_test.go b/pkg/sync/type_scoped_entitlements_test.go index 4a574138d..c73aff503 100644 --- a/pkg/sync/type_scoped_entitlements_test.go +++ b/pkg/sync/type_scoped_entitlements_test.go @@ -1,7 +1,7 @@ package sync //nolint:revive,nolintlint // we can't change the package name for backwards compatibility // End-to-end harness for TYPE-SCOPED entitlements (v2.TypeScopedEntitlements + -// SpawnCursors), mirroring type_scoped_grants_test.go: entitlement rows for a +// EnqueuePageTokens), mirroring type_scoped_grants_test.go: entitlement rows for a // whole resource type are served through connector-defined chunk cursors // instead of one ListEntitlements call per group. Each chunk is its own // source-cache scope, so warm syncs replay unchanged chunks and @@ -206,7 +206,7 @@ func (mc *chunkedEntitlementsMockConnector) ListEntitlements( tokens = append(tokens, fmt.Sprintf("chunk=%d", i)) } return v2.EntitlementsServiceListEntitlementsResponse_builder{ - Annotations: annotations.New(v2.SpawnCursors_builder{ + Annotations: annotations.New(v2.EnqueuePageTokens_builder{ PageTokens: tokens, EstimatedTotal: int64(len(mc.groupIDs)), }.Build()), @@ -250,11 +250,11 @@ func (mc *chunkedEntitlementsMockConnector) ListEntitlements( } if page == 0 { - entry, found, err := lookup.LookupPreviousSourceCache(ctx, sourcecache.RowKindEntitlements, scope) + entry, found, err := lookup.Lookup(ctx, sourcecache.RowKindEntitlements, scope) if err != nil { return nil, err } - if found && lastTok != "" && entry.ETag == lastTok { + if found && lastTok != "" && entry.CacheValidator == lastTok { mc.mu.Lock() mc.warmChunkRounds++ mc.mu.Unlock() @@ -266,10 +266,10 @@ func (mc *chunkedEntitlementsMockConnector) ListEntitlements( return v2.EntitlementsServiceListEntitlementsResponse_builder{ List: adds, Annotations: annotations.New(v2.SourceCacheReplay_builder{ - ScopeHash: scope, - Etag: newTok, - Overlay: true, - DeletedIds: deleted, + ScopeKey: scope, + CacheValidator: newTok, + Overlay: true, + DeletedIds: deleted, }.Build()), NextPageToken: next, }.Build(), nil @@ -284,8 +284,8 @@ func (mc *chunkedEntitlementsMockConnector) ListEntitlements( mc.mu.Unlock() return v2.EntitlementsServiceListEntitlementsResponse_builder{ List: ents[:half], - Annotations: annotations.New(v2.SourceCacheScope_builder{ - ScopeHash: scope, + Annotations: annotations.New(v2.SourceCacheRecord_builder{ + ScopeKey: scope, }.Build()), NextPageToken: fmt.Sprintf("chunk=%d&page=1", chunk), }.Build(), nil @@ -297,9 +297,9 @@ func (mc *chunkedEntitlementsMockConnector) ListEntitlements( } return v2.EntitlementsServiceListEntitlementsResponse_builder{ List: ents[half:], - Annotations: annotations.New(v2.SourceCacheScope_builder{ - ScopeHash: scope, - Etag: newTok, + Annotations: annotations.New(v2.SourceCacheRecord_builder{ + ScopeKey: scope, + CacheValidator: newTok, }.Build()), NextPageToken: next, }.Build(), nil diff --git a/pkg/sync/type_scoped_grants_test.go b/pkg/sync/type_scoped_grants_test.go index 041d44b82..6f3c3930f 100644 --- a/pkg/sync/type_scoped_grants_test.go +++ b/pkg/sync/type_scoped_grants_test.go @@ -1,7 +1,7 @@ package sync //nolint:revive,nolintlint // we can't change the package name for backwards compatibility // End-to-end harness for TYPE-SCOPED grants (v2.TypeScopedGrants + -// SpawnCursors), modeled on Microsoft Graph's chunked groups/delta: +// EnqueuePageTokens), modeled on Microsoft Graph's chunked groups/delta: // membership grants for a whole resource type are served through // connector-defined 50-id chunk cursors instead of one ListGrants call per // group. Each chunk is its own source-cache scope, so warm syncs replay @@ -10,7 +10,7 @@ package sync //nolint:revive,nolintlint // we can't change the package name for // The syncer-side behaviors under test: // - the grants planner excludes the annotated type from the per-resource // fan-out and enqueues exactly one type-scoped action, -// - the planning call's SpawnCursors annotation enqueues one action per +// - the planning call's EnqueuePageTokens annotation enqueues one action per // chunk, each checkpointed/paginated independently, // - chunk scopes replay warm with overlay adds and canonical-id // tombstones, and equivalence against a cold control sync holds. @@ -43,7 +43,7 @@ const chunkSize = 50 // chunkedGrantsMockConnector serves group-membership grants exclusively // through type-scoped cursors, mimicking chunked Graph delta: // -// - planning call (empty page token): no rows; SpawnCursors with one +// - planning call (empty page token): no rows; EnqueuePageTokens with one // token per 50-group chunk. // - chunk cursor, lookup miss (cold): enumerate the chunk's grants, // paginated, final page carrying the chunk's validator. @@ -245,7 +245,7 @@ func (mc *chunkedGrantsMockConnector) ListGrants(ctx context.Context, in *v2.Gra tokens = append(tokens, fmt.Sprintf("chunk=%d", i)) } return v2.GrantsServiceListGrantsResponse_builder{ - Annotations: annotations.New(v2.SpawnCursors_builder{ + Annotations: annotations.New(v2.EnqueuePageTokens_builder{ PageTokens: tokens, EstimatedTotal: int64(len(mc.groupIDs)), }.Build()), @@ -292,11 +292,11 @@ func (mc *chunkedGrantsMockConnector) ListGrants(ctx context.Context, in *v2.Gra } if page == 0 { - entry, found, err := lookup.LookupPreviousSourceCache(ctx, sourcecache.RowKindGrants, scope) + entry, found, err := lookup.Lookup(ctx, sourcecache.RowKindGrants, scope) if err != nil { return nil, err } - if found && lastTok != "" && entry.ETag == lastTok { + if found && lastTok != "" && entry.CacheValidator == lastTok { // Warm round: one page — replay, overlay adds, tombstones, // rotated validator. With unstampedExtra set, chunk 0 chains // its unstamped trailing page after the replay page: scoped @@ -312,10 +312,10 @@ func (mc *chunkedGrantsMockConnector) ListGrants(ctx context.Context, in *v2.Gra return v2.GrantsServiceListGrantsResponse_builder{ List: adds, Annotations: annotations.New(v2.SourceCacheReplay_builder{ - ScopeHash: scope, - Etag: newTok, - Overlay: true, - DeletedIds: deleted, + ScopeKey: scope, + CacheValidator: newTok, + Overlay: true, + DeletedIds: deleted, }.Build()), NextPageToken: next, }.Build(), nil @@ -333,8 +333,8 @@ func (mc *chunkedGrantsMockConnector) ListGrants(ctx context.Context, in *v2.Gra mc.mu.Unlock() return v2.GrantsServiceListGrantsResponse_builder{ List: grants[:half], - Annotations: annotations.New(v2.SourceCacheScope_builder{ - ScopeHash: scope, + Annotations: annotations.New(v2.SourceCacheRecord_builder{ + ScopeKey: scope, }.Build()), NextPageToken: fmt.Sprintf("chunk=%d&page=1", chunk), }.Build(), nil @@ -346,9 +346,9 @@ func (mc *chunkedGrantsMockConnector) ListGrants(ctx context.Context, in *v2.Gra } return v2.GrantsServiceListGrantsResponse_builder{ List: grants[half:], - Annotations: annotations.New(v2.SourceCacheScope_builder{ - ScopeHash: scope, - Etag: newTok, + Annotations: annotations.New(v2.SourceCacheRecord_builder{ + ScopeKey: scope, + CacheValidator: newTok, }.Build()), NextPageToken: next, }.Build(), nil diff --git a/proto/c1/connector/v2/annotation_source_cache.proto b/proto/c1/connector/v2/annotation_source_cache.proto index 9d0e41db9..02fbada11 100644 --- a/proto/c1/connector/v2/annotation_source_cache.proto +++ b/proto/c1/connector/v2/annotation_source_cache.proto @@ -18,39 +18,39 @@ option go_package = "github.com/conductorone/baton-sdk/pb/c1/connector/v2"; // upstream data the connector can cheaply revalidate as a unit: a page URL // for GitHub-style conditional GETs, a whole collection for Microsoft // Graph-style delta queries. The SDK never computes provider scopes; it -// only keys storage by the connector-supplied scope_hash. +// only keys storage by the connector-supplied scope_key. // -// The etag is an opaque validator persisted per scope: a literal HTTP ETag +// The cache_validator is an opaque validator persisted per scope: a literal HTTP ETag // for conditional requests, a delta/skip token for delta queries. The SDK // hands it back via the source-cache lookup on the next sync and never // interprets it. // // Flow per sync, per scope: // -// 1. Connector looks up the previous etag for (row_kind, scope_hash) via +// 1. Connector looks up the previous validator for (row_kind, scope_key) via // the SDK-provided lookup. Miss => fetch fresh, emit rows, attach -// SourceCacheScope. Hit => revalidate upstream with the etag. +// SourceCacheRecord. Hit => revalidate upstream with the validator. // 2. Unchanged (HTTP 304): attach SourceCacheReplay with overlay=false and -// the (unchanged) etag; emit no rows. +// the (unchanged) validator; emit no rows. // 3. Changed-with-diff (delta query): attach SourceCacheReplay with // overlay=true and any removed ids; emit ONLY the changed rows (tagged -// with SourceCacheScope); the final page's SourceCacheScope carries the -// new token as etag. +// with SourceCacheRecord); the final page's SourceCacheRecord carries the +// new token as cache_validator. // 4. Changed-without-diff (plain conditional GET): fetch fresh, emit rows, -// attach SourceCacheScope with the new etag. Nothing is replayed. +// attach SourceCacheRecord with the new validator. Nothing is replayed. // // Replay is only honored when the connector declared // SourceCacheCapability MODE_READ_WRITE on its Validate response AND the // sync is running on a storage engine that supports source-cache replay // with a usable previous sync. When either condition fails the SDK // installs a lookup that always misses, so a well-behaved connector -// (invariant: only emit SourceCacheReplay for a scope whose etag came +// (invariant: only emit SourceCacheReplay for a scope whose validator came // from this sync's lookup) naturally degrades to full fetch. // // "This sync's lookup" means the lookup happened during THIS sync — not // necessarily during the same call that emits the replay. A planning call // may batch-resolve validators for many scopes and hand the verdicts to -// sibling cursors via SpawnCursors page tokens; a replay emitted by such a +// sibling cursors via EnqueuePageTokens page tokens; a replay emitted by such a // cursor satisfies the invariant. What the invariant forbids is validators // that outlive the sync (connector-side caches, config, upstream echoes): // the lookup result must originate from the sync that consumes it. @@ -67,10 +67,10 @@ message SourceCacheCapability { Mode mode = 1; } -// SourceCacheScope is attached to a list-response page whose rows were +// SourceCacheRecord is attached to a list-response page whose rows were // freshly fetched from upstream. The SDK stamps the page's rows with -// scope_hash so a future sync can replay them as a unit. -message SourceCacheScope { +// scope_key so a future sync can replay them as a unit. +message SourceCacheRecord { // Connector-computed stable identifier for the canonical scope. Must be // byte-stable across syncs for the same logical scope. Prefer an ORDERED // natural identifier (e.g. the request URL, or "groups/{id}/members") @@ -78,13 +78,13 @@ message SourceCacheScope { // identifiers that correlate with fetch order keep index writes nearly // append-ordered at scale. sourcecache.HashScope is the fallback when no // compact natural form exists. - string scope_hash = 1; + string scope_key = 1; // Opaque validator to persist for this scope. May be empty on interim // pages of a multi-page scope (e.g. Graph @odata.nextLink pages); the - // SDK writes the scope's manifest entry when a non-empty etag arrives. + // SDK writes the scope's manifest entry when a non-empty cache_validator arrives. // A 200 response with zero rows still persists the entry. - string etag = 2; + string cache_validator = 2; // Tombstones applied after this page's rows commit. Lets every page of // a multi-page delta round carry its own deletions as the provider @@ -119,26 +119,26 @@ message SourceCacheScope { } // SourceCacheReplay is attached to a list-response page to tell the SDK to -// copy the previous sync's rows for scope_hash into the current sync. +// copy the previous sync's rows for scope_key into the current sync. // // The row kind (resources, entitlements, grants) is determined by which // RPC the annotation arrived on, never by the annotation itself. // -// The connector must only emit this for a scope whose etag it received +// The connector must only emit this for a scope whose validator it received // from the SDK's source-cache lookup during this same sync. A replay for // an unknown scope fails the sync: the connector has already skipped row // generation, so there is nothing to fall back to. message SourceCacheReplay { - string scope_hash = 1; + string scope_key = 1; // Validator to persist for this scope in the current sync. For an HTTP - // 304 this is the unchanged etag. For a delta query this is the NEW + // 304 this is the unchanged validator. For a delta query this is the NEW // token; it may instead be supplied by the final overlay page's - // SourceCacheScope.etag, in which case this field may be left empty. - string etag = 2; + // SourceCacheRecord.cache_validator, in which case this field may be left empty. + string cache_validator = 2; // When true, the response (and subsequent pages carrying - // SourceCacheScope with the same scope_hash) contains changed rows to + // SourceCacheRecord with the same scope_key) contains changed rows to // upsert on top of the replayed base. When false the response must // contain no rows for this scope. bool overlay = 3; @@ -147,11 +147,11 @@ message SourceCacheReplay { // RowKindResources) to delete from the current sync after the replay // copy and this page's upserts. Used for delta-query tombstones (e.g. // Microsoft Graph @removed entries). Subsequent pages of the round - // carry their tombstones on SourceCacheScope.deleted_ids. + // carry their tombstones on SourceCacheRecord.deleted_ids. repeated string deleted_ids = 4; // Principal-scoped tombstones for this page; see - // SourceCacheScope.deleted_principal_ids for semantics and + // SourceCacheRecord.deleted_principal_ids for semantics and // preconditions. repeated string deleted_principal_ids = 5; } @@ -205,7 +205,7 @@ message SourceCacheLookupAsk { min_bytes: 1 max_bytes: 64 }]; - string scope_hash = 2 [(validate.rules).string = { + string scope_key = 2 [(validate.rules).string = { min_bytes: 1 max_bytes: 256 }]; @@ -220,11 +220,11 @@ message SourceCacheLookupAsk { // REQUEST, carrying the resolution of a prior ask's queries. // // Every query of the prior ask is answered — found=true with the -// previous sync's etag, or found=false meaning fetch fresh. A found -// answer whose etag does not fit the per-request answer size budget is +// previous sync's validator, or found=false meaning fetch fresh. A found +// answer whose validator does not fit the per-request answer size budget is // delivered as found=false: the scope degrades to a cold fetch // (correct, just slower) instead of staying unresolved, because answers -// accumulate across bounces and an etag that did not fit once can never +// accumulate across bounces and a validator that did not fit once can never // fit a later re-invoke. An ABSENT query (a phase-2 lookup for a scope // the connector did not ask before) means unresolved: the connector may // ask again, subject to the bounce cap. @@ -234,14 +234,14 @@ message SourceCacheLookupAnswers { min_bytes: 1 max_bytes: 64 }]; - string scope_hash = 2 [(validate.rules).string = { + string scope_key = 2 [(validate.rules).string = { min_bytes: 1 max_bytes: 256 }]; bool found = 3; // The previous sync's validator; empty when found is false. Cap // matches the lookup RPC (Graph delta tokens run long). - string etag = 4 [(validate.rules).string = { + string cache_validator = 4 [(validate.rules).string = { ignore_empty: true max_bytes: 65536 }]; diff --git a/proto/c1/connector/v2/annotation_type_scoped_entitlements.proto b/proto/c1/connector/v2/annotation_type_scoped_entitlements.proto index 909eafbec..5f028c3eb 100644 --- a/proto/c1/connector/v2/annotation_type_scoped_entitlements.proto +++ b/proto/c1/connector/v2/annotation_type_scoped_entitlements.proto @@ -20,7 +20,7 @@ option go_package = "github.com/conductorone/baton-sdk/pb/c1/connector/v2"; // ({type, type} — wire validation requires a non-empty resource id and // the stub identifies the type, never a real resource). The connector // answers with entitlements for the whole type, across as many paginated -// cursors as it chooses to spawn (see SpawnCursors in +// cursors as it chooses to spawn (see EnqueuePageTokens in // annotation_type_scoped_grants.proto). // // This is the entitlements-phase analogue of TypeScopedGrants: the @@ -42,7 +42,7 @@ option go_package = "github.com/conductorone/baton-sdk/pb/c1/connector/v2"; // // ZERO-RESOURCE TYPES: the planner still enqueues the type-scoped action // when the store holds no resources of the type. The connector must -// return an empty page (or SpawnCursors that resolve empty), not an +// return an empty page (or EnqueuePageTokens that resolve empty), not an // error — an empty type is a valid outcome. // // TARGETED SYNC: SyncTargetedResource must not enqueue a per-resource diff --git a/proto/c1/connector/v2/annotation_type_scoped_grants.proto b/proto/c1/connector/v2/annotation_type_scoped_grants.proto index 28eef7de1..0fdc07907 100644 --- a/proto/c1/connector/v2/annotation_type_scoped_grants.proto +++ b/proto/c1/connector/v2/annotation_type_scoped_grants.proto @@ -22,7 +22,7 @@ option go_package = "github.com/conductorone/baton-sdk/pb/c1/connector/v2"; // validation requires a non-empty resource id and the stub identifies // the type, never a real resource). The connector answers with grants // for the whole type, across as many paginated cursors as it chooses to -// spawn (see SpawnCursors). +// spawn (see EnqueuePageTokens). // // This is the grants-phase analogue of StaticEntitlements: the connector // takes over enumeration for the type, and every downstream behavior @@ -41,7 +41,7 @@ option go_package = "github.com/conductorone/baton-sdk/pb/c1/connector/v2"; // // ZERO-RESOURCE TYPES: the planner still enqueues the type-scoped action // when the store holds no resources of the type. The connector must -// return an empty page (or SpawnCursors that resolve empty), not an error. +// return an empty page (or EnqueuePageTokens that resolve empty), not an error. // // TARGETED SYNC: SyncTargetedResource must not enqueue a per-resource // SyncGrantsOp for a type-scoped type. Type-scoped grant enumeration is @@ -52,7 +52,7 @@ option go_package = "github.com/conductorone/baton-sdk/pb/c1/connector/v2"; // imply post-ingest repair beyond ordinary grant handling. message TypeScopedGrants {} -// SpawnCursors is attached to a ListGrants or ListEntitlements response to +// EnqueuePageTokens is attached to a ListGrants or ListEntitlements response to // enqueue additional independent sibling cursors. Each token is delivered // back to the connector as the page token of its own action — scheduled by // the syncer's worker pool, rate-limited, and checkpointed like any other @@ -87,7 +87,7 @@ message TypeScopedGrants {} // connector needs to serve the cursor must be inside the token (or // re-derivable from it) — for per-resource spawns the resource identity // rides the action, so tokens only need the page coordinate. -message SpawnCursors { +message EnqueuePageTokens { // Page tokens for the sibling cursors to enqueue, one action each. // Capped per response (spawned actions persist in the checkpointed // state token); chain additional spawns across pages to fan out wider. diff --git a/proto/c1/connectorapi/baton/v1/source_cache.proto b/proto/c1/connectorapi/baton/v1/source_cache.proto index 5180996be..a03b9ebc9 100644 --- a/proto/c1/connectorapi/baton/v1/source_cache.proto +++ b/proto/c1/connectorapi/baton/v1/source_cache.proto @@ -7,7 +7,7 @@ import "validate/validate.proto"; option go_package = "gitlab.com/ductone/c1/pkg/pb/c1/connectorapi/baton/v1"; // BatonSourceCacheService is the dedicated parent-side RPC the connector -// subprocess uses to ask "do I have a previous validator (etag / delta +// subprocess uses to ask "do I have a previous validator (HTTP ETag / delta // token) for this scope?" before revalidating upstream. // // This is intentionally NOT routed through the session-store gRPC service: @@ -31,18 +31,18 @@ message LookupRequest { max_len: 64 }]; - // Connector-defined stable scope identifier (conventionally a hex hash - // of the canonical scope; see pkg/sourcecache.HashScope). Opaque to the + // Connector-defined stable scope identifier (a stable identifier for the canonical scope + // (often a hex hash; see pkg/sourcecache.HashScope). Opaque to the // parent; matched verbatim against the previous sync's source-cache // entries. - string scope_hash = 2 [(validate.rules).string = { + string scope_key = 2 [(validate.rules).string = { min_len: 1 max_len: 256 }]; } message LookupResponse { - // False means no prior entry exists for (row_kind, scope_hash): the + // False means no prior entry exists for (row_kind, scope_key): the // connector must fetch fresh and must not emit SourceCacheReplay for // this scope. bool found = 1; @@ -51,5 +51,5 @@ message LookupResponse { // ETag, delta token, ...). Empty when found is false. The cap is a // sanity bound sized for Microsoft Graph delta tokens, which are known // to run to thousands of characters; storage imposes no limit. - string etag = 2 [(validate.rules).string = {max_len: 65536}]; + string cache_validator = 2 [(validate.rules).string = {max_len: 65536}]; } diff --git a/proto/c1/storage/v3/records.proto b/proto/c1/storage/v3/records.proto index fe87a5b05..292d35f65 100644 --- a/proto/c1/storage/v3/records.proto +++ b/proto/c1/storage/v3/records.proto @@ -106,9 +106,9 @@ message ResourceRecord { // Source-cache scope stamp (see annotation_source_cache.proto). // Empty for rows not produced under a source-cache scope. - string source_scope_hash = 9 [(index) = { + string source_scope_key = 9 [(index) = { name: "by_source_scope" - partial: "source_scope_hash != ''" + partial: "source_scope_key != ''" }]; } @@ -152,9 +152,9 @@ message EntitlementRecord { // Source-cache scope stamp (see annotation_source_cache.proto). // Empty for rows not produced under a source-cache scope. - string source_scope_hash = 11 [(index) = { + string source_scope_key = 11 [(index) = { name: "by_source_scope" - partial: "source_scope_hash != ''" + partial: "source_scope_key != ''" }]; } @@ -208,9 +208,9 @@ message GrantRecord { // expander-derived grants, which are recreated by expansion each sync // and never replayed. StoreExpandedGrants preserves this field on // rewrites of existing rows, exactly like expansion/needs_expansion. - string source_scope_hash = 10 [(index) = { + string source_scope_key = 10 [(index) = { name: "by_source_scope" - partial: "source_scope_hash != ''" + partial: "source_scope_key != ''" }]; } @@ -248,7 +248,7 @@ message SyncRunRecord { // compacted marks a sync produced by compaction (fold or rebuild) // rather than by a real connector run. Compacted artifacts are // keep-newer UPSERT merges — base rows a newer input deleted survive — - // so no input sync's upstream validators (source-cache etags) describe + // so no input sync's upstream validators (source-cache validators) describe // their contents. // // "Can this sync be used as a source-cache replay source?" is the @@ -302,7 +302,7 @@ message SessionRecord { // SourceCacheEntryRecord is the per-scope manifest entry for source-cache // replay (see c1/connector/v2/annotation_source_cache.proto). One entry -// per (row_kind, scope_hash), written for every freshly fetched scope — +// per (row_kind, scope_key), written for every freshly fetched scope — // including zero-row responses — and rewritten on replay with the scope's // current validator. The previous sync's entries (read from the previous // c1z) are the lookup surface connectors revalidate against. @@ -311,7 +311,7 @@ message SourceCacheEntryRecord { name: "source_cache_entries" primary_key: [ "row_kind", - "scope_hash" + "scope_key" ] }; @@ -319,11 +319,12 @@ message SourceCacheEntryRecord { // (pkg/sourcecache.RowKind values). string row_kind = 1; - // Connector-computed scope hash (lowercase hex). - string scope_hash = 2; + // Connector-computed stable scope identifier (conventionally lowercase + // hex when hashed via sourcecache.HashScope; any byte-stable string). + string scope_key = 2; // Opaque upstream validator: HTTP ETag, delta token, etc. - string etag = 3; + string cache_validator = 3; google.protobuf.Timestamp discovered_at = 4; } From 4542f48004129a413204b981a45cc90bc9ddc821 Mon Sep 17 00:00:00 2001 From: Matt Kaniaris Date: Tue, 14 Jul 2026 19:57:57 -0600 Subject: [PATCH 06/17] model oracle for syncs completed, seal time counts for sanity checks, halt point crash sweeps, and a new baton utility to deal with all of it --- cmd/baton/main.go | 1 + cmd/baton/source_cache.go | 158 ++++++++++++++++++ pkg/dotc1z/c1file.go | 12 +- pkg/dotc1z/engine/pebble/source_cache.go | 115 ++++++++++++- .../engine/pebble/source_cache_engine_test.go | 47 ++++++ pkg/dotc1z/pool_test.go | 12 +- pkg/dotc1z/race_disabled_test.go | 6 + pkg/dotc1z/race_enabled_test.go | 9 + pkg/dotc1z/source_cache.go | 10 ++ pkg/sync/ingest_invariants.go | 41 ++++- pkg/sync/source_cache.go | 20 +++ pkg/sync/source_cache_equivalence_test.go | 61 ++++++- pkg/sync/source_cache_halt_test.go | 115 +++++++++++++ pkg/sync/state.go | 66 ++++---- pkg/sync/syncer.go | 13 +- 15 files changed, 637 insertions(+), 49 deletions(-) create mode 100644 cmd/baton/source_cache.go create mode 100644 pkg/dotc1z/race_disabled_test.go create mode 100644 pkg/dotc1z/race_enabled_test.go create mode 100644 pkg/sync/source_cache_halt_test.go diff --git a/cmd/baton/main.go b/cmd/baton/main.go index e0e8da82c..fcd6515db 100644 --- a/cmd/baton/main.go +++ b/cmd/baton/main.go @@ -34,6 +34,7 @@ func main() { cliCmd.AddCommand(accessCmd()) cliCmd.AddCommand(dumpDBCmd()) cliCmd.AddCommand(syncsCmd()) + cliCmd.AddCommand(sourceCacheCmd()) cliCmd.AddCommand(optimizeDb()) cliCmd.AddCommand(toPebbleCmd()) cliCmd.AddCommand(explorerCmd()) diff --git a/cmd/baton/source_cache.go b/cmd/baton/source_cache.go new file mode 100644 index 000000000..99386fbd4 --- /dev/null +++ b/cmd/baton/source_cache.go @@ -0,0 +1,158 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "os" + "sort" + "strings" + "text/tabwriter" + "time" + + "github.com/spf13/cobra" + + pebbleengine "github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble" + "github.com/conductorone/baton-sdk/pkg/logging" +) + +// source-cache: the attribution surface for source-cache replay. Rows in +// a c1z are stamped with the scope that produced them and the manifest +// records each scope's validator + write time — this command joins the +// two so "why is this row missing / stale?" becomes a per-scope answer: +// when was the scope last actually fetched, what validator did it record, +// how many rows does it hold, and is its index/manifest state consistent. +func sourceCacheCmd() *cobra.Command { + cmd := &cobra.Command{ + Use: "source-cache", + Short: "Audit the C1Z's source-cache replay state (scopes, validators, row counts, consistency)", + RunE: runSourceCacheAudit, + } + return cmd +} + +type sourceCacheScopeReport struct { + RowKind string `json:"row_kind"` + ScopeKey string `json:"scope_key"` + Rows int `json:"rows"` + Validator string `json:"cache_validator,omitempty"` + DiscoveredAt time.Time `json:"discovered_at,omitempty"` + // Status: "ok" (manifest + rows), "zero-row" (manifest entry with no + // stamped rows — legal: the scope legitimately returned nothing), or + // "orphan-index" (stamped rows with NO manifest entry — an invariant + // violation that would poison the next sync's replay; see ingestion + // invariant I6). + Status string `json:"status"` +} + +func runSourceCacheAudit(cmd *cobra.Command, args []string) error { + ctx, err := logging.Init(context.Background(), logging.WithLogFormat("console"), logging.WithLogLevel("error")) + if err != nil { + return err + } + c1zPath, err := cmd.Flags().GetString("file") + if err != nil { + return err + } + outputFormat, err := cmd.Flags().GetString("output-format") + if err != nil { + return err + } + + store, err := openReadOnlyC1ZStore(ctx, c1zPath) + if err != nil { + return err + } + defer store.Close(ctx) + + holder, ok := store.(interface{ PebbleEngine() *pebbleengine.Engine }) + var engine *pebbleengine.Engine + if ok { + engine = holder.PebbleEngine() + } + if engine == nil { + return fmt.Errorf("source-cache state exists only in pebble-format c1z files; %q is not one", c1zPath) + } + + manifest, err := engine.SourceCacheManifestSnapshot(ctx) + if err != nil { + return fmt.Errorf("reading source-cache manifest: %w", err) + } + indexCounts, err := engine.SourceScopeIndexSnapshot(ctx) + if err != nil { + return fmt.Errorf("reading scope indexes: %w", err) + } + + reports := make([]sourceCacheScopeReport, 0, len(manifest)) + for key := range manifest { + kind, scope, found := strings.Cut(key, "\x00") + if !found { + return fmt.Errorf("malformed manifest snapshot key %q", key) + } + rep := sourceCacheScopeReport{RowKind: kind, ScopeKey: scope, Rows: indexCounts[kind][scope]} + rec, err := engine.GetSourceCacheEntry(ctx, kind, scope) + if err != nil { + return fmt.Errorf("reading manifest entry for %s/%q: %w", kind, scope, err) + } + rep.Validator = rec.GetCacheValidator() + rep.DiscoveredAt = rec.GetDiscoveredAt().AsTime() + if rep.Rows == 0 { + rep.Status = "zero-row" + } else { + rep.Status = "ok" + } + reports = append(reports, rep) + } + // Index scopes with no manifest entry: invariant I6 violations. + for kind, scopes := range indexCounts { + for scope, n := range scopes { + if _, ok := manifest[kind+"\x00"+scope]; !ok { + reports = append(reports, sourceCacheScopeReport{ + RowKind: kind, ScopeKey: scope, Rows: n, Status: "orphan-index", + }) + } + } + } + sort.Slice(reports, func(i, j int) bool { + if reports[i].RowKind != reports[j].RowKind { + return reports[i].RowKind < reports[j].RowKind + } + return reports[i].ScopeKey < reports[j].ScopeKey + }) + + if outputFormat == "json" { + enc := json.NewEncoder(os.Stdout) + enc.SetIndent("", " ") + return enc.Encode(reports) + } + + if len(reports) == 0 { + fmt.Fprintln(os.Stdout, "no source-cache state (cold artifact: every scope will fetch fresh next sync)") + return nil + } + w := tabwriter.NewWriter(os.Stdout, 2, 4, 2, ' ', 0) + fmt.Fprintln(w, "KIND\tSCOPE\tROWS\tSTATUS\tDISCOVERED\tVALIDATOR") + orphans := 0 + for _, r := range reports { + validator := r.Validator + if len(validator) > 48 { + validator = validator[:45] + "..." + } + discovered := "" + if !r.DiscoveredAt.IsZero() { + discovered = r.DiscoveredAt.UTC().Format(time.RFC3339) + } + if r.Status == "orphan-index" { + orphans++ + } + fmt.Fprintf(w, "%s\t%s\t%d\t%s\t%s\t%s\n", r.RowKind, r.ScopeKey, r.Rows, r.Status, discovered, validator) + } + if err := w.Flush(); err != nil { + return err + } + if orphans > 0 { + fmt.Fprintf(os.Stdout, "\nWARNING: %d orphan-index scope(s): stamped rows with no manifest entry (ingestion invariant I6). "+ + "A future sync replaying from this file could silently drop or resurrect these scopes' rows.\n", orphans) + } + return nil +} diff --git a/pkg/dotc1z/c1file.go b/pkg/dotc1z/c1file.go index 7cf1d5ba1..d858bde7b 100644 --- a/pkg/dotc1z/c1file.go +++ b/pkg/dotc1z/c1file.go @@ -44,12 +44,12 @@ type pragma struct { } type C1File struct { - rawDb *sql.DB - db *goqu.Database - currentSyncID string - viewSyncID string - outputFilePath string - dbFilePath string + rawDb *sql.DB + db *goqu.Database + currentSyncID string + viewSyncID string + outputFilePath string + dbFilePath string // dbUpdated records that any mutation ran; parallel sync workers set // it concurrently, so it is atomic (it was a plain bool data race // under -race). Only ever set true during a session; read at Close. diff --git a/pkg/dotc1z/engine/pebble/source_cache.go b/pkg/dotc1z/engine/pebble/source_cache.go index e33d1ef5c..d19fa6330 100644 --- a/pkg/dotc1z/engine/pebble/source_cache.go +++ b/pkg/dotc1z/engine/pebble/source_cache.go @@ -4,6 +4,7 @@ import ( "context" "errors" "fmt" + "sort" "github.com/cockroachdb/pebble/v2" "google.golang.org/protobuf/encoding/protowire" @@ -152,6 +153,114 @@ func (e *Engine) SourceScopeIndexSnapshot(ctx context.Context) (map[string]map[s return out, nil } +// countSourceScopeIndexRange counts the index entries under one scope's +// by_source_scope prefix — a contiguous key-only scan, no value reads. +func (e *Engine) countSourceScopeIndexRange(ctx context.Context, prefix []byte) (int64, error) { + iter, err := e.db.NewIter(&pebble.IterOptions{ + LowerBound: prefix, + UpperBound: upperBoundOf(prefix), + }) + if err != nil { + return 0, err + } + defer iter.Close() + var n int64 + for iter.First(); iter.Valid(); iter.Next() { + if err := ctx.Err(); err != nil { + return 0, err + } + n++ + } + return n, iter.Error() +} + +// verifyReplayedScopeCount is the replay-time count check: immediately +// after a replay's final commit, the destination's scope-index range must +// hold exactly the rows the copy claims to have written. Sound because +// the copy is the scope's first writer in the sync (fresh rows for a +// replayed scope arrive only via the overlay, after this) and a resumed +// re-copy overwrites the identical identity set. A mismatch means a +// partial copy landed — the silent-bad-data class — so it fails the +// replay rather than letting the manifest entry seal it as complete. +func (e *Engine) verifyReplayedScopeCount(ctx context.Context, kind, scopeKey string, prefix []byte, copied int64) error { + got, err := e.countSourceScopeIndexRange(ctx, prefix) + if err != nil { + return fmt.Errorf("source cache replay: recount %s scope %q: %w", kind, scopeKey, err) + } + if got != copied { + return fmt.Errorf( + "source cache replay: %s scope %q index holds %d entries after copy but %d rows were copied (partial replay copy)", + kind, scopeKey, got, copied) + } + return nil +} + +// SourceCacheOrphanScopes returns, per row kind, every scope key present +// in a by_source_scope index that has NO manifest entry. At a sealed +// sync's quiesce point this is an invariant violation: every stamped row +// was written under a scope whose completion writes the manifest entry, +// and post-processing (external match, expansion) only re-stamps rows +// with scopes that already completed. An orphan means a manifest write +// was lost or a stamp leaked — either would poison the NEXT sync's +// replay. One skip-scan per distinct scope; O(distinct scopes) seeks. +func (e *Engine) SourceCacheOrphanScopes(ctx context.Context) (map[string][]string, error) { + out := map[string][]string{} + families := []struct { + kind string + lower []byte + prefix func(scope string) []byte + }{ + {"resources", ResourceBySourceScopeLowerBound(), encodeResourceBySourceScopePrefix}, + {"entitlements", EntitlementBySourceScopeLowerBound(), encodeEntitlementBySourceScopePrefix}, + {"grants", GrantBySourceScopeLowerBound(), encodeGrantBySourceScopePrefix}, + } + for _, fam := range families { + famPrefix := codec.AppendTupleSeparator(append([]byte{}, fam.lower...)) + iter, err := e.db.NewIter(&pebble.IterOptions{ + LowerBound: famPrefix, + UpperBound: upperBoundOf(fam.lower), + }) + if err != nil { + return nil, err + } + for valid := iter.First(); valid; { + if err := ctx.Err(); err != nil { + iter.Close() + return nil, err + } + tail := iter.Key()[len(famPrefix):] + scopeBytes, _, ok := codec.DecodeTupleStringAlias(tail, 0) + if !ok { + iter.Close() + return nil, fmt.Errorf("SourceCacheOrphanScopes: malformed %s index key %x", fam.kind, iter.Key()) + } + scope := string(scopeBytes) + if _, closer, getErr := e.db.Get(encodeSourceCacheEntryKey(fam.kind, scope)); getErr == nil { + closer.Close() + } else if errors.Is(getErr, pebble.ErrNotFound) { + out[fam.kind] = append(out[fam.kind], scope) + } else { + iter.Close() + return nil, getErr + } + // Skip the rest of this scope's entries. + valid = iter.SeekGE(upperBoundOf(fam.prefix(scope))) + } + if err := iter.Error(); err != nil { + iter.Close() + return nil, err + } + iter.Close() + } + if len(out) == 0 { + return nil, nil + } + for _, scopes := range out { + sort.Strings(scopes) + } + return out, nil +} + // ClearSourceCacheEntries removes every source-cache manifest entry. // Compaction calls this on fold outputs: a keep-newer upsert merge is not // a faithful snapshot of ANY input sync (base rows the newer sync deleted @@ -804,7 +913,7 @@ func (e *Engine) ReplaySourceCacheGrants(ctx context.Context, prev *Engine, scop // keyspace is still empty and leaving replayed index entries stale // (consumed by the defer above). committedRows += rowsInBatch - return nil + return e.verifyReplayedScopeCount(ctx, "grants", scopeKey, prefix, res.Rows) }) if err != nil { return SourceCacheReplayResult{}, err @@ -992,7 +1101,7 @@ func (e *Engine) ReplaySourceCacheEntitlements(ctx context.Context, prev *Engine // invalidated; bumping earlier would let such a build record the // fresh generation against partially-replayed state. committedRows += rowsInBatch - return nil + return e.verifyReplayedScopeCount(ctx, "entitlements", scopeKey, prefix, res.Rows) }) if err != nil { return SourceCacheReplayResult{}, err @@ -1138,7 +1247,7 @@ func (e *Engine) ReplaySourceCacheResources(ctx context.Context, prev *Engine, s // fast-path, or old by_parent/by_source_scope entries survive // (consumed by the defer above). committedRows += rowsInBatch - return nil + return e.verifyReplayedScopeCount(ctx, "resources", scopeKey, prefix, res.Rows) }) if err != nil { return SourceCacheReplayResult{}, err diff --git a/pkg/dotc1z/engine/pebble/source_cache_engine_test.go b/pkg/dotc1z/engine/pebble/source_cache_engine_test.go index 2dd3be6f7..9b8b728cb 100644 --- a/pkg/dotc1z/engine/pebble/source_cache_engine_test.go +++ b/pkg/dotc1z/engine/pebble/source_cache_engine_test.go @@ -128,6 +128,53 @@ func TestSourceCacheReplayGrantsAcrossEngines(t *testing.T) { require.Equal(t, int64(2), res2.Rows) } +// TestSourceCacheOrphanScopes pins invariant I6's evidence surface: +// scope-index entries whose scope has no manifest entry are reported as +// orphans (a lost manifest write or stamp leak would poison a future +// sync's replay); writing the entry clears the report. Zero-row manifest +// entries (the reverse direction) are legal and never reported. +func TestSourceCacheOrphanScopes(t *testing.T) { + ctx := context.Background() + a := newAdapter(t) + _, err := a.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + e := a.PebbleEngine() + + // Stamped rows in two scopes; a manifest entry for only one. + require.NoError(t, a.PutGrants(sourcecache.WithScope(ctx, scopeA), scGrant("member", "alice", false))) + require.NoError(t, a.PutGrants(sourcecache.WithScope(ctx, scopeB), scGrant("member", "bob", false))) + require.NoError(t, e.PutSourceCacheEntry(ctx, "grants", scopeA, "etag-a")) + + orphans, err := e.SourceCacheOrphanScopes(ctx) + require.NoError(t, err) + require.Equal(t, map[string][]string{"grants": {scopeB}}, orphans) + + // Writing the missing entry clears the violation. + require.NoError(t, e.PutSourceCacheEntry(ctx, "grants", scopeB, "etag-b")) + orphans, err = e.SourceCacheOrphanScopes(ctx) + require.NoError(t, err) + require.Empty(t, orphans) + + // A zero-row scope (manifest entry, no index entries) is legal. + require.NoError(t, e.PutSourceCacheEntry(ctx, "resources", scopeA, "etag-r")) + orphans, err = e.SourceCacheOrphanScopes(ctx) + require.NoError(t, err) + require.Empty(t, orphans) + + // Row-kind partitions are independent: entitlement stamps under a + // scope key that only has a GRANTS manifest entry are orphans. + ent := v2.Entitlement_builder{ + Id: "group:g1:member", + Resource: v2.Resource_builder{ + Id: v2.ResourceId_builder{ResourceType: "group", Resource: "g1"}.Build(), + }.Build(), + }.Build() + require.NoError(t, a.PutEntitlements(sourcecache.WithScope(ctx, scopeB), ent)) + orphans, err = e.SourceCacheOrphanScopes(ctx) + require.NoError(t, err) + require.Equal(t, map[string][]string{"entitlements": {scopeB}}, orphans) +} + // TestSourceCacheReplayIsIdempotent pins double-execution safety: a // resumed sync can re-run a replay page whose first execution already // committed (the checkpoint races the crash), so replaying the same scope diff --git a/pkg/dotc1z/pool_test.go b/pkg/dotc1z/pool_test.go index b5be5239f..092685335 100644 --- a/pkg/dotc1z/pool_test.go +++ b/pkg/dotc1z/pool_test.go @@ -28,7 +28,11 @@ func TestEncoderPool(t *testing.T) { enc2, fromPool2 := getEncoder() require.NotNil(t, enc2) - require.True(t, fromPool2) + if !raceEnabled { + // sync.Pool randomly discards Puts under -race; reuse is + // only deterministic without it. + require.True(t, fromPool2) + } putEncoder(enc2) }) @@ -121,7 +125,11 @@ func TestDecoderPool(t *testing.T) { dec2, fromPool2 := getDecoder() require.NotNil(t, dec2) - require.True(t, fromPool2) + if !raceEnabled { + // See the encoder variant: sync.Pool discards randomly + // under -race. + require.True(t, fromPool2) + } putDecoder(dec2) }) diff --git a/pkg/dotc1z/race_disabled_test.go b/pkg/dotc1z/race_disabled_test.go new file mode 100644 index 000000000..75dfc5747 --- /dev/null +++ b/pkg/dotc1z/race_disabled_test.go @@ -0,0 +1,6 @@ +//go:build !race + +package dotc1z + +// See race_enabled_test.go. +const raceEnabled = false diff --git a/pkg/dotc1z/race_enabled_test.go b/pkg/dotc1z/race_enabled_test.go new file mode 100644 index 000000000..c4e1d39ac --- /dev/null +++ b/pkg/dotc1z/race_enabled_test.go @@ -0,0 +1,9 @@ +//go:build race + +package dotc1z + +// raceEnabled reports whether this test binary was built with -race. +// sync.Pool intentionally randomizes item retention under the race +// detector (Put may discard), so tests asserting pool REUSE are +// structurally flaky there and must skip that assertion. +const raceEnabled = true diff --git a/pkg/dotc1z/source_cache.go b/pkg/dotc1z/source_cache.go index 707feab42..ff750dc20 100644 --- a/pkg/dotc1z/source_cache.go +++ b/pkg/dotc1z/source_cache.go @@ -90,6 +90,12 @@ type IngestFactStore interface { // HasResourceRecord reports whether a resource row exists. HasResourceRecord(ctx context.Context, resourceTypeID, resourceID string) (bool, error) + + // SourceCacheOrphanScopes returns, per row kind, scope keys present + // in the by_source_scope indexes with no manifest entry — an + // invariant violation at a sealed sync's quiesce point (invariant + // I6): the orphaned stamps would poison a future sync's replay. + SourceCacheOrphanScopes(ctx context.Context) (map[string][]string, error) } var _ IngestFactStore = (*pebbleStore)(nil) @@ -110,6 +116,10 @@ func (s *pebbleStore) HasResourceRecord(ctx context.Context, resourceTypeID, res return s.engine.HasResourceRecord(ctx, resourceTypeID, resourceID) } +func (s *pebbleStore) SourceCacheOrphanScopes(ctx context.Context) (map[string][]string, error) { + return s.engine.SourceCacheOrphanScopes(ctx) +} + // sourceCacheEngine recovers the Pebble engine from an arbitrary store, // nil-safe. Mirrors pebble.AsEngine but accepts any value so the syncer // can probe its previous-sync reader without caring about its static type. diff --git a/pkg/sync/ingest_invariants.go b/pkg/sync/ingest_invariants.go index c42a14d45..2d1688a65 100644 --- a/pkg/sync/ingest_invariants.go +++ b/pkg/sync/ingest_invariants.go @@ -35,6 +35,13 @@ package sync //nolint:revive,nolintlint // we can't change the package name for // type per group, size cap). This REPLACES the former streaming // validation entirely — it is engine-agnostic and covers replayed, // fresh, and generated (static) entitlements uniformly. +// - I6 source-cache scope consistency: every scope present in a +// by_source_scope index has a manifest entry at seal. Orphaned +// stamps mean a lost manifest write or a stamp leak — either would +// poison a FUTURE sync's replay while this sync reads clean. +// Pebble-only (the only engine with source-cache state); the +// replay-time counterpart (copied-row count vs the scope index) +// lives in the engine's ReplaySourceCache* and always hard-fails. // // Evidence is monotone (existence bits, set-union scheduling records), // so parallel workers and mixed stream/replay arrival orders yield @@ -329,8 +336,37 @@ func (s *syncer) validateStoredExclusionGroups(ctx context.Context) error { return nil } +// checkSourceCacheScopeConsistency is invariant I6: at the sealed sync's +// quiesce point, every scope present in a by_source_scope index must have +// a manifest entry. Post-processing only re-stamps rows with scopes that +// already completed (external-match transformed grants and expansion +// writes inherit the SOURCE scope), so an orphan at this seam is always a +// lost manifest write or a stamp leak — and the damage is deferred: THIS +// sync reads clean, the NEXT sync replays from the damaged state. +func (s *syncer) checkSourceCacheScopeConsistency(ctx context.Context) error { + facts, ok := s.store.(dotc1z.IngestFactStore) + if !ok { + return nil // engine without source-cache state + } + orphans, err := facts.SourceCacheOrphanScopes(ctx) + if err != nil { + return fmt.Errorf("ingest invariant I6: scanning scope indexes: %w", err) + } + if len(orphans) == 0 { + return nil + } + err = fmt.Errorf( + "ingest invariant I6 violated: scope index entries exist with no manifest entry (lost manifest write or stamp leak): %v", + orphans) + if s.strictIngestionInvariants { + return err + } + ctxzap.Extract(ctx).Warn("ingest invariant I6 violated (lenient mode)", zap.Error(err)) + return nil +} + // runIngestionInvariants evaluates the post-collection (check-flavored) -// invariants: I5, I4, I3, in that order (cheapest and most +// invariants: I5, I6, I4, I3, in that order (cheapest and most // behavior-critical first). Runs after the action loop drains and before // EndSync, so every ingestion path — stream, replay, resume re-runs, // expansion, external-match processing — has finished writing. @@ -340,6 +376,9 @@ func (s *syncer) runIngestionInvariants(ctx context.Context) error { if err := s.validateStoredExclusionGroups(ctx); err != nil { return err } + if err := s.checkSourceCacheScopeConsistency(ctx); err != nil { + return err + } if err := s.checkChildScheduling(ctx); err != nil { return err } diff --git a/pkg/sync/source_cache.go b/pkg/sync/source_cache.go index eb12c4a9b..aa8512f69 100644 --- a/pkg/sync/source_cache.go +++ b/pkg/sync/source_cache.go @@ -439,6 +439,11 @@ func (s *syncer) beginSourceCachePage( zap.Int("deleted_ids", len(page.deletedIDs)), zap.Int("deleted_principal_ids", len(page.deletedPrincipalIDs)), ) + if s.testSourceCacheHaltHook != nil { + if err := s.testSourceCacheHaltHook("replay-copied", page.scopeKey); err != nil { + return ctx, nil, err + } + } } return sourcecache.WithScope(ctx, page.scopeKey), page, nil @@ -452,6 +457,11 @@ func (s *syncer) finishSourceCachePage(ctx context.Context, page *sourceCachePag if page == nil { return nil } + if s.testSourceCacheHaltHook != nil { + if err := s.testSourceCacheHaltHook("rows-committed", page.scopeKey); err != nil { + return err + } + } var rowsDeleted int64 if len(page.deletedIDs)+len(page.deletedPrincipalIDs) > 0 { tombstoneStart := time.Now() @@ -492,10 +502,20 @@ func (s *syncer) finishSourceCachePage(ctx context.Context, page *sourceCachePag zap.Int64("rows_deleted", deleted), ) } + if s.testSourceCacheHaltHook != nil { + if err := s.testSourceCacheHaltHook("tombstones-applied", page.scopeKey); err != nil { + return err + } + } if page.etag != "" { if err := s.sourceCache.current.PutSourceCacheEntry(ctx, page.kind, page.scopeKey, page.etag); err != nil { return fmt.Errorf("source cache: error writing manifest entry for scope %q: %w", page.scopeKey, err) } + if s.testSourceCacheHaltHook != nil { + if err := s.testSourceCacheHaltHook("manifest-written", page.scopeKey); err != nil { + return err + } + } } stats := SourceCacheStats{ diff --git a/pkg/sync/source_cache_equivalence_test.go b/pkg/sync/source_cache_equivalence_test.go index 058ab7c23..2745a7696 100644 --- a/pkg/sync/source_cache_equivalence_test.go +++ b/pkg/sync/source_cache_equivalence_test.go @@ -1603,6 +1603,57 @@ func assertNoStaleResources(t *testing.T, round int, m *equivModel, label string } } +// expectedResourceKeys derives the COMPLETE resource set ("rt/id" keys) +// the model implies: users (local + matched external principals), groups, +// orgs, per-org child projects, InsertResourceGrants repos, and teams. +// Exact set equality against the cold output removes the differential +// test's structural blind spot for the resources phase (see +// expectedGrantIDs). +func expectedResourceKeys(m *equivModel) []string { + set := map[string]struct{}{} + for uid := range m.users { + set[userResourceType.GetId()+"/"+uid] = struct{}{} + } + // Every external user is matched by the MatchAll(USER) grant, so + // every one is inserted as a principal resource. + for extID := range m.extUsers { + set[userResourceType.GetId()+"/"+extID] = struct{}{} + } + for gid := range m.groups { + set[groupResourceType.GetId()+"/"+gid] = struct{}{} + } + for orgID, org := range m.orgs { + set[equivOrgRT.GetId()+"/"+orgID] = struct{}{} + for projID := range org.projects { + set[equivProjectRT.GetId()+"/"+projID] = struct{}{} + } + for repoID := range org.repoGrants { + set[equivRepoRT.GetId()+"/"+repoID] = struct{}{} + } + } + for tid := range m.teams { + set[equivTeamRT.GetId()+"/"+tid] = struct{}{} + } + return sortedKeys(set) +} + +// expectedEntitlementIDs derives the COMPLETE entitlement set the model +// implies: one member entitlement per group (including the external-match +// group — its entitlement anchors the transformed grants) and one per +// type-scoped team. Users, projects, and repos are skip-annotated or +// never enumerated; orgs serve no entitlements; expansion and external +// processing create no entitlement rows. +func expectedEntitlementIDs(m *equivModel) []string { + set := map[string]struct{}{} + for gid, g := range m.groups { + set[equivMemberEnt(equivGroupResource(gid), g.exclusionDefault).GetId()] = struct{}{} + } + for tid := range m.teams { + set[equivMemberEnt(equivTeamResource(tid), false).GetId()] = struct{}{} + } + return sortedKeys(set) +} + // expectedGrantIDs derives the COMPLETE grant set the model implies: // group memberships, the expandable membership plus its expansion-derived // grants, match-transformed external grants, repo (InsertResourceGrants) @@ -1707,9 +1758,13 @@ func assertColdArtifacts(t *testing.T, round int, m *equivModel, snap *c1zSnapsh require.NotContains(t, id, "placeholder-", "round %d: coverage: untransformed placeholder grant leaked into output", round) } - // The independent oracle: the cold grant set must EXACTLY equal the - // model-derived expectation — the check that doesn't trust the - // syncer code both outputs share. + // The independent oracle: the cold output's resource, entitlement, + // and grant sets must EXACTLY equal the model-derived expectations — + // the checks that don't trust the syncer code both outputs share. + require.Equal(t, expectedResourceKeys(m), sortedKeys(snap.resources), + "round %d: cold output's resource set must equal the model-derived expectation", round) + require.Equal(t, expectedEntitlementIDs(m), sortedKeys(snap.entitlements), + "round %d: cold output's entitlement set must equal the model-derived expectation", round) require.Equal(t, expectedGrantIDs(m), sortedKeys(snap.grants), "round %d: cold output's grant set must equal the model-derived expectation", round) } diff --git a/pkg/sync/source_cache_halt_test.go b/pkg/sync/source_cache_halt_test.go new file mode 100644 index 000000000..fab3c4730 --- /dev/null +++ b/pkg/sync/source_cache_halt_test.go @@ -0,0 +1,115 @@ +package sync //nolint:revive,nolintlint // we can't change the package name for backwards compatibility + +// Halt-point sweep: crash the warm sync at EVERY ordering-sensitive seam +// of the scope lifecycle (via the syncer's testSourceCacheHaltHook), +// resume the same file, and require the resumed output to be semantically +// identical to a cold sync — the FoundationDB-style "halt at interesting +// points" check, scoped to the seams whose ordering bugs would produce +// silent bad data (e.g. a manifest entry committed ahead of its rows +// would poison the NEXT sync's replay while this sync reads clean). +// +// Seams swept (see beginSourceCachePage / finishSourceCachePage): +// - replay-copied: after the engine replay copy commits, before +// overlay rows / tombstones / manifest. +// - rows-committed: after the page's rows committed, before +// tombstones. +// - tombstones-applied: after delta tombstones applied, before the +// manifest entry write. +// - manifest-written: after the manifest entry committed — the +// double-execution seam (the resumed action +// re-runs a page whose scope already sealed). + +import ( + "fmt" + "math/rand" + "path/filepath" + stdsync "sync" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/conductorone/baton-sdk/pkg/logging" + "github.com/conductorone/baton-sdk/pkg/types" +) + +// haltOnce arms a one-shot failure at the named stage; every later hook +// call (including after resume, where the hook is still installed) passes. +type haltOnce struct { + mu stdsync.Mutex + stage string + fired bool + scope string +} + +func (h *haltOnce) hook(stage, scopeKey string) error { + h.mu.Lock() + defer h.mu.Unlock() + if h.fired || stage != h.stage { + return nil + } + h.fired = true + h.scope = scopeKey + return fmt.Errorf("injected halt at %s (scope %q)", stage, scopeKey) +} + +func TestSourceCache_HaltPointSweep(t *testing.T) { + stages := []string{"replay-copied", "rows-committed", "tombstones-applied", "manifest-written"} + for _, stage := range stages { + t.Run(stage, func(t *testing.T) { + runHaltPointScenario(t, stage) + }) + } +} + +func runHaltPointScenario(t *testing.T, stage string) { + ctx, err := logging.Init(t.Context()) + require.NoError(t, err) + tmpDir := t.TempDir() + rng := rand.New(rand.NewSource(42)) //nolint:gosec // deterministic test randomness + + model := newEquivModel() + warmConn := newEquivConnector(model, false) + coldConn := newEquivConnector(model, true) + var warmClient types.ConnectorClient = warmConn + var coldClient types.ConnectorClient = coldConn + + extPath := filepath.Join(tmpDir, "external-r0.c1z") + buildExternalC1z(ctx, t, model, extPath, tmpDir) + model.extDirty = false + syncOpts := []SyncOpt{WithExternalResourceC1ZPath(extPath)} + + // Round 0: initial warm-chain sync (a cold fetch — no previous file). + basePath := filepath.Join(tmpDir, "warm-r0.c1z") + require.NoError(t, runEquivSync(ctx, t, warmClient, basePath, "", tmpDir, syncOpts...)) + + // Round 1 mutations: tombstones, overlays, cold refetches, external + // churn — the full mixed shape, so every swept seam actually fires. + model.mutate(1, rng) + if model.extDirty { + extPath = filepath.Join(tmpDir, "external-r1.c1z") + buildExternalC1z(ctx, t, model, extPath, tmpDir) + syncOpts = []SyncOpt{WithExternalResourceC1ZPath(extPath)} + } + + // Warm sync, halted once at the stage under test, then resumed. + halt := &haltOnce{stage: stage} + warmPath := filepath.Join(tmpDir, "warm-halt.c1z") + hookOpt := func(s *syncer) { s.testSourceCacheHaltHook = halt.hook } + haltErr := runEquivSync(ctx, t, warmClient, warmPath, basePath, tmpDir, append(syncOpts, hookOpt)...) + require.Error(t, haltErr, "the armed halt must crash the warm sync") + require.Contains(t, haltErr.Error(), "injected halt at "+stage) + require.NoError(t, runEquivSync(ctx, t, warmClient, warmPath, basePath, tmpDir, append(syncOpts, hookOpt)...), + "resume after halt at %s (scope %q) must complete", stage, halt.scope) + + // Cold reference for the same model state. + coldPath := filepath.Join(tmpDir, "cold-r1.c1z") + require.NoError(t, runEquivSync(ctx, t, coldClient, coldPath, "", tmpDir, syncOpts...)) + + coldSnap := snapshotC1z(ctx, t, coldPath) + warmSnap := snapshotC1z(ctx, t, warmPath) + require.Equal(t, 1, warmSnap.syncRuns, + "halt at %s: the crashed warm sync must RESUME (one sync run), not restart", stage) + assertColdArtifacts(t, 1, model, coldSnap) + assertNoStaleResources(t, 1, model, "warm", warmSnap) + requireSnapshotsEqual(t, 1, coldSnap, warmSnap) +} diff --git a/pkg/sync/state.go b/pkg/sync/state.go index b309884d1..544dd069a 100644 --- a/pkg/sync/state.go +++ b/pkg/sync/state.go @@ -6,6 +6,7 @@ import ( "fmt" "slices" "sync" + "sync/atomic" "time" "github.com/conductorone/baton-sdk/pkg/sync/expand" @@ -233,16 +234,20 @@ var _ State = &state{} // state is an object used for tracking the current status of a connector sync. It operates like a stack. type state struct { - mtx sync.RWMutex - actions map[string]Action - actionOrder []string - currentActionID uint64 // Counter for generating new action IDs. - entitlementGraph *expand.EntitlementGraph - needsExpansion bool - hasExternalResourceGrants bool - shouldFetchRelatedResources bool - shouldSkipEntitlementsAndGrants bool - shouldSkipGrants bool + mtx sync.RWMutex + actions map[string]Action + actionOrder []string + currentActionID uint64 // Counter for generating new action IDs. + entitlementGraph *expand.EntitlementGraph + // Monotone set-once flags, written concurrently by parallel sync + // workers (grant pages arm needsExpansion / hasExternalResourceGrants + // from any worker goroutine) — atomic, or -race flags every parallel + // sync. Set-once semantics make the ordering irrelevant. + needsExpansion atomic.Bool + hasExternalResourceGrants atomic.Bool + shouldFetchRelatedResources atomic.Bool + shouldSkipEntitlementsAndGrants atomic.Bool + shouldSkipGrants atomic.Bool completedActionsCount uint64 exclusionGroupResourceTypes map[string]string exclusionGroupDefaults map[string]string @@ -320,7 +325,6 @@ func newState() *state { actionOrder: []string{}, currentActionID: 0, entitlementGraph: nil, - needsExpansion: false, exclusionGroupResourceTypes: make(map[string]string), exclusionGroupDefaults: make(map[string]string), exclusionGroupCounts: make(map[string]uint32), @@ -444,12 +448,12 @@ func (st *state) Unmarshal(input string) error { st.actionOrder = []string{} } st.currentActionID = token.CurrentActionID - st.needsExpansion = token.NeedsExpansion + st.needsExpansion.Store(token.NeedsExpansion) st.entitlementGraph = token.EntitlementGraph - st.hasExternalResourceGrants = token.HasExternalResourceGrants - st.shouldSkipEntitlementsAndGrants = token.ShouldSkipEntitlementsAndGrants - st.shouldSkipGrants = token.ShouldSkipGrants - st.shouldFetchRelatedResources = token.ShouldFetchRelatedResources + st.hasExternalResourceGrants.Store(token.HasExternalResourceGrants) + st.shouldSkipEntitlementsAndGrants.Store(token.ShouldSkipEntitlementsAndGrants) + st.shouldSkipGrants.Store(token.ShouldSkipGrants) + st.shouldFetchRelatedResources.Store(token.ShouldFetchRelatedResources) st.completedActionsCount = token.CompletedActionsCount st.exclusionGroupResourceTypes = token.ExclusionGroupResourceTypes if st.exclusionGroupResourceTypes == nil { @@ -512,12 +516,12 @@ func (st *state) Marshal() (string, error) { ActionsMap: st.actions, ActionOrder: st.actionOrder, CurrentActionID: st.currentActionID, - NeedsExpansion: st.needsExpansion, + NeedsExpansion: st.needsExpansion.Load(), EntitlementGraph: st.entitlementGraph, - HasExternalResourceGrants: st.hasExternalResourceGrants, - ShouldFetchRelatedResources: st.shouldFetchRelatedResources, - ShouldSkipEntitlementsAndGrants: st.shouldSkipEntitlementsAndGrants, - ShouldSkipGrants: st.shouldSkipGrants, + HasExternalResourceGrants: st.hasExternalResourceGrants.Load(), + ShouldFetchRelatedResources: st.shouldFetchRelatedResources.Load(), + ShouldSkipEntitlementsAndGrants: st.shouldSkipEntitlementsAndGrants.Load(), + ShouldSkipGrants: st.shouldSkipGrants.Load(), CompletedActionsCount: st.completedActionsCount, ExclusionGroupResourceTypes: st.exclusionGroupResourceTypes, ExclusionGroupDefaults: st.exclusionGroupDefaults, @@ -717,43 +721,43 @@ func (st *state) NextPage(ctx context.Context, actionID string, pageToken string } func (st *state) NeedsExpansion() bool { - return st.needsExpansion + return st.needsExpansion.Load() } func (st *state) SetNeedsExpansion() { - st.needsExpansion = true + st.needsExpansion.Store(true) } func (st *state) HasExternalResourcesGrants() bool { - return st.hasExternalResourceGrants + return st.hasExternalResourceGrants.Load() } func (st *state) SetHasExternalResourcesGrants() { - st.hasExternalResourceGrants = true + st.hasExternalResourceGrants.Store(true) } func (st *state) ShouldFetchRelatedResources() bool { - return st.shouldFetchRelatedResources + return st.shouldFetchRelatedResources.Load() } func (st *state) SetShouldFetchRelatedResources() { - st.shouldFetchRelatedResources = true + st.shouldFetchRelatedResources.Store(true) } func (st *state) ShouldSkipEntitlementsAndGrants() bool { - return st.shouldSkipEntitlementsAndGrants + return st.shouldSkipEntitlementsAndGrants.Load() } func (st *state) SetShouldSkipEntitlementsAndGrants() { - st.shouldSkipEntitlementsAndGrants = true + st.shouldSkipEntitlementsAndGrants.Store(true) } func (st *state) ShouldSkipGrants() bool { - return st.shouldSkipGrants + return st.shouldSkipGrants.Load() } func (st *state) SetShouldSkipGrants() { - st.shouldSkipGrants = true + st.shouldSkipGrants.Store(true) } // EntitlementGraph returns the entitlement graph for the current action. diff --git a/pkg/sync/syncer.go b/pkg/sync/syncer.go index bbe5510c8..617f2d8dc 100644 --- a/pkg/sync/syncer.go +++ b/pkg/sync/syncer.go @@ -148,9 +148,16 @@ type syncer struct { // resourcesPhaseRanHere gates I4 to processes that actually ran the // resources phase; strictIngestionInvariants promotes I4 from // warn to hard-fail (tests and the equivalence harness set it). - childSchedule childScheduleSet - resourcesPhaseRanHere bool - strictIngestionInvariants bool + childSchedule childScheduleSet + resourcesPhaseRanHere bool + strictIngestionInvariants bool + // testSourceCacheHaltHook, when non-nil, fires at named seams of the + // scope lifecycle (replay-copied, rows-committed, tombstones-applied, + // manifest-written); returning an error fails the sync at exactly + // that boundary. The halt-point sweep (source_cache_halt_test.go) + // uses it to prove crash/resume equivalence at every + // ordering-sensitive point. Nil in production: one pointer check. + testSourceCacheHaltHook func(stage, scopeKey string) error connector types.ConnectorClient state State runDuration time.Duration From 61405b0744092aa6a970ad428f29de48024688b5 Mon Sep 17 00:00:00 2001 From: Matt Kaniaris Date: Tue, 14 Jul 2026 22:36:27 -0600 Subject: [PATCH 07/17] rename and big code cleanup, plus a fix or two --- cmd/baton/source_cache.go | 110 ++++--- internal/connector/connector.go | 16 +- .../v2/annotation_type_scoped_grants.pb.go | 5 +- ...ation_type_scoped_grants_protoopaque.pb.go | 5 +- .../connectorapi/baton/v1/source_cache.pb.go | 4 +- .../baton/v1/source_cache_protoopaque.pb.go | 2 +- pkg/connectorbuilder/connectorbuilder.go | 8 +- pkg/connectorbuilder/continuation_test.go | 34 +++ pkg/connectorbuilder/resource_syncer.go | 93 +++++- .../type_scoped_entitlements_test.go | 24 ++ pkg/dotc1z/c1file.go | 23 +- pkg/dotc1z/c1file_concurrent_test.go | 12 +- pkg/dotc1z/engine/pebble/entitlements.go | 11 +- pkg/dotc1z/engine/pebble/grants.go | 16 +- pkg/dotc1z/engine/pebble/lookup_test.go | 6 +- pkg/dotc1z/engine/pebble/resources.go | 15 +- pkg/dotc1z/engine/pebble/source_cache.go | 4 +- .../engine/pebble/source_cache_engine_test.go | 4 +- pkg/dotc1z/source_cache.go | 217 +++++++------ pkg/field/defaults.go | 8 +- pkg/sourcecache/continuation.go | 2 +- pkg/sourcecache/grpc_server.go | 2 +- pkg/sourcecache/sourcecache.go | 18 +- pkg/sync/ingest_invariants.go | 39 ++- pkg/sync/ingest_invariants_test.go | 36 +++ pkg/sync/source_cache.go | 289 +++++++++--------- pkg/sync/source_cache_bench_test.go | 134 ++++++++ pkg/sync/source_cache_continuation.go | 4 +- pkg/sync/source_cache_continuation_test.go | 2 +- pkg/sync/source_cache_equivalence_test.go | 4 +- pkg/sync/source_cache_stats.go | 22 +- pkg/sync/source_cache_stats_test.go | 6 +- pkg/sync/source_cache_test.go | 2 +- pkg/sync/state.go | 21 +- pkg/sync/syncer.go | 220 ++----------- pkg/sync/type_scoped.go | 148 +++++++++ pkg/sync/whale_invariants_cost_test.go | 129 ++++++++ pkg/tasks/c1api/full_sync.go | 14 +- pkg/tasks/c1api/manager.go | 2 +- pkg/types/resource/resource.go | 2 +- .../v2/annotation_type_scoped_grants.proto | 5 +- .../connectorapi/baton/v1/source_cache.proto | 2 +- 42 files changed, 1130 insertions(+), 590 deletions(-) create mode 100644 pkg/sync/source_cache_bench_test.go create mode 100644 pkg/sync/type_scoped.go create mode 100644 pkg/sync/whale_invariants_cost_test.go diff --git a/cmd/baton/source_cache.go b/cmd/baton/source_cache.go index 99386fbd4..eb82539b4 100644 --- a/cmd/baton/source_cache.go +++ b/cmd/baton/source_cache.go @@ -12,8 +12,9 @@ import ( "github.com/spf13/cobra" - pebbleengine "github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble" + "github.com/conductorone/baton-sdk/pkg/dotc1z" "github.com/conductorone/baton-sdk/pkg/logging" + "github.com/conductorone/baton-sdk/pkg/sourcecache" ) // source-cache: the attribution surface for source-cache replay. Rows in @@ -22,6 +23,9 @@ import ( // two so "why is this row missing / stale?" becomes a per-scope answer: // when was the scope last actually fetched, what validator did it record, // how many rows does it hold, and is its index/manifest state consistent. +// +// Exits non-zero when any orphan-index scope is found (ingestion +// invariant I6): such a file would poison a future sync's replay. func sourceCacheCmd() *cobra.Command { cmd := &cobra.Command{ Use: "source-cache", @@ -45,6 +49,18 @@ type sourceCacheScopeReport struct { Status string `json:"status"` } +type sourceCacheAuditReport struct { + SyncID string `json:"sync_id,omitempty"` + // SyncType / Compacted / UsableAsReplaySource describe whether a + // FUTURE sync may use this file as its replay source (the metadata + // gate; see c1zstore.SyncRun.UsableAsReplaySource). + SyncType string `json:"sync_type,omitempty"` + Compacted bool `json:"compacted"` + UsableAsReplaySource bool `json:"usable_as_replay_source"` + Scopes []sourceCacheScopeReport `json:"scopes"` + OrphanIndexScopes int `json:"orphan_index_scopes"` +} + func runSourceCacheAudit(cmd *cobra.Command, args []string) error { ctx, err := logging.Init(context.Background(), logging.WithLogFormat("console"), logging.WithLogLevel("error")) if err != nil { @@ -65,75 +81,95 @@ func runSourceCacheAudit(cmd *cobra.Command, args []string) error { } defer store.Close(ctx) - holder, ok := store.(interface{ PebbleEngine() *pebbleengine.Engine }) - var engine *pebbleengine.Engine - if ok { - engine = holder.PebbleEngine() - } - if engine == nil { + inspector, ok := store.(dotc1z.SourceCacheInspector) + if !ok { return fmt.Errorf("source-cache state exists only in pebble-format c1z files; %q is not one", c1zPath) } - manifest, err := engine.SourceCacheManifestSnapshot(ctx) + report := sourceCacheAuditReport{} + if latest, err := store.SyncMeta().LatestFullSync(ctx); err == nil && latest != nil { + report.SyncID = latest.ID + report.SyncType = string(latest.Type) + report.Compacted = latest.Compacted + report.UsableAsReplaySource = latest.UsableAsReplaySource() + } + + manifest, err := inspector.SourceCacheManifestSnapshot(ctx) if err != nil { return fmt.Errorf("reading source-cache manifest: %w", err) } - indexCounts, err := engine.SourceScopeIndexSnapshot(ctx) + indexCounts, err := inspector.SourceScopeIndexSnapshot(ctx) if err != nil { return fmt.Errorf("reading scope indexes: %w", err) } + orphans, err := inspector.SourceCacheOrphanScopes(ctx) + if err != nil { + return fmt.Errorf("checking scope consistency: %w", err) + } - reports := make([]sourceCacheScopeReport, 0, len(manifest)) for key := range manifest { kind, scope, found := strings.Cut(key, "\x00") if !found { return fmt.Errorf("malformed manifest snapshot key %q", key) } rep := sourceCacheScopeReport{RowKind: kind, ScopeKey: scope, Rows: indexCounts[kind][scope]} - rec, err := engine.GetSourceCacheEntry(ctx, kind, scope) + entry, entryFound, err := inspector.LookupSourceCacheEntry(ctx, sourcecache.RowKind(kind), scope) if err != nil { return fmt.Errorf("reading manifest entry for %s/%q: %w", kind, scope, err) } - rep.Validator = rec.GetCacheValidator() - rep.DiscoveredAt = rec.GetDiscoveredAt().AsTime() + if entryFound { + rep.Validator = entry.CacheValidator + rep.DiscoveredAt = entry.DiscoveredAt + } if rep.Rows == 0 { rep.Status = "zero-row" } else { rep.Status = "ok" } - reports = append(reports, rep) - } - // Index scopes with no manifest entry: invariant I6 violations. - for kind, scopes := range indexCounts { - for scope, n := range scopes { - if _, ok := manifest[kind+"\x00"+scope]; !ok { - reports = append(reports, sourceCacheScopeReport{ - RowKind: kind, ScopeKey: scope, Rows: n, Status: "orphan-index", - }) - } + report.Scopes = append(report.Scopes, rep) + } + for kind, scopes := range orphans { + for _, scope := range scopes { + report.Scopes = append(report.Scopes, sourceCacheScopeReport{ + RowKind: kind, ScopeKey: scope, Rows: indexCounts[kind][scope], Status: "orphan-index", + }) + report.OrphanIndexScopes++ } } - sort.Slice(reports, func(i, j int) bool { - if reports[i].RowKind != reports[j].RowKind { - return reports[i].RowKind < reports[j].RowKind + sort.Slice(report.Scopes, func(i, j int) bool { + if report.Scopes[i].RowKind != report.Scopes[j].RowKind { + return report.Scopes[i].RowKind < report.Scopes[j].RowKind } - return reports[i].ScopeKey < reports[j].ScopeKey + return report.Scopes[i].ScopeKey < report.Scopes[j].ScopeKey }) if outputFormat == "json" { enc := json.NewEncoder(os.Stdout) enc.SetIndent("", " ") - return enc.Encode(reports) + if err := enc.Encode(report); err != nil { + return err + } + } else if err := printSourceCacheAudit(report); err != nil { + return err } + if report.OrphanIndexScopes > 0 { + return fmt.Errorf("%d orphan-index scope(s) found (ingestion invariant I6); a future sync replaying from this file could silently drop or resurrect rows", report.OrphanIndexScopes) + } + return nil +} - if len(reports) == 0 { +func printSourceCacheAudit(report sourceCacheAuditReport) error { + if report.SyncID != "" { + fmt.Fprintf(os.Stdout, "sync %s type=%s compacted=%v usable-as-replay-source=%v\n\n", + report.SyncID, report.SyncType, report.Compacted, report.UsableAsReplaySource) + } + if len(report.Scopes) == 0 { fmt.Fprintln(os.Stdout, "no source-cache state (cold artifact: every scope will fetch fresh next sync)") return nil } w := tabwriter.NewWriter(os.Stdout, 2, 4, 2, ' ', 0) fmt.Fprintln(w, "KIND\tSCOPE\tROWS\tSTATUS\tDISCOVERED\tVALIDATOR") - orphans := 0 - for _, r := range reports { + for _, r := range report.Scopes { validator := r.Validator if len(validator) > 48 { validator = validator[:45] + "..." @@ -142,17 +178,7 @@ func runSourceCacheAudit(cmd *cobra.Command, args []string) error { if !r.DiscoveredAt.IsZero() { discovered = r.DiscoveredAt.UTC().Format(time.RFC3339) } - if r.Status == "orphan-index" { - orphans++ - } fmt.Fprintf(w, "%s\t%s\t%d\t%s\t%s\t%s\n", r.RowKind, r.ScopeKey, r.Rows, r.Status, discovered, validator) } - if err := w.Flush(); err != nil { - return err - } - if orphans > 0 { - fmt.Fprintf(os.Stdout, "\nWARNING: %d orphan-index scope(s): stamped rows with no manifest entry (ingestion invariant I6). "+ - "A future sync replaying from this file could silently drop or resurrect these scopes' rows.\n", orphans) - } - return nil + return w.Flush() } diff --git a/internal/connector/connector.go b/internal/connector/connector.go index 066fcff7d..336a8949e 100644 --- a/internal/connector/connector.go +++ b/internal/connector/connector.go @@ -56,12 +56,12 @@ type connectorClient struct { connectorV2.TicketsServiceClient connectorV2.ActionServiceClient - sessionStoreSetter sessions.SetSessionStore // this is the session store server - sourceCacheSetter sourcecache.SetLookup // this is the source-cache lookup server + sessionStoreSetter sessions.SetSessionStore // this is the session store server + sourceCacheSetter sourcecache.SourceCacheSetter // this is the source-cache lookup server } var _ sessions.SetSessionStore = (*connectorClient)(nil) -var _ sourcecache.SetLookup = (*connectorClient)(nil) +var _ sourcecache.SourceCacheSetter = (*connectorClient)(nil) var _ SetSessionStoreSetter = (*connectorClient)(nil) var _ SetSourceCacheSetter = (*connectorClient)(nil) @@ -70,14 +70,14 @@ type SetSessionStoreSetter interface { } type SetSourceCacheSetter interface { - SetSourceCacheSetter(sourceCacheSetter sourcecache.SetLookup) + SetSourceCacheSetter(sourceCacheSetter sourcecache.SourceCacheSetter) } func (c *connectorClient) SetSessionStoreSetter(sessionStoreSetter sessions.SetSessionStore) { c.sessionStoreSetter = sessionStoreSetter } -func (c *connectorClient) SetSourceCacheSetter(sourceCacheSetter sourcecache.SetLookup) { +func (c *connectorClient) SetSourceCacheSetter(sourceCacheSetter sourcecache.SourceCacheSetter) { c.sourceCacheSetter = sourceCacheSetter } @@ -135,7 +135,7 @@ type wrapper struct { now func() time.Time SessionServer sessions.SetSessionStore - SourceCacheServer sourcecache.SetLookup + SourceCacheServer sourcecache.SourceCacheSetter } type Option func(ctx context.Context, w *wrapper) error @@ -224,9 +224,9 @@ func NewWrapper(ctx context.Context, server interface{}, opts ...Option) (*wrapp now: time.Now, } // In-process delivery: a connectorbuilder-based server implements - // sourcecache.SetLookup itself, so the syncer's per-sync lookup can be + // sourcecache.SourceCacheSetter itself, so the syncer's per-sync lookup can be // installed without the subprocess gRPC hop. - if sourceCacheServer, ok := connectorServer.(sourcecache.SetLookup); ok { + if sourceCacheServer, ok := connectorServer.(sourcecache.SourceCacheSetter); ok { w.SourceCacheServer = sourceCacheServer } diff --git a/pb/c1/connector/v2/annotation_type_scoped_grants.pb.go b/pb/c1/connector/v2/annotation_type_scoped_grants.pb.go index 07fc8ee76..609293177 100644 --- a/pb/c1/connector/v2/annotation_type_scoped_grants.pb.go +++ b/pb/c1/connector/v2/annotation_type_scoped_grants.pb.go @@ -41,8 +41,9 @@ const ( // for the whole type, across as many paginated cursors as it chooses to // spawn (see EnqueuePageTokens). // -// This is the grants-phase analogue of StaticEntitlements: the connector -// takes over enumeration for the type, and every downstream behavior +// The connector takes over enumeration for the type (the same contract +// as TypeScopedEntitlements in +// annotation_type_scoped_entitlements.proto), and every downstream behavior // (storage, source-cache scopes/replay/tombstones, grant expansion, rate // limiting, page-token checkpointing) is unchanged because grants are // self-describing rows. diff --git a/pb/c1/connector/v2/annotation_type_scoped_grants_protoopaque.pb.go b/pb/c1/connector/v2/annotation_type_scoped_grants_protoopaque.pb.go index 00baef3c4..862fc1fc6 100644 --- a/pb/c1/connector/v2/annotation_type_scoped_grants_protoopaque.pb.go +++ b/pb/c1/connector/v2/annotation_type_scoped_grants_protoopaque.pb.go @@ -41,8 +41,9 @@ const ( // for the whole type, across as many paginated cursors as it chooses to // spawn (see EnqueuePageTokens). // -// This is the grants-phase analogue of StaticEntitlements: the connector -// takes over enumeration for the type, and every downstream behavior +// The connector takes over enumeration for the type (the same contract +// as TypeScopedEntitlements in +// annotation_type_scoped_entitlements.proto), and every downstream behavior // (storage, source-cache scopes/replay/tombstones, grant expansion, rate // limiting, page-token checkpointing) is unchanged because grants are // self-describing rows. diff --git a/pb/c1/connectorapi/baton/v1/source_cache.pb.go b/pb/c1/connectorapi/baton/v1/source_cache.pb.go index ef50c399a..0b2bc4823 100644 --- a/pb/c1/connectorapi/baton/v1/source_cache.pb.go +++ b/pb/c1/connectorapi/baton/v1/source_cache.pb.go @@ -27,7 +27,7 @@ type LookupRequest struct { state protoimpl.MessageState `protogen:"hybrid.v1"` // Row kind: resources / entitlements / grants // (pkg/sourcecache.RowKind values). Entries are partitioned by row - // kind, so one scope hash can carry a different validator per kind. + // kind, so one scope key can carry a different validator per kind. RowKind string `protobuf:"bytes,1,opt,name=row_kind,json=rowKind,proto3" json:"row_kind,omitempty"` // Connector-defined stable scope identifier (a stable identifier for the canonical scope // (often a hex hash; see pkg/sourcecache.HashScope). Opaque to the @@ -90,7 +90,7 @@ type LookupRequest_builder struct { // Row kind: resources / entitlements / grants // (pkg/sourcecache.RowKind values). Entries are partitioned by row - // kind, so one scope hash can carry a different validator per kind. + // kind, so one scope key can carry a different validator per kind. RowKind string // Connector-defined stable scope identifier (a stable identifier for the canonical scope // (often a hex hash; see pkg/sourcecache.HashScope). Opaque to the diff --git a/pb/c1/connectorapi/baton/v1/source_cache_protoopaque.pb.go b/pb/c1/connectorapi/baton/v1/source_cache_protoopaque.pb.go index 50e612196..a645a377a 100644 --- a/pb/c1/connectorapi/baton/v1/source_cache_protoopaque.pb.go +++ b/pb/c1/connectorapi/baton/v1/source_cache_protoopaque.pb.go @@ -83,7 +83,7 @@ type LookupRequest_builder struct { // Row kind: resources / entitlements / grants // (pkg/sourcecache.RowKind values). Entries are partitioned by row - // kind, so one scope hash can carry a different validator per kind. + // kind, so one scope key can carry a different validator per kind. RowKind string // Connector-defined stable scope identifier (a stable identifier for the canonical scope // (often a hex hash; see pkg/sourcecache.HashScope). Opaque to the diff --git a/pkg/connectorbuilder/connectorbuilder.go b/pkg/connectorbuilder/connectorbuilder.go index 17677d98d..34c67f46d 100644 --- a/pkg/connectorbuilder/connectorbuilder.go +++ b/pkg/connectorbuilder/connectorbuilder.go @@ -204,6 +204,9 @@ func NewConnector(ctx context.Context, in interface{}, opts ...Opt) (types.Conne if cb, ok := in.(ConnectorBuilder); ok { for _, rb := range cb.ResourceSyncers(ctx) { rType := rb.ResourceType(ctx) + if err := validateTypeScopedRegistration(rType, rb); err != nil { + return nil, err + } if err := addResourceType(ctx, rType.GetId(), rb); err != nil { return nil, err } @@ -214,6 +217,9 @@ func NewConnector(ctx context.Context, in interface{}, opts ...Opt) (types.Conne if cb2, ok := in.(ConnectorBuilderV2); ok { for _, rb := range cb2.ResourceSyncers(ctx) { rType := rb.ResourceType(ctx) + if err := validateTypeScopedRegistration(rType, rb); err != nil { + return nil, err + } if err := addResourceType(ctx, rType.GetId(), rb); err != nil { return nil, err } @@ -269,7 +275,7 @@ func WithSourceCache(lookup sourcecache.Lookup) Opt { } } -// SetSourceCache implements sourcecache.SetLookup so an in-process runner +// SetSourceCache implements sourcecache.SourceCacheSetter so an in-process runner // (the syncer, via its connector client) can install/clear the active // lookup per sync. Subprocess mode instead routes lookups over // BatonSourceCacheService and never calls this. diff --git a/pkg/connectorbuilder/continuation_test.go b/pkg/connectorbuilder/continuation_test.go index cf679d151..b22db4100 100644 --- a/pkg/connectorbuilder/continuation_test.go +++ b/pkg/connectorbuilder/continuation_test.go @@ -99,6 +99,40 @@ func continuationGrantsRequest(annos annotations.Annotations) *v2.GrantsServiceL }.Build() } +// clearedSlotTestLookup is a trivially non-noop direct lookup for the +// cleared-slot test below. +type clearedSlotTestLookup struct{} + +func (clearedSlotTestLookup) Lookup(context.Context, sourcecache.RowKind, string) (sourcecache.Entry, bool, error) { + return sourcecache.Entry{CacheValidator: "current"}, true, nil +} + +// TestContinuation_SurvivesClearedLookupSlot pins the slot-semantics +// contract between SetSourceCache and continuationOpAttrs: clearing the +// slot (SetSourceCache(nil) stores NoopLookup so late RPCs read a +// deterministic miss) must NOT count as "a direct lookup is installed" — +// that would silently disable the ask/answer continuation for every +// later request, degrading to permanent cold syncs with no error. +func TestContinuation_SurvivesClearedLookupSlot(t *testing.T) { + ctx := context.Background() + ts := &continuationTestSyncer{testResourceSyncerV2Simple: testResourceSyncerV2Simple{resourceType: "group"}, scope: "groups/g1/members"} + b := newContinuationTestBuilder(t, ts) + + // Install then clear, as a completed direct-topology sync would. + b.SetSourceCache(ctx, clearedSlotTestLookup{}) + b.SetSourceCache(ctx, nil) + + // A continuation request must still defer into an ask. + resp, err := b.ListGrants(ctx, continuationGrantsRequest(annotations.New(&v2.SourceCacheLookupOffer{}))) + require.NoError(t, err) + require.Empty(t, resp.GetList()) + ask := &v2.SourceCacheLookupAsk{} + respAnnos := annotations.Annotations(resp.GetAnnotations()) + hasAsk, err := respAnnos.Pick(ask) + require.NoError(t, err) + require.True(t, hasAsk, "a cleared lookup slot must not disable the continuation") +} + func TestContinuation_DeferThenAnswer(t *testing.T) { ctx := context.Background() ts := &continuationTestSyncer{testResourceSyncerV2Simple: testResourceSyncerV2Simple{resourceType: "group"}, scope: "groups/g1/members"} diff --git a/pkg/connectorbuilder/resource_syncer.go b/pkg/connectorbuilder/resource_syncer.go index e48052cf7..bcaf3d166 100644 --- a/pkg/connectorbuilder/resource_syncer.go +++ b/pkg/connectorbuilder/resource_syncer.go @@ -67,12 +67,14 @@ type StaticEntitlementSyncerV2 interface { StaticEntitlements(ctx context.Context, opts resource.SyncOpAttrs) ([]*v2.Entitlement, *resource.SyncOpResults, error) } -// TypeScopedGrantsSyncer is the grants-phase analogue of -// StaticEntitlementSyncerV2: the connector enumerates grants for a WHOLE +// TypeScopedGrantsSyncer lets a connector enumerate grants for a WHOLE // resource type instead of being called once per resource. Implement it on // a resource syncer whose ResourceType carries the v2.TypeScopedGrants -// annotation; the syncer then issues ListGrants calls with an empty -// resource id, which the builder routes here. +// annotation; the syncer then issues ListGrants requests carrying the +// v2.TypeScopedGrants REQUEST annotation as the routing marker (the +// request's resource is a self-referential {type, type} stub — wire +// validation requires a non-empty resource id), which the builder routes +// here. // // The first call of a sync arrives with an empty page token (the planning // call); the connector may answer with rows directly and/or spawn @@ -80,6 +82,12 @@ type StaticEntitlementSyncerV2 interface { // whose page tokens are delivered back through opts.PageToken, one action // each. Source-cache scope/replay/tombstone annotations work exactly as on // per-resource grants pages. +// +// The per-resource Grants method is still required by ResourceSyncerV2 +// but is NEVER called for an annotated type during a full sync; the +// convention is to return empty results (registration fails loudly if +// the annotation is present without this interface — see +// validateTypeScopedRegistration). type TypeScopedGrantsSyncer interface { GrantsForResourceType(ctx context.Context, resourceTypeID string, opts resource.SyncOpAttrs) ([]*v2.Grant, *resource.SyncOpResults, error) } @@ -88,8 +96,9 @@ type TypeScopedGrantsSyncer interface { // TypeScopedGrantsSyncer: the connector enumerates entitlements for a WHOLE // resource type instead of being called once per resource. Implement it on // a resource syncer whose ResourceType carries the v2.TypeScopedEntitlements -// annotation; the syncer then issues ListEntitlements calls with an empty -// resource id, which the builder routes here. +// annotation; the syncer then issues ListEntitlements requests carrying +// that annotation as the routing marker (same {type, type} stub resource +// as the grants variant), which the builder routes here. // // Planning / EnqueuePageTokens / source-cache behavior matches TypeScopedGrantsSyncer. type TypeScopedEntitlementsSyncer interface { @@ -103,7 +112,16 @@ type TypeScopedEntitlementsSyncer interface { // configured store, connectors see the erroring no-op store instead of a // panic on first use. func (b *builder) syncOpAttrs(activeSyncID string, token pagination.Token) resource.SyncOpAttrs { - sc := b.sourceCacheLookup() + return b.syncOpAttrsWithLookup(activeSyncID, token, b.sourceCacheLookup()) +} + +// syncOpAttrsWithLookup is syncOpAttrs over a caller-held slot snapshot: +// SetSourceCache can install/clear the slot concurrently with an +// in-flight RPC, so a caller that also BRANCHES on the slot value +// (continuationOpAttrs) must read it exactly once and thread the same +// snapshot into both decisions — two reads could hand a connector +// NoopLookup while treating the topology as direct. +func (b *builder) syncOpAttrsWithLookup(activeSyncID string, token pagination.Token, sc sourcecache.Lookup) resource.SyncOpAttrs { if sc == nil { sc = sourcecache.NoopLookup{} } @@ -134,9 +152,23 @@ func (b *builder) syncOpAttrs(activeSyncID string, token pagination.Token) resou // The returned ContinuationLookup is nil when the continuation is not in // play for this request. func (b *builder) continuationOpAttrs(activeSyncID string, token pagination.Token, reqAnnos annotations.Annotations) (resource.SyncOpAttrs, *sourcecache.ContinuationLookup, error) { - attrs := b.syncOpAttrs(activeSyncID, token) - if b.sourceCacheLookup() != nil { - return attrs, nil, nil + // One slot read for both the attrs and the topology branch below — + // see syncOpAttrsWithLookup for why two reads would be a race. + slot := b.sourceCacheLookup() + attrs := b.syncOpAttrsWithLookup(activeSyncID, token, slot) + if l := slot; l != nil { + // A NoopLookup slot is a CLEARED slot, not a direct lookup: + // SetSourceCache(nil) stores NoopLookup so late in-flight RPCs + // read a deterministic miss (see SetSourceCache), and treating + // it as "direct lookup wins" here would silently disable the + // ask/answer continuation for every later request — permanent + // cold syncs with no error. A degraded sync can't sneak into + // continuation through this branch either: without a warm + // lookup the syncer never attaches SourceCacheLookupOffer, so + // the offer check below already returns the noop path. + if _, isNoop := l.(sourcecache.NoopLookup); !isNoop { + return attrs, nil, nil + } } answersMsg := &v2.SourceCacheLookupAnswers{} hasAnswers, err := reqAnnos.Pick(answersMsg) @@ -270,7 +302,10 @@ func (b *builder) ListResources(ctx context.Context, request *v2.ResourcesServic } if deferred { b.m.RecordTaskSuccess(ctx, tt, b.nowFunc().Sub(start)) - return v2.ResourcesServiceListResourcesResponse_builder{Annotations: askAnnos}.Build(), nil + // Session usage rides the ask too: the phase-1 handler may have + // touched the session before deferring, and the protocol turn is + // the only response that work will ever produce. + return v2.ResourcesServiceListResourcesResponse_builder{Annotations: appendSessionUsage(askAnnos, sessionUsage)}.Build(), nil } resp := v2.ResourcesServiceListResourcesResponse_builder{ @@ -451,7 +486,8 @@ func (b *builder) ListEntitlements(ctx context.Context, request *v2.Entitlements } if deferred { b.m.RecordTaskSuccess(ctx, tt, b.nowFunc().Sub(start)) - return v2.EntitlementsServiceListEntitlementsResponse_builder{Annotations: askAnnos}.Build(), nil + // Session usage rides the ask; see the ListResources deferred path. + return v2.EntitlementsServiceListEntitlementsResponse_builder{Annotations: appendSessionUsage(askAnnos, sessionUsage)}.Build(), nil } resp := v2.EntitlementsServiceListEntitlementsResponse_builder{ @@ -544,7 +580,8 @@ func (b *builder) ListGrants(ctx context.Context, request *v2.GrantsServiceListG } if deferred { b.m.RecordTaskSuccess(ctx, tt, b.nowFunc().Sub(start)) - return v2.GrantsServiceListGrantsResponse_builder{Annotations: askAnnos}.Build(), nil + // Session usage rides the ask; see the ListResources deferred path. + return v2.GrantsServiceListGrantsResponse_builder{Annotations: appendSessionUsage(askAnnos, sessionUsage)}.Build(), nil } resp := v2.GrantsServiceListGrantsResponse_builder{ @@ -613,6 +650,36 @@ func (rw *resourceSyncerV1toV2) Grants(ctx context.Context, r *v2.Resource, opts return grants, ret, err } +// validateTypeScopedRegistration fails connector construction when a +// resource type's annotations promise a type-scoped phase the syncer +// cannot serve: a TypeScopedGrants / TypeScopedEntitlements annotation +// without the matching TypeScoped*Syncer interface would otherwise die +// mid-sync with InvalidArgument on the first type-scoped call — a +// runtime failure for what is a wiring bug, caught here at build. +// +// The reverse (interface implemented, annotation absent) is deliberately +// allowed: shared embedded bases may implement the interface across many +// syncers while only some types opt in via the annotation, and the +// unreferenced method is harmless. +func validateTypeScopedRegistration(rType *v2.ResourceType, in any) error { + rtAnnos := annotations.Annotations(rType.GetAnnotations()) + if rtAnnos.Contains(&v2.TypeScopedGrants{}) { + if _, ok := in.(TypeScopedGrantsSyncer); !ok { + return fmt.Errorf( + "resource type %s carries the TypeScopedGrants annotation but its syncer does not implement TypeScopedGrantsSyncer", + rType.GetId()) + } + } + if rtAnnos.Contains(&v2.TypeScopedEntitlements{}) { + if _, ok := in.(TypeScopedEntitlementsSyncer); !ok { + return fmt.Errorf( + "resource type %s carries the TypeScopedEntitlements annotation but its syncer does not implement TypeScopedEntitlementsSyncer", + rType.GetId()) + } + } + return nil +} + func (b *builder) addTargetedSyncer(_ context.Context, typeId string, in any) error { if targetedSyncer, ok := in.(ResourceTargetedSyncerLimited); ok { if _, ok := b.resourceTargetedSyncers[typeId]; ok { diff --git a/pkg/connectorbuilder/type_scoped_entitlements_test.go b/pkg/connectorbuilder/type_scoped_entitlements_test.go index b0bb66826..2bec00f4a 100644 --- a/pkg/connectorbuilder/type_scoped_entitlements_test.go +++ b/pkg/connectorbuilder/type_scoped_entitlements_test.go @@ -54,6 +54,30 @@ func typeScopedEntitlementsRequest(annos annotations.Annotations) *v2.Entitlemen }.Build() } +// annotatedNoInterfaceSyncer carries the TypeScopedEntitlements annotation +// on its resource type but does NOT implement TypeScopedEntitlementsSyncer. +type annotatedNoInterfaceSyncer struct { + testResourceSyncerV2Simple +} + +func (s *annotatedNoInterfaceSyncer) ResourceType(ctx context.Context) *v2.ResourceType { + rt := s.testResourceSyncerV2Simple.ResourceType(ctx) + rt.SetAnnotations(annotations.New(&v2.TypeScopedEntitlements{})) + return rt +} + +// TestNewConnector_TypeScopedAnnotationRequiresInterface pins the +// registration contract: an annotated type whose syncer lacks the +// matching TypeScoped*Syncer interface fails at NewConnector instead of +// dying mid-sync with InvalidArgument on the first type-scoped call. +func TestNewConnector_TypeScopedAnnotationRequiresInterface(t *testing.T) { + ctx := context.Background() + bad := &annotatedNoInterfaceSyncer{testResourceSyncerV2Simple{resourceType: "group"}} + _, err := NewConnector(ctx, &testConnectorV2{resourceSyncers: []ResourceSyncerV2{bad}}) + require.Error(t, err) + require.Contains(t, err.Error(), "TypeScopedEntitlements annotation") +} + func TestListEntitlements_TypeScopedRoutes(t *testing.T) { ctx := context.Background() ts := &typeScopedEntitlementsSyncer{testResourceSyncerV2Simple: testResourceSyncerV2Simple{resourceType: "group"}} diff --git a/pkg/dotc1z/c1file.go b/pkg/dotc1z/c1file.go index d858bde7b..564a44e35 100644 --- a/pkg/dotc1z/c1file.go +++ b/pkg/dotc1z/c1file.go @@ -61,6 +61,15 @@ type C1File struct { closed bool closedMu sync.Mutex + // dbClosed is set by closeRawDB just before the pool closes. It is + // the race-safe "no more operations" signal: c.db itself is never + // nil'ed on close (an operation concurrent with Close may already + // hold the pointer, and the unsynchronized nil-write is itself a + // data race against validateDb's read), so validateDb checks this + // flag and anything already past validateDb fails on the closed + // pool's own "sql: database is closed" error instead. + dbClosed atomic.Bool + // bulkLoad defers secondary-index creation on a freshly-created // destination. When set, the per-table non-unique secondary indexes // are dropped right after table creation (instant on the empty table) @@ -815,8 +824,16 @@ func (c *C1File) closeRawDB(ctx context.Context) error { // Copy the rawDb to a local variable to avoid race conditions. rawDb := c.rawDb c.rawDb = nil + // Flag first, then close: operations that haven't passed validateDb + // yet are rejected with ErrDbNotOpen; anything already past it fails + // on the closed pool's own "sql: database is closed". c.db is + // deliberately NOT nil'ed — a concurrent operation may already have + // read the pointer and goqu dereferences it (nil here means a + // SIGSEGV in BeginTx, and the unsynchronized write itself is a data + // race against validateDb's read); the closed pool underneath makes + // the handle inert. + c.dbClosed.Store(true) err = rawDb.Close() - c.db = nil return err } @@ -1418,9 +1435,9 @@ func (c *C1File) countBySyncAndResourceType( return out, nil } -// validateDb ensures that the database has been opened. +// validateDb ensures that the database has been opened and not closed. func (c *C1File) validateDb(ctx context.Context) error { - if c.db == nil { + if c.db == nil || c.dbClosed.Load() { return ErrDbNotOpen } diff --git a/pkg/dotc1z/c1file_concurrent_test.go b/pkg/dotc1z/c1file_concurrent_test.go index 32fb8fa90..84c761d1a 100644 --- a/pkg/dotc1z/c1file_concurrent_test.go +++ b/pkg/dotc1z/c1file_concurrent_test.go @@ -1,9 +1,11 @@ package dotc1z import ( + "errors" "fmt" "os" "path/filepath" + "strings" "sync" "testing" "time" @@ -115,7 +117,11 @@ func TestC1ZConcurrentClose(t *testing.T) { var err error // Close will finish at some point, causing DB operations to fail. defer wg.Done() - // Put grants in a loop until we get a DbNotOpen error. + // Put grants in a loop until Close shuts the database down under + // us. Two benign shapes exist: ErrDbNotOpen (validateDb saw the + // closed flag) or database/sql's "database is closed" (the + // operation passed validateDb just before Close flagged the + // handle, then hit the closed pool). i := 0 for { err = f.PutGrants(ctx, v2.Grant_builder{ @@ -137,7 +143,9 @@ func TestC1ZConcurrentClose(t *testing.T) { }.Build(), }.Build()) if err != nil { - require.ErrorIs(t, err, ErrDbNotOpen) + require.Truef(t, + errors.Is(err, ErrDbNotOpen) || strings.Contains(err.Error(), "database is closed"), + "PutGrants during close error = %v, want ErrDbNotOpen or closed pool", err) break } i++ diff --git a/pkg/dotc1z/engine/pebble/entitlements.go b/pkg/dotc1z/engine/pebble/entitlements.go index 401175571..6ac0e1cd8 100644 --- a/pkg/dotc1z/engine/pebble/entitlements.go +++ b/pkg/dotc1z/engine/pebble/entitlements.go @@ -142,10 +142,11 @@ func (e *Engine) GetEntitlementRecord(ctx context.Context, externalID string) (* } // DeleteEntitlementRecord deletes by raw public id. A missing id is a -// no-op; an ambiguous id is an error (a lossy string must never guess a -// delete). -func (e *Engine) DeleteEntitlementRecord(ctx context.Context, externalID string) error { - return e.withWrite(func() error { +// no-op (returns false); an ambiguous id is an error (a lossy string +// must never guess a delete). Returns whether a row was deleted. +func (e *Engine) DeleteEntitlementRecord(ctx context.Context, externalID string) (bool, error) { + deleted := false + err := e.withWrite(func() error { id, err := e.resolveEntitlementIdentityByExternalID(ctx, externalID) if err != nil { if errors.Is(err, pebble.ErrNotFound) { @@ -179,8 +180,10 @@ func (e *Engine) DeleteEntitlementRecord(ctx context.Context, externalID string) return err } e.noteEntitlementKeyspaceWrite() + deleted = true return nil }) + return deleted, err } func (e *Engine) IterateEntitlements(ctx context.Context, yield func(*v3.EntitlementRecord) bool) error { diff --git a/pkg/dotc1z/engine/pebble/grants.go b/pkg/dotc1z/engine/pebble/grants.go index cacf44ef8..a194738b4 100644 --- a/pkg/dotc1z/engine/pebble/grants.go +++ b/pkg/dotc1z/engine/pebble/grants.go @@ -849,18 +849,10 @@ func (e *Engine) UnsafePutUniqueGrantRecords(ctx context.Context, records ...*v3 if r == nil { continue } - // Dense ingestion facts (see PutGrantRecords). - // Thread-safe: atomic load once armed, CAS + - // concurrent-safe db.Set on the first hit. - if err := e.noteGrantRecordFacts(r); err != nil { - errMu.Lock() - if encErr == nil { - encErr = err - } - errMu.Unlock() - failed.Store(true) - return - } + // No noteGrantRecordFacts here (unlike PutGrantRecords + // and PutGrantRecordsIfNewer): this is the trusted-import + // path — nothing in a bulk-import lifecycle consumes the + // external-match fact, so the hot loop skips the walk. val, err := marshalRecord(r) if err != nil { errMu.Lock() diff --git a/pkg/dotc1z/engine/pebble/lookup_test.go b/pkg/dotc1z/engine/pebble/lookup_test.go index cffafc464..ab67f6f9d 100644 --- a/pkg/dotc1z/engine/pebble/lookup_test.go +++ b/pkg/dotc1z/engine/pebble/lookup_test.go @@ -68,7 +68,7 @@ func TestBareIDEntitlementLookupExactlyOne(t *testing.T) { // Two matches → explicit ambiguity, for reads AND deletes. _, err = e.GetEntitlementRecord(ctx, "shared-id") require.ErrorIs(t, err, ErrAmbiguousExternalID) - err = e.DeleteEntitlementRecord(ctx, "shared-id") + _, err = e.DeleteEntitlementRecord(ctx, "shared-id") require.ErrorIs(t, err, ErrAmbiguousExternalID, "an ambiguous string must never guess a delete") // Both ambiguous rows are intact. @@ -101,7 +101,9 @@ func TestBareIDEntitlementLookupInvalidation(t *testing.T) { require.NoError(t, err, "write must invalidate the cached map") require.Equal(t, "second", got.GetExternalId()) - require.NoError(t, e.DeleteEntitlementRecord(ctx, "first")) + deletedFirst, err := e.DeleteEntitlementRecord(ctx, "first") + require.NoError(t, err) + require.True(t, deletedFirst) _, err = e.GetEntitlementRecord(ctx, "first") require.ErrorIs(t, err, pebble.ErrNotFound, "delete must invalidate the cached map") } diff --git a/pkg/dotc1z/engine/pebble/resources.go b/pkg/dotc1z/engine/pebble/resources.go index 582e5ce35..6a73356d6 100644 --- a/pkg/dotc1z/engine/pebble/resources.go +++ b/pkg/dotc1z/engine/pebble/resources.go @@ -124,8 +124,12 @@ func (e *Engine) GetResourceRecord(ctx context.Context, resourceTypeID, resource return r, nil } -func (e *Engine) DeleteResourceRecord(ctx context.Context, resourceTypeID, resourceID string) error { - return e.withWrite(func() error { +// DeleteResourceRecord deletes one resource row and its index entries. +// Returns whether a row existed (a missing row is a no-op, not an error — +// tombstone semantics). +func (e *Engine) DeleteResourceRecord(ctx context.Context, resourceTypeID, resourceID string) (bool, error) { + deleted := false + err := e.withWrite(func() error { key := encodeResourceKey(resourceTypeID, resourceID) batch := e.db.NewBatch() @@ -146,8 +150,13 @@ func (e *Engine) DeleteResourceRecord(ctx context.Context, resourceTypeID, resou if err := batch.Delete(key, nil); err != nil { return err } - return batch.Commit(writeOpts(e.opts.durability)) + if err := batch.Commit(writeOpts(e.opts.durability)); err != nil { + return err + } + deleted = true + return nil }) + return deleted, err } func (e *Engine) writeResourceIndexes(batch *pebble.Batch, r *v3.ResourceRecord) error { diff --git a/pkg/dotc1z/engine/pebble/source_cache.go b/pkg/dotc1z/engine/pebble/source_cache.go index d19fa6330..73c6fed44 100644 --- a/pkg/dotc1z/engine/pebble/source_cache.go +++ b/pkg/dotc1z/engine/pebble/source_cache.go @@ -281,7 +281,7 @@ func (e *Engine) ClearSourceCacheEntries(ctx context.Context) error { // PutSourceCacheEntry writes the manifest entry for (rowKind, scopeKey). // Zero-row scopes still get entries — the validator must survive to the // next sync even when the scope produced no rows. -func (e *Engine) PutSourceCacheEntry(ctx context.Context, rowKind, scopeKey, etag string) error { +func (e *Engine) PutSourceCacheEntry(ctx context.Context, rowKind, scopeKey, cacheValidator string) error { return e.withWrite(func() error { if err := e.requireCurrentSync(); err != nil { return err @@ -289,7 +289,7 @@ func (e *Engine) PutSourceCacheEntry(ctx context.Context, rowKind, scopeKey, eta rec := &v3.SourceCacheEntryRecord{} rec.SetRowKind(rowKind) rec.SetScopeKey(scopeKey) - rec.SetCacheValidator(etag) + rec.SetCacheValidator(cacheValidator) rec.SetDiscoveredAt(timestamppb.Now()) val, err := marshalRecord(rec) if err != nil { diff --git a/pkg/dotc1z/engine/pebble/source_cache_engine_test.go b/pkg/dotc1z/engine/pebble/source_cache_engine_test.go index 9b8b728cb..6ce970d5d 100644 --- a/pkg/dotc1z/engine/pebble/source_cache_engine_test.go +++ b/pkg/dotc1z/engine/pebble/source_cache_engine_test.go @@ -488,7 +488,9 @@ func TestReplayInvalidatesEntitlementIDLookup(t *testing.T) { // ... and, the actual production stake: a delta tombstone against the // replayed row must DELETE it, not silently no-op. - require.NoError(t, cur.PebbleEngine().DeleteEntitlementRecord(ctx, entID)) + tombstoned, err := cur.PebbleEngine().DeleteEntitlementRecord(ctx, entID) + require.NoError(t, err) + require.True(t, tombstoned, "the tombstone must delete the replayed row, not silently no-op") _, err = cur.PebbleEngine().GetEntitlementRecord(ctx, entID) require.Error(t, err, "tombstoned replayed entitlement must be gone") } diff --git a/pkg/dotc1z/source_cache.go b/pkg/dotc1z/source_cache.go index ff750dc20..b8f447197 100644 --- a/pkg/dotc1z/source_cache.go +++ b/pkg/dotc1z/source_cache.go @@ -29,7 +29,7 @@ type SourceCacheStore interface { // PutSourceCacheEntry writes the current sync's manifest entry for // (kind, scopeKey). Zero-row scopes still get entries. - PutSourceCacheEntry(ctx context.Context, kind sourcecache.RowKind, scopeKey string, etag string) error + PutSourceCacheEntry(ctx context.Context, kind sourcecache.RowKind, scopeKey string, cacheValidator string) error // ReplaySourceCache copies every row stamped with scopeKey from prev // (the previous sync's store, opened read-only) into this store. prev @@ -38,35 +38,61 @@ type SourceCacheStore interface { // failed replay can't leave a phantom hit for the next sync. ReplaySourceCache(ctx context.Context, prev connectorstore.Reader, kind sourcecache.RowKind, scopeKey string) (SourceCacheReplayResult, error) - // DeleteSourceCacheRows removes rows by public canonical ID from the - // current sync, after replay + overlay (delta-query tombstones). - // ID formats per kind: grants and entitlements use their canonical - // IDs; resources use Baton resource BIDs ("bid:r:..."). + // ApplyTombstones applies one page's delta tombstones for a scope + // after replay + overlay, returning the number of rows deleted. // - // Grant resolution is BOUNDED: candidate probing only, never the - // O(all grants) stored-external-id scan. Grants stored under - // connector-custom ids are unreachable here (delete no-ops); such - // connectors use DeleteSourceCacheRowsInScope instead. - DeleteSourceCacheRows(ctx context.Context, kind sourcecache.RowKind, ids []string) error - - // DeleteSourceCacheRowsInScope removes rows stamped with scopeKey by - // bare object id — grants by principal id (no principal type, no - // canonical-id reconstruction), resources by resource id (any type). - // One index scan of the scope per call; a page's tombstones are - // batched into one call. Ids with no matching rows are no-ops. - // Not supported for entitlements. - DeleteSourceCacheRowsInScope(ctx context.Context, kind sourcecache.RowKind, scopeKey string, ids []string) (int64, error) - - // DeleteSourceCacheGrantsByIDInScope removes grant rows stamped with - // scopeKey whose STORED grant id is in ids — works for - // connector-custom grant-id shapes that the global bounded delete - // cannot resolve, and stays bounded by the scope's row count. Ids with - // no matching rows are no-ops. - DeleteSourceCacheGrantsByIDInScope(ctx context.Context, scopeKey string, ids []string) (int64, error) + // - deletedIDs are public canonical IDs (grant/entitlement IDs, or + // resource BIDs "bid:r:..."). Grants resolve WITHIN the scope's + // own rows by stored id, so connector-custom grant-id shapes work + // and the cost stays bounded by the scope's size; entitlements + // and resources resolve by bounded id probing (never an O(all + // rows) scan). + // - deletedPrincipalIDs are bare object ids: grants delete every + // row in the scope whose principal id matches (no principal + // type, no canonical-id reconstruction), resources by resource + // id (any type). Not supported for entitlements. + // + // Ids with no matching rows are no-ops. One scope-index scan per + // call; a page's tombstones are batched into one call. + ApplyTombstones(ctx context.Context, kind sourcecache.RowKind, scopeKey string, deletedIDs, deletedPrincipalIDs []string) (int64, error) + + // SourceCacheOrphanScopes returns, per row kind, scope keys present + // in the by_source_scope indexes with no manifest entry — an + // invariant violation at a sealed sync's quiesce point (ingestion + // invariant I6): the orphaned stamps would poison a future sync's + // replay. + SourceCacheOrphanScopes(ctx context.Context) (map[string][]string, error) } var _ SourceCacheStore = (*pebbleStore)(nil) +// SourceCacheInspector is the read-only inspection surface for tooling +// (the `baton source-cache` audit command) — typed access to the +// manifest, the per-scope index counts, and the I6 orphan check, without +// reaching for the raw engine. +type SourceCacheInspector interface { + // SourceCacheManifestSnapshot returns every manifest entry as + // "row_kind\x00scope_key" -> cache validator. + SourceCacheManifestSnapshot(ctx context.Context) (map[string]string, error) + // SourceScopeIndexSnapshot returns, per row kind, scope_key -> the + // number of by_source_scope index entries. + SourceScopeIndexSnapshot(ctx context.Context) (map[string]map[string]int, error) + // SourceCacheOrphanScopes: see SourceCacheStore. + SourceCacheOrphanScopes(ctx context.Context) (map[string][]string, error) + // LookupSourceCacheEntry: see SourceCacheStore. + LookupSourceCacheEntry(ctx context.Context, kind sourcecache.RowKind, scopeKey string) (sourcecache.Entry, bool, error) +} + +var _ SourceCacheInspector = (*pebbleStore)(nil) + +func (s *pebbleStore) SourceCacheManifestSnapshot(ctx context.Context) (map[string]string, error) { + return s.engine.SourceCacheManifestSnapshot(ctx) +} + +func (s *pebbleStore) SourceScopeIndexSnapshot(ctx context.Context) (map[string]map[string]int, error) { + return s.engine.SourceScopeIndexSnapshot(ctx) +} + // IngestFactStore is the optional store capability backing the syncer's // ingestion invariants (see docs/tasks/source-cache-ingestion-invariants.md): // dense row facts tracked as existence bits, and the ordered-key @@ -90,12 +116,6 @@ type IngestFactStore interface { // HasResourceRecord reports whether a resource row exists. HasResourceRecord(ctx context.Context, resourceTypeID, resourceID string) (bool, error) - - // SourceCacheOrphanScopes returns, per row kind, scope keys present - // in the by_source_scope indexes with no manifest entry — an - // invariant violation at a sealed sync's quiesce point (invariant - // I6): the orphaned stamps would poison a future sync's replay. - SourceCacheOrphanScopes(ctx context.Context) (map[string][]string, error) } var _ IngestFactStore = (*pebbleStore)(nil) @@ -149,11 +169,11 @@ func (s *pebbleStore) LookupSourceCacheEntry(ctx context.Context, kind sourcecac }, true, nil } -func (s *pebbleStore) PutSourceCacheEntry(ctx context.Context, kind sourcecache.RowKind, scopeKey string, etag string) error { +func (s *pebbleStore) PutSourceCacheEntry(ctx context.Context, kind sourcecache.RowKind, scopeKey string, cacheValidator string) error { if err := sourcecache.ValidateRowKind(kind); err != nil { return err } - return s.markDirty(s.engine.PutSourceCacheEntry(ctx, string(kind), scopeKey, etag)) + return s.markDirty(s.engine.PutSourceCacheEntry(ctx, string(kind), scopeKey, cacheValidator)) } func (s *pebbleStore) ReplaySourceCache(ctx context.Context, prev connectorstore.Reader, kind sourcecache.RowKind, scopeKey string) (SourceCacheReplayResult, error) { @@ -182,85 +202,88 @@ func (s *pebbleStore) ReplaySourceCache(ctx context.Context, prev connectorstore return res, nil } -// DeleteSourceCacheRows deletes delta tombstones by public id string. +// ApplyTombstones applies one page's delta tombstones; see the interface +// doc for id semantics. Every delete path returns its row count so the +// syncer's tombstone accounting is symmetric across kinds. // // NOTE on the bare-id lookup safety contract (engine/pebble/lookup.go): -// sync paths normally must not resolve grants by string. This path is a -// deliberate, narrow exception: tombstone ids are strings the connector -// itself emitted for these rows, volumes are delta-sized (not O(rows)), -// and resolution keeps the exactly-one rule — an ambiguous id fails the -// sync loudly rather than guessing a delete, which matches the -// source-cache replay-phase error policy. -func (s *pebbleStore) DeleteSourceCacheRows(ctx context.Context, kind sourcecache.RowKind, ids []string) error { - for _, id := range ids { +// sync paths normally must not resolve rows by string. Canonical-id +// tombstones are a deliberate, narrow exception: the ids are strings the +// connector itself emitted for these rows, volumes are delta-sized (not +// O(rows)), and resolution keeps the exactly-one rule — an ambiguous id +// fails the sync loudly rather than guessing a delete. Grant tombstones +// avoid string resolution entirely by matching stored ids within the +// scope's own rows. +func (s *pebbleStore) ApplyTombstones(ctx context.Context, kind sourcecache.RowKind, scopeKey string, deletedIDs, deletedPrincipalIDs []string) (int64, error) { + var total int64 + + if len(deletedIDs) > 0 { switch kind { case sourcecache.RowKindGrants: - if err := s.markDirty(s.engine.DeleteGrantRecordBounded(ctx, id)); err != nil { - return fmt.Errorf("source cache delete grant %q: %w", id, err) + idSet := make(map[string]struct{}, len(deletedIDs)) + for _, id := range deletedIDs { + idSet[id] = struct{}{} } + deleted, err := s.engine.DeleteGrantsByExternalIDsInScope(ctx, scopeKey, idSet) + if err != nil { + return total, fmt.Errorf("source cache grant-id delete for scope %q: %w", scopeKey, err) + } + total += deleted case sourcecache.RowKindEntitlements: - if err := s.markDirty(s.engine.DeleteEntitlementRecord(ctx, id)); err != nil { - return fmt.Errorf("source cache delete entitlement %q: %w", id, err) + for _, id := range deletedIDs { + deleted, err := s.engine.DeleteEntitlementRecord(ctx, id) + if err != nil { + return total, fmt.Errorf("source cache delete entitlement %q: %w", id, err) + } + if deleted { + total++ + } } case sourcecache.RowKindResources: - r, err := bid.ParseResourceBid(id) - if err != nil { - return fmt.Errorf("source cache delete resource: invalid resource bid %q: %w", id, err) - } - rid := r.GetId() - if err := s.markDirty(s.engine.DeleteResourceRecord(ctx, rid.GetResourceType(), rid.GetResource())); err != nil { - return fmt.Errorf("source cache delete resource %q: %w", id, err) + for _, id := range deletedIDs { + r, err := bid.ParseResourceBid(id) + if err != nil { + return total, fmt.Errorf("source cache delete resource: invalid resource bid %q: %w", id, err) + } + rid := r.GetId() + deleted, err := s.engine.DeleteResourceRecord(ctx, rid.GetResourceType(), rid.GetResource()) + if err != nil { + return total, fmt.Errorf("source cache delete resource %q: %w", id, err) + } + if deleted { + total++ + } } default: - return fmt.Errorf("source cache delete: invalid row kind %q", kind) + return total, fmt.Errorf("source cache delete: invalid row kind %q", kind) } } - return nil -} -func (s *pebbleStore) DeleteSourceCacheGrantsByIDInScope(ctx context.Context, scopeKey string, ids []string) (int64, error) { - if len(ids) == 0 { - return 0, nil - } - idSet := make(map[string]struct{}, len(ids)) - for _, id := range ids { - idSet[id] = struct{}{} - } - deleted, err := s.engine.DeleteGrantsByExternalIDsInScope(ctx, scopeKey, idSet) - if err != nil { - return 0, fmt.Errorf("source cache grant-id delete for scope %q: %w", scopeKey, err) - } - if deleted > 0 { - s.MarkDirty() + if len(deletedPrincipalIDs) > 0 { + idSet := make(map[string]struct{}, len(deletedPrincipalIDs)) + for _, id := range deletedPrincipalIDs { + idSet[id] = struct{}{} + } + var deleted int64 + var err error + switch kind { + case sourcecache.RowKindGrants: + deleted, err = s.engine.DeleteGrantsByPrincipalsInScope(ctx, scopeKey, idSet) + case sourcecache.RowKindResources: + deleted, err = s.engine.DeleteResourcesByIDsInScope(ctx, scopeKey, idSet) + case sourcecache.RowKindEntitlements: + return total, fmt.Errorf("source cache scoped delete: not supported for entitlements") + default: + return total, fmt.Errorf("source cache scoped delete: invalid row kind %q", kind) + } + if err != nil { + return total, fmt.Errorf("source cache scoped delete for scope %q: %w", scopeKey, err) + } + total += deleted } - return deleted, nil -} -func (s *pebbleStore) DeleteSourceCacheRowsInScope(ctx context.Context, kind sourcecache.RowKind, scopeKey string, ids []string) (int64, error) { - if len(ids) == 0 { - return 0, nil - } - idSet := make(map[string]struct{}, len(ids)) - for _, id := range ids { - idSet[id] = struct{}{} - } - var deleted int64 - var err error - switch kind { - case sourcecache.RowKindGrants: - deleted, err = s.engine.DeleteGrantsByPrincipalsInScope(ctx, scopeKey, idSet) - case sourcecache.RowKindResources: - deleted, err = s.engine.DeleteResourcesByIDsInScope(ctx, scopeKey, idSet) - case sourcecache.RowKindEntitlements: - return 0, fmt.Errorf("source cache scoped delete: not supported for entitlements") - default: - return 0, fmt.Errorf("source cache scoped delete: invalid row kind %q", kind) - } - if err != nil { - return 0, fmt.Errorf("source cache scoped delete for scope %q: %w", scopeKey, err) - } - if deleted > 0 { + if total > 0 { s.MarkDirty() } - return deleted, nil + return total, nil } diff --git a/pkg/field/defaults.go b/pkg/field/defaults.go index 40c4e5511..03a76342b 100644 --- a/pkg/field/defaults.go +++ b/pkg/field/defaults.go @@ -281,17 +281,17 @@ var ( WithExportTarget(ExportTargetNone)) // KeepPreviousSyncC1ZField is the CUSTOMER's runtime half of the - // service-mode ETag-replay opt-in: keep the last successfully + // service-mode source-cache-replay opt-in: keep the last successfully // uploaded c1z on disk as a spare and feed it to the next full sync // as the previous-sync replay source. It only takes effect on - // connectors whose author also declared ETag-replay support at build + // connectors whose author also declared source-cache-replay support at build // time (connectorrunner.WithKeepPreviousSyncC1Z) — both are // required. Costs one c1z of local disk. // Hidden by default for the same reason as PreviousSyncC1ZField; unhidden // when the connector author declares the replay capability. KeepPreviousSyncC1ZField = BoolField("keep-previous-sync-c1z", - WithDescription("Keep the previously synced c1z on disk to enable ETag replay across service-mode syncs "+ - "(requires a connector that supports ETag replay; costs one c1z of local disk)"), + WithDescription("Keep the previously synced c1z on disk to enable source-cache replay across service-mode syncs "+ + "(requires a connector that supports source-cache replay; costs one c1z of local disk)"), WithDefaultValue(false), WithPersistent(true), WithHidden(true), diff --git a/pkg/sourcecache/continuation.go b/pkg/sourcecache/continuation.go index 9ef68c040..21f24e5c5 100644 --- a/pkg/sourcecache/continuation.go +++ b/pkg/sourcecache/continuation.go @@ -62,7 +62,7 @@ type BatchLookup interface { // The returned answers are exact and complete for the queried set — one // Answer per Query, order preserved, explicit Found per entry. (A // deferring lookup returns ErrLookupDeferred instead of answers; phase 2 -// then answers the same calls. A found answer whose etag does not fit the +// then answers the same calls. A found answer whose validator does not fit the // transport size budget arrives as not-found — the scope goes cold, which // is correct, just slower — never as a silent omission.) func LookupMany(ctx context.Context, lookup Lookup, queries []Query) ([]Answer, error) { diff --git a/pkg/sourcecache/grpc_server.go b/pkg/sourcecache/grpc_server.go index e6d5d0206..ea4356004 100644 --- a/pkg/sourcecache/grpc_server.go +++ b/pkg/sourcecache/grpc_server.go @@ -27,7 +27,7 @@ type GRPCServer struct { } var _ v1.BatonSourceCacheServiceServer = (*GRPCServer)(nil) -var _ SetLookup = (*GRPCServer)(nil) +var _ SourceCacheSetter = (*GRPCServer)(nil) // NewGRPCServer returns a GRPCServer with no active Lookup registered. func NewGRPCServer() *GRPCServer { diff --git a/pkg/sourcecache/sourcecache.go b/pkg/sourcecache/sourcecache.go index bc378eaf6..514a6668f 100644 --- a/pkg/sourcecache/sourcecache.go +++ b/pkg/sourcecache/sourcecache.go @@ -10,7 +10,7 @@ // with SourceCacheReplay. // // The connector owns scope computation; the SDK only keys storage by the -// connector-supplied scope key. The validator (etag, delta token) is opaque +// connector-supplied scope key. The validator (HTTP ETag, delta token, ...) is opaque // to the SDK. // // Invariant that keeps replay safe: a connector must only emit @@ -41,6 +41,7 @@ import ( "crypto/sha256" "encoding/hex" "fmt" + "strings" "time" ) @@ -79,9 +80,11 @@ func ValidateRowKind(rowKind RowKind) error { // real providers. const maxScopeKeyLen = 256 -// ValidateScopeKey returns an error when scopeKey is empty or -// unreasonably long. Connectors conventionally use HashScope, but any -// stable identifier is accepted. +// ValidateScopeKey returns an error when scopeKey is empty, unreasonably +// long, or contains a NUL byte. Connectors conventionally use HashScope, +// but any stable identifier is accepted. NUL is reserved: composite +// "kind\x00scope" keys appear in inspection surfaces (engine snapshots), +// and a scope key containing NUL could alias another entry there. func ValidateScopeKey(scopeKey string) error { if scopeKey == "" { return fmt.Errorf("source cache scope key is required") @@ -89,6 +92,9 @@ func ValidateScopeKey(scopeKey string) error { if len(scopeKey) > maxScopeKeyLen { return fmt.Errorf("source cache scope key too long: %d bytes (max %d)", len(scopeKey), maxScopeKeyLen) } + if strings.ContainsRune(scopeKey, 0) { + return fmt.Errorf("source cache scope key must not contain NUL bytes") + } return nil } @@ -123,11 +129,11 @@ func (NoopLookup) Lookup(context.Context, RowKind, string) (Entry, bool, error) return Entry{}, false, nil } -// SetLookup is implemented by connector clients/servers that can receive a +// SourceCacheSetter is implemented by connector clients/servers that can receive a // source-cache lookup implementation from the sync runner. The SDK calls // SetSourceCache(lookup) at the start of each sync and SetSourceCache(nil) // when the sync ends so a late RPC can't read stale state. -type SetLookup interface { +type SourceCacheSetter interface { SetSourceCache(ctx context.Context, lookup Lookup) } diff --git a/pkg/sync/ingest_invariants.go b/pkg/sync/ingest_invariants.go index 2d1688a65..d53860086 100644 --- a/pkg/sync/ingest_invariants.go +++ b/pkg/sync/ingest_invariants.go @@ -29,7 +29,10 @@ package sync //nolint:revive,nolintlint // we can't change the package name for // ChildResourceType must have had its (childType, parent) action // scheduled. Check-only; skipped when the resources phase did not // run in this process (a zero-row child listing leaves no store -// evidence that its action ran). Full syncs only. +// evidence that its action ran). Full syncs only. STRICT MODE ONLY: +// the check costs a full post-collection resource scan and its +// lenient outcome was only ever a warning, so production (lenient) +// skips it entirely — tests and the equivalence harness enforce it. // - I5 exclusion-group validation: the stored entitlement keyspace is // validated post-collection (one default per group, one resource // type per group, size cap). This REPLACES the former streaming @@ -93,13 +96,25 @@ func childScheduleKey(childTypeID, parentTypeID, parentID string) string { return childTypeID + "\x00" + parentTypeID + "\x00" + parentID } -func (c *childScheduleSet) record(childTypeID, parentTypeID, parentID string) { +// recordIfNew records the pair and reports whether it was NEW this sync. +// The set doubles as the scheduling dedupe: a parent can be discovered by +// several ingestion paths in one sync (replay copies its stored row AND a +// delta overlay re-emits it changed), and its children only need one +// SyncResourcesOp per pair — the second discovery would double the +// child-listing work. Atomic check-and-set so parallel workers can't both +// see "new". +func (c *childScheduleSet) recordIfNew(childTypeID, parentTypeID, parentID string) bool { c.mu.Lock() defer c.mu.Unlock() if c.m == nil { c.m = make(map[string]struct{}) } - c.m[childScheduleKey(childTypeID, parentTypeID, parentID)] = struct{}{} + key := childScheduleKey(childTypeID, parentTypeID, parentID) + if _, ok := c.m[key]; ok { + return false + } + c.m[key] = struct{}{} + return true } func (c *childScheduleSet) has(childTypeID, parentTypeID, parentID string) bool { @@ -185,6 +200,14 @@ func (s *syncer) checkGrantResourceReferences(ctx context.Context) error { // cannot be derived after the fact (an executed child action that // returned zero rows leaves no store evidence). func (s *syncer) checkChildScheduling(ctx context.Context) error { + if !s.strictIngestionInvariants { + // Strict mode only: the check costs a full post-collection + // resource scan (value decode + annotation walk per row) and in + // lenient mode a violation was only ever a warning — a bad + // trade at whale scale. Tests and the harness run strict and + // enforce it there. + return nil + } if s.syncType != connectorstore.SyncTypeFull || !s.resourcesPhaseRanHere { // Resumed past the resources phase: the scheduled set from the // prior process is gone; the predicate is unverifiable. @@ -236,13 +259,9 @@ func (s *syncer) checkChildScheduling(ctx context.Context) error { // Sorted so the error text is byte-stable regardless of store // iteration order (the verdict already is). sort.Strings(violations) - err := fmt.Errorf( + return fmt.Errorf( "ingest invariant I4 violated: %d child resource sync(s) were never scheduled: %v", len(violations), violations) - if s.strictIngestionInvariants { - return err - } - ctxzap.Extract(ctx).Warn("ingest invariant I4 violated (lenient mode)", zap.Error(err)) } return nil } @@ -344,11 +363,11 @@ func (s *syncer) validateStoredExclusionGroups(ctx context.Context) error { // lost manifest write or a stamp leak — and the damage is deferred: THIS // sync reads clean, the NEXT sync replays from the damaged state. func (s *syncer) checkSourceCacheScopeConsistency(ctx context.Context) error { - facts, ok := s.store.(dotc1z.IngestFactStore) + sc, ok := s.store.(dotc1z.SourceCacheStore) if !ok { return nil // engine without source-cache state } - orphans, err := facts.SourceCacheOrphanScopes(ctx) + orphans, err := sc.SourceCacheOrphanScopes(ctx) if err != nil { return fmt.Errorf("ingest invariant I6: scanning scope indexes: %w", err) } diff --git a/pkg/sync/ingest_invariants_test.go b/pkg/sync/ingest_invariants_test.go index a817bce22..b57b5acf0 100644 --- a/pkg/sync/ingest_invariants_test.go +++ b/pkg/sync/ingest_invariants_test.go @@ -1,6 +1,7 @@ package sync //nolint:revive,nolintlint // we can't change the package name for backwards compatibility import ( + "context" "testing" "github.com/stretchr/testify/require" @@ -9,6 +10,41 @@ import ( v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" ) +// TestPushChildResourceActionsDedupesPerSync pins the per-sync child +// scheduling dedupe: a parent discovered by several ingestion paths in +// one sync (replay copy + delta overlay re-emission) must schedule each +// (childType, parent) pair exactly once — the duplicate would double the +// child-listing connector work on every warm overlay. The dedupe rides +// the I4 evidence set (childScheduleSet.recordIfNew), so the evidence +// stays complete for every pair regardless of which discovery scheduled. +func TestPushChildResourceActionsDedupesPerSync(t *testing.T) { + ctx := context.Background() + s := &syncer{state: newState()} + + countActions := func() int { + st, ok := s.state.(*state) + require.True(t, ok) + return len(st.actions) + } + + s.pushChildResourceActions(ctx, []string{"project", "repo"}, "org", "org0") + require.Equal(t, 2, countActions()) + + // Second discovery of the same parent (e.g. the overlay re-emission + // after a replay copy already scheduled it): no new actions. + s.pushChildResourceActions(ctx, []string{"project", "repo"}, "org", "org0") + require.Equal(t, 2, countActions()) + + // A different parent still schedules. + s.pushChildResourceActions(ctx, []string{"project"}, "org", "org1") + require.Equal(t, 3, countActions()) + + // The I4 evidence is intact for every recorded pair. + require.True(t, s.childSchedule.has("project", "org", "org0")) + require.True(t, s.childSchedule.has("repo", "org", "org0")) + require.True(t, s.childSchedule.has("project", "org", "org1")) +} + // TestIngestInvariantAnnotationCoverage is the completeness meta-test for // the ingestion-invariant registry: every connector annotation known to // imply a syncer side effect must map to the mechanism that guarantees it diff --git a/pkg/sync/source_cache.go b/pkg/sync/source_cache.go index aa8512f69..41eba8a9d 100644 --- a/pkg/sync/source_cache.go +++ b/pkg/sync/source_cache.go @@ -57,16 +57,21 @@ type syncerSourceCache struct { // page's manifest write is not also counted as a fresh stamp — one // warm delta round would otherwise book as both a hit and a miss in // the replay ratio. Only rounds that can span pages are recorded - // (overlay, or replay without an inline etag), keeping the set tiny: + // (overlay, or replay without an inline validator), keeping the set tiny: // single-page 304 rounds never enter it. Pointer so copying the // per-sync struct shares the set; guarded internally for parallel // workers. Best-effort across resume (in-memory), stats-only. replayedScopes *replayedScopeSet } +type scopeRef struct { + kind sourcecache.RowKind + scopeKey string +} + type replayedScopeSet struct { mu stdsync.Mutex - m map[string]struct{} + m map[scopeRef]struct{} } func (c *syncerSourceCache) markScopeReplayed(kind sourcecache.RowKind, scopeKey string) { @@ -76,9 +81,9 @@ func (c *syncerSourceCache) markScopeReplayed(kind sourcecache.RowKind, scopeKey c.replayedScopes.mu.Lock() defer c.replayedScopes.mu.Unlock() if c.replayedScopes.m == nil { - c.replayedScopes.m = make(map[string]struct{}) + c.replayedScopes.m = make(map[scopeRef]struct{}) } - c.replayedScopes.m[string(kind)+"\x00"+scopeKey] = struct{}{} + c.replayedScopes.m[scopeRef{kind: kind, scopeKey: scopeKey}] = struct{}{} } func (c *syncerSourceCache) scopeReplayedThisSync(kind sourcecache.RowKind, scopeKey string) bool { @@ -87,7 +92,7 @@ func (c *syncerSourceCache) scopeReplayedThisSync(kind sourcecache.RowKind, scop } c.replayedScopes.mu.Lock() defer c.replayedScopes.mu.Unlock() - _, ok := c.replayedScopes.m[string(kind)+"\x00"+scopeKey] + _, ok := c.replayedScopes.m[scopeRef{kind: kind, scopeKey: scopeKey}] return ok } @@ -120,7 +125,7 @@ func (s *syncer) configureSourceCache(ctx context.Context, resp *v2.ConnectorSer l := ctxzap.Extract(ctx) s.sourceCache = syncerSourceCache{} - setLookup, canSetLookup := s.connector.(sourcecache.SetLookup) + setLookup, canSetLookup := s.connector.(sourcecache.SourceCacheSetter) degrade := func(reason string) error { if canSetLookup { setLookup.SetSourceCache(ctx, sourcecache.NoopLookup{}) @@ -235,7 +240,7 @@ func previousSyncReplayUnusableReason(ctx context.Context, reader connectorstore // clearSourceCacheLookup detaches the per-sync lookup so a late RPC from // the connector cannot read a store the syncer no longer owns. func (s *syncer) clearSourceCacheLookup(ctx context.Context) { - if setLookup, ok := s.connector.(sourcecache.SetLookup); ok { + if setLookup, ok := s.connector.(sourcecache.SourceCacheSetter); ok { setLookup.SetSourceCache(ctx, nil) } } @@ -244,9 +249,9 @@ func (s *syncer) clearSourceCacheLookup(ctx context.Context) { // from beginSourceCachePage (before rows are written) to // finishSourceCachePage (after rows are written). type sourceCachePage struct { - kind sourcecache.RowKind - scopeKey string - etag string + kind sourcecache.RowKind + scopeKey string + cacheValidator string // replayed reports that beginSourceCachePage copied the previous // sync's rows for this scope into the current sync BEFORE the page's // own rows commit. Consumers that dedupe against "already synced this @@ -325,9 +330,9 @@ func (s *syncer) beginSourceCachePage( } // Prefer the scope annotation's etag (the freshest validator on // overlay pages); fall back to the replay's. - page.etag = scope.GetCacheValidator() - if page.etag == "" { - page.etag = replay.GetCacheValidator() + page.cacheValidator = scope.GetCacheValidator() + if page.cacheValidator == "" { + page.cacheValidator = replay.GetCacheValidator() } // Tombstones may ride either annotation — the replay annotation on a // round's first page, the scope annotation on every page (so a @@ -338,115 +343,126 @@ func (s *syncer) beginSourceCachePage( page.deletedPrincipalIDs = append(append([]string{}, replay.GetDeletedPrincipalIds()...), scope.GetDeletedPrincipalIds()...) if hasReplay { - if s.sourceCache.prev == nil { - // Same invariant violation as the disabled case: the connector - // can only have gotten a lookup hit if a previous source exists. - return ctx, nil, fmt.Errorf("source cache: connector requested replay for scope %q but no previous sync is available", replay.GetScopeKey()) - } - page.replayed = true - if !replay.GetOverlay() && rowCount > 0 { - // The contract says a 304-style replay page is empty, but rows - // arriving here are more data, not less — upsert them on top of - // the replayed base (overlay semantics) rather than failing the - // sync. Kept lenient while the model is proven against real - // providers. - ctxzap.Extract(ctx).Warn("source cache: non-overlay replay returned rows; treating them as an overlay", - zap.String("scope_key", page.scopeKey), - zap.Int("rows", rowCount), - ) - } - // Advisory check: a well-behaved connector only replays a scope - // whose validator came from this sync's lookup, so a missing - // previous manifest entry is suspicious — but not by itself data - // loss (the stamped rows may still exist, e.g. a partially carried - // file). The hard error below is reserved for a replay that - // produces nothing. - _, entryFound, err := s.sourceCache.prev.LookupSourceCacheEntry(ctx, kind, page.scopeKey) - if err != nil { - return ctx, nil, fmt.Errorf("source cache: error reading previous manifest for scope %q: %w", page.scopeKey, err) - } - if !entryFound { - ctxzap.Extract(ctx).Warn("source cache: replay requested for scope with no previous manifest entry", - zap.String("scope_key", page.scopeKey)) - } - replayStart := time.Now() - res, err := s.sourceCache.current.ReplaySourceCache(ctx, s.sourceCache.prevReader, kind, page.scopeKey) - s.state.AddStepDuration("source_cache_replay", time.Since(replayStart)) - if err != nil { - return ctx, nil, fmt.Errorf("source cache: replay for scope %q failed: %w", page.scopeKey, err) + if err := s.executeScopeReplay(ctx, kind, page, replay, rowCount); err != nil { + return ctx, nil, err } - // Stats are stashed on the page and recorded in - // finishSourceCachePage so a page re-run after failing between - // replay and finish doesn't double-count. - page.replayRows = res.Rows - page.overlayRows = rowCount - if replay.GetOverlay() || page.etag == "" { - // This replay round can span pages: the validator arrives on - // a LATER page's scope annotation (delta rounds), and that - // page must count toward ScopesReplayed's scope, not as a - // freshly stamped one — see finishSourceCachePage. - s.sourceCache.markScopeReplayed(kind, page.scopeKey) - } - if res.Rows == 0 && !entryFound { - // The connector skipped row generation expecting a base that - // does not exist anywhere in the previous file — this sync - // would silently drop the scope's rows. - return ctx, nil, fmt.Errorf("source cache: replay for scope %q found no previous rows and no manifest entry; the connector replayed a scope it never looked up", page.scopeKey) - } - // Empty-vs-dropped discrimination via the previous file's scope - // index: a legitimately empty scope has NO index entries - // (StaleSkipped == 0, fine to proceed); a scope whose index says - // rows existed but none carried a matching value stamp means the - // rows were rewritten without index cleanup — proceeding would - // silently drop the scope's contents from this sync. - if res.Rows == 0 && res.StaleSkipped > 0 { - return ctx, nil, fmt.Errorf( - "source cache: replay for scope %q copied no rows but the previous file's scope index holds %d entries "+ - "whose rows no longer carry this scope's stamp; refusing to silently drop the scope", - page.scopeKey, res.StaleSkipped) - } - if res.StaleSkipped > 0 { - // Partial staleness: some rows copied, some index entries were - // dead. Not data loss by itself (the live rows carried their - // true stamp), but a signal a writer skipped index cleanup. - ctxzap.Extract(ctx).Warn("source cache: replay skipped stale scope-index entries", - zap.String("scope_key", page.scopeKey), - zap.Int64("rows", res.Rows), - zap.Int64("stale_skipped", res.StaleSkipped), - ) - } - // Side effects for replayed rows are STORE-DERIVED, not armed - // here — the engine maintains the needs_expansion index and the - // external-match existence bit for replayed rows exactly as for - // fresh ones, and the ingestion invariants read the store at - // their consuming seams (expansion: the PendingExpansion probe at - // SyncGrantExpansionOp; external match: the fact probe at - // SyncExternalResourcesOp; exclusion groups: the post-collection - // keyspace validation). See - // docs/tasks/source-cache-ingestion-invariants.md. The one - // replay-carried side effect is child scheduling, which cannot be - // derived after the fact (a zero-row child listing leaves no - // store evidence), so replay carries the parents' child types: - if kind == sourcecache.RowKindResources { - for _, parent := range res.ChildResources { - s.pushChildResourceActions(ctx, parent.ChildTypeIDs, parent.ResourceTypeID, parent.ResourceID) - } - } - ctxzap.Extract(ctx).Debug("source cache replayed scope", - zap.String("row_kind", string(kind)), + } + + return sourcecache.WithScope(ctx, page.scopeKey), page, nil +} + +// executeScopeReplay is beginSourceCachePage's replay step: validate the +// replay request against this sync's state, copy the previous sync's +// rows for the scope, and record the page-level bookkeeping. The page's +// scope key, validator, and tombstones have already been parsed. +func (s *syncer) executeScopeReplay(ctx context.Context, kind sourcecache.RowKind, page *sourceCachePage, replay *v2.SourceCacheReplay, rowCount int) error { + if s.sourceCache.prev == nil { + // Same invariant violation as the disabled case: the connector + // can only have gotten a lookup hit if a previous source exists. + return fmt.Errorf("source cache: connector requested replay for scope %q but no previous sync is available", replay.GetScopeKey()) + } + page.replayed = true + if !replay.GetOverlay() && rowCount > 0 { + // The contract says a 304-style replay page is empty, but rows + // arriving here are more data, not less — upsert them on top of + // the replayed base (overlay semantics) rather than failing the + // sync. Kept lenient while the model is proven against real + // providers. + ctxzap.Extract(ctx).Warn("source cache: non-overlay replay returned rows; treating them as an overlay", + zap.String("scope_key", page.scopeKey), + zap.Int("rows", rowCount), + ) + } + // Advisory check: a well-behaved connector only replays a scope + // whose validator came from this sync's lookup, so a missing + // previous manifest entry is suspicious — but not by itself data + // loss (the stamped rows may still exist, e.g. a partially carried + // file). The hard error below is reserved for a replay that + // produces nothing. + _, entryFound, err := s.sourceCache.prev.LookupSourceCacheEntry(ctx, kind, page.scopeKey) + if err != nil { + return fmt.Errorf("source cache: error reading previous manifest for scope %q: %w", page.scopeKey, err) + } + if !entryFound { + ctxzap.Extract(ctx).Warn("source cache: replay requested for scope with no previous manifest entry", + zap.String("scope_key", page.scopeKey)) + } + replayStart := time.Now() + res, err := s.sourceCache.current.ReplaySourceCache(ctx, s.sourceCache.prevReader, kind, page.scopeKey) + s.state.AddStepDuration("source_cache_replay", time.Since(replayStart)) + if err != nil { + return fmt.Errorf("source cache: replay for scope %q failed: %w", page.scopeKey, err) + } + // Stats are stashed on the page and recorded in + // finishSourceCachePage so a page re-run after failing between + // replay and finish doesn't double-count. + page.replayRows = res.Rows + page.overlayRows = rowCount + if replay.GetOverlay() || page.cacheValidator == "" { + // This replay round can span pages: the validator arrives on + // a LATER page's scope annotation (delta rounds), and that + // page must count toward ScopesReplayed's scope, not as a + // freshly stamped one — see finishSourceCachePage. + s.sourceCache.markScopeReplayed(kind, page.scopeKey) + } + if res.Rows == 0 && !entryFound { + // The connector skipped row generation expecting a base that + // does not exist anywhere in the previous file — this sync + // would silently drop the scope's rows. + return fmt.Errorf("source cache: replay for scope %q found no previous rows and no manifest entry; the connector replayed a scope it never looked up", page.scopeKey) + } + // Empty-vs-dropped discrimination via the previous file's scope + // index: a legitimately empty scope has NO index entries + // (StaleSkipped == 0, fine to proceed); a scope whose index says + // rows existed but none carried a matching value stamp means the + // rows were rewritten without index cleanup — proceeding would + // silently drop the scope's contents from this sync. + if res.Rows == 0 && res.StaleSkipped > 0 { + return fmt.Errorf( + "source cache: replay for scope %q copied no rows but the previous file's scope index holds %d entries "+ + "whose rows no longer carry this scope's stamp; refusing to silently drop the scope", + page.scopeKey, res.StaleSkipped) + } + if res.StaleSkipped > 0 { + // Partial staleness: some rows copied, some index entries were + // dead. Not data loss by itself (the live rows carried their + // true stamp), but a signal a writer skipped index cleanup. + ctxzap.Extract(ctx).Warn("source cache: replay skipped stale scope-index entries", zap.String("scope_key", page.scopeKey), zap.Int64("rows", res.Rows), - zap.Int("deleted_ids", len(page.deletedIDs)), - zap.Int("deleted_principal_ids", len(page.deletedPrincipalIDs)), + zap.Int64("stale_skipped", res.StaleSkipped), ) - if s.testSourceCacheHaltHook != nil { - if err := s.testSourceCacheHaltHook("replay-copied", page.scopeKey); err != nil { - return ctx, nil, err - } + } + // Side effects for replayed rows are STORE-DERIVED, not armed + // here — the engine maintains the needs_expansion index and the + // external-match existence bit for replayed rows exactly as for + // fresh ones, and the ingestion invariants read the store at + // their consuming seams (expansion: the PendingExpansion probe at + // SyncGrantExpansionOp; external match: the fact probe at + // SyncExternalResourcesOp; exclusion groups: the post-collection + // keyspace validation). See + // docs/tasks/source-cache-ingestion-invariants.md. The one + // replay-carried side effect is child scheduling, which cannot be + // derived after the fact (a zero-row child listing leaves no + // store evidence), so replay carries the parents' child types: + if kind == sourcecache.RowKindResources { + for _, parent := range res.ChildResources { + s.pushChildResourceActions(ctx, parent.ChildTypeIDs, parent.ResourceTypeID, parent.ResourceID) } } - - return sourcecache.WithScope(ctx, page.scopeKey), page, nil + ctxzap.Extract(ctx).Debug("source cache replayed scope", + zap.String("row_kind", string(kind)), + zap.String("scope_key", page.scopeKey), + zap.Int64("rows", res.Rows), + zap.Int("deleted_ids", len(page.deletedIDs)), + zap.Int("deleted_principal_ids", len(page.deletedPrincipalIDs)), + ) + if s.testSourceCacheHaltHook != nil { + if err := s.testSourceCacheHaltHook("replay-copied", page.scopeKey); err != nil { + return err + } + } + return nil } // finishSourceCachePage runs after the page's rows committed: applies @@ -468,37 +484,16 @@ func (s *syncer) finishSourceCachePage(ctx context.Context, page *sourceCachePag defer func() { s.state.AddStepDuration("source_cache_tombstones", time.Since(tombstoneStart)) }() - } - if len(page.deletedIDs) > 0 { - if page.kind == sourcecache.RowKindGrants { - // Grant-id tombstones resolve within the scope's own rows so - // connector-custom grant-id shapes (unreachable by the global - // bounded delete) work, and the cost stays bounded by the - // scope's size. - deleted, err := s.sourceCache.current.DeleteSourceCacheGrantsByIDInScope(ctx, page.scopeKey, page.deletedIDs) - if err != nil { - return fmt.Errorf("source cache: error applying grant deletions for scope %q: %w", page.scopeKey, err) - } - rowsDeleted += deleted - ctxzap.Extract(ctx).Debug("source cache applied grant-id deletions", - zap.String("scope_key", page.scopeKey), - zap.Int("tombstones", len(page.deletedIDs)), - zap.Int64("rows_deleted", deleted), - ) - } else if err := s.sourceCache.current.DeleteSourceCacheRows(ctx, page.kind, page.deletedIDs); err != nil { - return fmt.Errorf("source cache: error applying deletions for scope %q: %w", page.scopeKey, err) - } - } - if len(page.deletedPrincipalIDs) > 0 { - deleted, err := s.sourceCache.current.DeleteSourceCacheRowsInScope(ctx, page.kind, page.scopeKey, page.deletedPrincipalIDs) + deleted, err := s.sourceCache.current.ApplyTombstones(ctx, page.kind, page.scopeKey, page.deletedIDs, page.deletedPrincipalIDs) if err != nil { - return fmt.Errorf("source cache: error applying scoped deletions for scope %q: %w", page.scopeKey, err) + return fmt.Errorf("source cache: error applying tombstones for scope %q: %w", page.scopeKey, err) } - rowsDeleted += deleted - ctxzap.Extract(ctx).Debug("source cache applied scoped deletions", + rowsDeleted = deleted + ctxzap.Extract(ctx).Debug("source cache applied tombstones", zap.String("row_kind", string(page.kind)), zap.String("scope_key", page.scopeKey), - zap.Int("tombstones", len(page.deletedPrincipalIDs)), + zap.Int("tombstone_ids", len(page.deletedIDs)), + zap.Int("tombstone_principals", len(page.deletedPrincipalIDs)), zap.Int64("rows_deleted", deleted), ) } @@ -507,8 +502,8 @@ func (s *syncer) finishSourceCachePage(ctx context.Context, page *sourceCachePag return err } } - if page.etag != "" { - if err := s.sourceCache.current.PutSourceCacheEntry(ctx, page.kind, page.scopeKey, page.etag); err != nil { + if page.cacheValidator != "" { + if err := s.sourceCache.current.PutSourceCacheEntry(ctx, page.kind, page.scopeKey, page.cacheValidator); err != nil { return fmt.Errorf("source cache: error writing manifest entry for scope %q: %w", page.scopeKey, err) } if s.testSourceCacheHaltHook != nil { @@ -528,10 +523,10 @@ func (s *syncer) finishSourceCachePage(ctx context.Context, page *sourceCachePag // re-run after failing between replay and finish doesn't // double-count the scope. stats.ScopesReplayed = 1 - stats.RowsReplayed = map[string]int64{string(page.kind): page.replayRows} + stats.RowsReplayed = map[sourcecache.RowKind]int64{page.kind: page.replayRows} stats.OverlayRows = int64(page.overlayRows) } - if page.etag != "" && !page.replayed && !s.sourceCache.scopeReplayedThisSync(page.kind, page.scopeKey) { + if page.cacheValidator != "" && !page.replayed && !s.sourceCache.scopeReplayedThisSync(page.kind, page.scopeKey) { // A cold/fresh page that persisted a validator: the denominator // of the replay hit ratio. The replayed-scope check keeps a // multi-page overlay round — whose new validator legally arrives diff --git a/pkg/sync/source_cache_bench_test.go b/pkg/sync/source_cache_bench_test.go new file mode 100644 index 000000000..b53e99ca5 --- /dev/null +++ b/pkg/sync/source_cache_bench_test.go @@ -0,0 +1,134 @@ +package sync //nolint:revive,nolintlint // we can't change the package name for backwards compatibility + +// Source-cache path benchmarks: the write-side cost of scope stamping +// (inline by_source_scope index + manifest writes + the seal-time I6 +// scan) and the replay path (engine copy + the replay-time count check), +// against an unstamped cold sync of the same rows as the baseline. +// +// go test -bench=BenchmarkSourceCacheSync -benchtime=1x -count=3 \ +// -run='^$' ./pkg/sync/ +// +// Shape: nGroups groups × membersPerGroup member grants, one grants +// scope per group (the per-resource conditional-request shape), users +// listed unstamped — a realistic mixed sync. The warm variant replays +// every scope (all 304s): the maximum-replay / maximum-count-check case. + +import ( + "context" + "fmt" + "os" + "path/filepath" + "testing" + "time" + + "github.com/conductorone/baton-sdk/pkg/dotc1z" + "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" + "github.com/conductorone/baton-sdk/pkg/logging" +) + +const ( + benchGroups = 500 + benchUsers = 5_000 + benchMembersPerGroup = 50 // 500 × 50 = 25k grants, 500 scopes +) + +func buildSourceCacheBenchConnector(b *testing.B, ctx context.Context, stamped bool) *sourceCacheMockConnector { + b.Helper() + mc := newSourceCacheMockConnector() + users := make([]string, benchUsers) + for i := range benchUsers { + u, err := mc.AddUser(ctx, fmt.Sprintf("u-%05d", i)) + if err != nil { + b.Fatalf("AddUser: %v", err) + } + users[i] = u.GetId().GetResource() + _ = users[i] + } + userRes := mc.resourceDB[userResourceType.GetId()] + for gi := range benchGroups { + g, _, err := mc.AddGroup(ctx, fmt.Sprintf("g-%04d", gi)) + if err != nil { + b.Fatalf("AddGroup: %v", err) + } + for m := range benchMembersPerGroup { + mc.AddGroupMember(ctx, g, userRes[(gi*benchMembersPerGroup+m)%len(userRes)]) + } + if stamped { + mc.etagByResource[g.GetId().GetResource()] = "v1" + } + } + return mc +} + +func benchOneSync(b *testing.B, ctx context.Context, cc *sourceCacheMockConnector, path, prevPath, tmpDir string) { + b.Helper() + store, err := dotc1z.NewStore(ctx, path, + dotc1z.WithEngine(c1zstore.EnginePebble), + dotc1z.WithTmpDir(tmpDir), + ) + if err != nil { + b.Fatalf("NewStore: %v", err) + } + opts := []SyncOpt{WithConnectorStore(store), WithTmpDir(tmpDir)} + if prevPath != "" { + opts = append(opts, WithPreviousSyncC1ZPath(prevPath)) + } + syncer, nerr := NewSyncer(ctx, cc, opts...) + if nerr != nil { + b.Fatalf("NewSyncer: %v", nerr) + } + if err := syncer.Sync(ctx); err != nil { + b.Fatalf("Sync: %v", err) + } + if err := syncer.Close(ctx); err != nil { + b.Fatalf("Close: %v", err) + } +} + +func BenchmarkSourceCacheSync(b *testing.B) { + ctx, err := logging.Init(context.Background(), logging.WithLogLevel("error")) + if err != nil { + b.Fatalf("logging.Init: %v", err) + } + + b.Run("cold_unstamped", func(b *testing.B) { + mc := buildSourceCacheBenchConnector(b, ctx, false) + tmpDir := b.TempDir() + b.ResetTimer() + for i := 0; i < b.N; i++ { + start := time.Now() + benchOneSync(b, ctx, mc, filepath.Join(tmpDir, fmt.Sprintf("cold-un-%d.c1z", i)), "", tmpDir) + b.ReportMetric(float64(time.Since(start).Milliseconds()), "ms/sync") + } + }) + + b.Run("cold_stamped", func(b *testing.B) { + mc := buildSourceCacheBenchConnector(b, ctx, true) + tmpDir := b.TempDir() + b.ResetTimer() + for i := 0; i < b.N; i++ { + start := time.Now() + path := filepath.Join(tmpDir, fmt.Sprintf("cold-st-%d.c1z", i)) + benchOneSync(b, ctx, mc, path, "", tmpDir) + b.ReportMetric(float64(time.Since(start).Milliseconds()), "ms/sync") + if i == 0 { + if fi, err := os.Stat(path); err == nil { + b.ReportMetric(float64(fi.Size())/(1<<20), "MB/c1z") + } + } + } + }) + + b.Run("warm_replay", func(b *testing.B) { + mc := buildSourceCacheBenchConnector(b, ctx, true) + tmpDir := b.TempDir() + prevPath := filepath.Join(tmpDir, "prev.c1z") + benchOneSync(b, ctx, mc, prevPath, "", tmpDir) // seed: fresh fetch, stamps + manifest + b.ResetTimer() + for i := 0; i < b.N; i++ { + start := time.Now() + benchOneSync(b, ctx, mc, filepath.Join(tmpDir, fmt.Sprintf("warm-%d.c1z", i)), prevPath, tmpDir) + b.ReportMetric(float64(time.Since(start).Milliseconds()), "ms/sync") + } + }) +} diff --git a/pkg/sync/source_cache_continuation.go b/pkg/sync/source_cache_continuation.go index a99516c7a..0dc8b5967 100644 --- a/pkg/sync/source_cache_continuation.go +++ b/pkg/sync/source_cache_continuation.go @@ -44,11 +44,11 @@ const ( // unbounded fan-out bloats every checkpoint). Matches the proto // max_items validate rule on EnqueuePageTokens.page_tokens. maxEnqueuePageTokensPerResponse = 1024 - // maxSpawnCursorTokenBytes matches the page-token request field and + // maxEnqueuedPageTokenBytes matches the page-token request field and // the EnqueuePageTokens item validation. Enforced before creating Actions: // action tokens are checkpointed before they are sent through the // later request field's validator. - maxSpawnCursorTokenBytes = 1 << 20 + maxEnqueuedPageTokenBytes = 1 << 20 // sourceCacheAnswerBudget caps the FOUND-etag payload attached to one // re-invoke, keeping the request under single-shot transport payload diff --git a/pkg/sync/source_cache_continuation_test.go b/pkg/sync/source_cache_continuation_test.go index 66caddc3e..89018a885 100644 --- a/pkg/sync/source_cache_continuation_test.go +++ b/pkg/sync/source_cache_continuation_test.go @@ -2,7 +2,7 @@ package sync //nolint:revive,nolintlint // we can't change the package name for // Syncer-side tests for the source-cache lookup continuation (ask/answer). // The mocks here play the LAMBDA side of the protocol by hand: they do NOT -// implement sourcecache.SetLookup (no direct lookup install — exactly the +// implement sourcecache.SourceCacheSetter (no direct lookup install — exactly the // single-shot-transport topology), and instead answer list RPCs with // SourceCacheLookupAsk and consume SourceCacheLookupAnswers from the // re-invoked request. diff --git a/pkg/sync/source_cache_equivalence_test.go b/pkg/sync/source_cache_equivalence_test.go index 2745a7696..919290bd9 100644 --- a/pkg/sync/source_cache_equivalence_test.go +++ b/pkg/sync/source_cache_equivalence_test.go @@ -475,7 +475,7 @@ type equivConnector struct { } // equivContinuationClient hides the embedded SetSourceCache from the -// syncer's sourcecache.SetLookup type assertion, making the connector a +// syncer's sourcecache.SourceCacheSetter type assertion, making the connector a // single-shot transport: the syncer attaches SourceCacheLookupOffer to // requests and the connector answers with asks. type equivContinuationClient struct { @@ -483,7 +483,7 @@ type equivContinuationClient struct { } // SetSourceCache shadows the embedded method with a different signature -// so equivContinuationClient does NOT satisfy sourcecache.SetLookup. +// so equivContinuationClient does NOT satisfy sourcecache.SourceCacheSetter. func (equivContinuationClient) SetSourceCache() {} func (c *equivConnector) injectGrantsFailure(resourceID string) { diff --git a/pkg/sync/source_cache_stats.go b/pkg/sync/source_cache_stats.go index bbd00d585..b360de043 100644 --- a/pkg/sync/source_cache_stats.go +++ b/pkg/sync/source_cache_stats.go @@ -10,6 +10,8 @@ import ( "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" "go.uber.org/zap" + + "github.com/conductorone/baton-sdk/pkg/sourcecache" ) // SourceCacheStats accumulates per-sync source-cache counters: replay @@ -24,10 +26,18 @@ type SourceCacheStats struct { // ScopesStamped counts manifest entries written for NON-replayed // (cold/fresh) pages. Interim pages with no validator count in // neither bucket. + // + // APPROXIMATE UNDER RESUME: the suppression that keeps a multi-page + // overlay round's final (validator-carrying) page out of this count + // rides an in-memory set (syncerSourceCache.replayedScopes); a + // resume landing mid-round books that round as both a hit and a + // miss. Rows are unaffected — but do NOT gate behavior on the + // replay ratio without first persisting that set (or deriving + // suppression from durable manifest state). ScopesStamped int64 `json:"scopes_stamped,omitempty"` // RowsReplayed counts rows bulk-copied from the previous sync, keyed // by row kind (resources / entitlements / grants). - RowsReplayed map[string]int64 `json:"rows_replayed,omitempty"` + RowsReplayed map[sourcecache.RowKind]int64 `json:"rows_replayed,omitempty"` // OverlayRows counts connector-emitted rows upserted on top of a // replayed base (delta adds/changes). OverlayRows int64 `json:"overlay_rows,omitempty"` @@ -40,9 +50,9 @@ type SourceCacheStats struct { TombstonePrincipals int64 `json:"tombstone_principals,omitempty"` RowsDeleted int64 `json:"rows_deleted,omitempty"` - // SpawnedCursors counts sibling actions enqueued via EnqueuePageTokens + // EnqueuedPageTokens counts sibling actions enqueued via EnqueuePageTokens // (pairs with the per-response cap). - SpawnedCursors int64 `json:"spawned_cursors,omitempty"` + EnqueuedPageTokens int64 `json:"enqueued_page_tokens,omitempty"` // Lookup continuation (ask/answer) counters. BouncesByOp splits // bounces by list-op kind, distinguishing planner asks (expected: one @@ -67,7 +77,7 @@ func (s *SourceCacheStats) merge(delta SourceCacheStats) { s.ScopesStamped += delta.ScopesStamped for kind, n := range delta.RowsReplayed { if s.RowsReplayed == nil { - s.RowsReplayed = make(map[string]int64, len(delta.RowsReplayed)) + s.RowsReplayed = make(map[sourcecache.RowKind]int64, len(delta.RowsReplayed)) } s.RowsReplayed[kind] += n } @@ -75,7 +85,7 @@ func (s *SourceCacheStats) merge(delta SourceCacheStats) { s.TombstoneIDs += delta.TombstoneIDs s.TombstonePrincipals += delta.TombstonePrincipals s.RowsDeleted += delta.RowsDeleted - s.SpawnedCursors += delta.SpawnedCursors + s.EnqueuedPageTokens += delta.EnqueuedPageTokens s.LookupBounces += delta.LookupBounces s.LookupRequestsBounced += delta.LookupRequestsBounced s.LookupScopesAsked += delta.LookupScopesAsked @@ -95,7 +105,7 @@ func (s *SourceCacheStats) merge(delta SourceCacheStats) { func (s *SourceCacheStats) copy() *SourceCacheStats { out := *s if s.RowsReplayed != nil { - out.RowsReplayed = make(map[string]int64, len(s.RowsReplayed)) + out.RowsReplayed = make(map[sourcecache.RowKind]int64, len(s.RowsReplayed)) for k, v := range s.RowsReplayed { out.RowsReplayed[k] = v } diff --git a/pkg/sync/source_cache_stats_test.go b/pkg/sync/source_cache_stats_test.go index 0b93a9ee4..610cfe967 100644 --- a/pkg/sync/source_cache_stats_test.go +++ b/pkg/sync/source_cache_stats_test.go @@ -5,6 +5,8 @@ import ( "path/filepath" "testing" + "github.com/conductorone/baton-sdk/pkg/sourcecache" + "github.com/stretchr/testify/require" "github.com/conductorone/baton-sdk/pkg/dotc1z" @@ -22,7 +24,7 @@ func TestSourceCacheStats_TokenRoundTrip(t *testing.T) { st.AddSourceCacheStats(SourceCacheStats{ ScopesReplayed: 3, ScopesStamped: 2, - RowsReplayed: map[string]int64{"grants": 40, "resources": 5}, + RowsReplayed: map[sourcecache.RowKind]int64{"grants": 40, "resources": 5}, OverlayRows: 4, TombstoneIDs: 2, LookupBounces: 6, @@ -33,7 +35,7 @@ func TestSourceCacheStats_TokenRoundTrip(t *testing.T) { // Second merge accumulates. st.AddSourceCacheStats(SourceCacheStats{ ScopesReplayed: 1, - RowsReplayed: map[string]int64{"grants": 10}, + RowsReplayed: map[sourcecache.RowKind]int64{"grants": 10}, LookupBouncesByOp: map[string]int64{"sync-resources": 1}, }) diff --git a/pkg/sync/source_cache_test.go b/pkg/sync/source_cache_test.go index 8b713da01..a58bf3970 100644 --- a/pkg/sync/source_cache_test.go +++ b/pkg/sync/source_cache_test.go @@ -543,7 +543,7 @@ func TestSourceCacheStats_OverlayRoundBooksOneHit(t *testing.T) { require.NotNil(t, stats) require.Equal(t, int64(1), stats.ScopesReplayed, "one warm round, one hit") require.Equal(t, int64(1), stats.ScopesStamped, "only the cold scope counts as stamped") - require.Equal(t, int64(2), stats.RowsReplayed[string(sourcecache.RowKindGrants)]) + require.Equal(t, int64(2), stats.RowsReplayed[sourcecache.RowKindGrants]) require.Equal(t, int64(1), stats.OverlayRows) } diff --git a/pkg/sync/state.go b/pkg/sync/state.go index 544dd069a..c118fc30d 100644 --- a/pkg/sync/state.go +++ b/pkg/sync/state.go @@ -211,13 +211,20 @@ const ( // Action stores the current operation, page token, and optional fields for which resource is being worked with. type Action struct { - ID string `json:"id,omitempty"` - Op ActionOp `json:"operation,omitempty"` - PageToken string `json:"page_token,omitempty"` - ResourceTypeID string `json:"resource_type_id,omitempty"` - ResourceID string `json:"resource_id,omitempty"` - ParentResourceTypeID string `json:"parent_resource_type_id,omitempty"` - ParentResourceID string `json:"parent_resource_id,omitempty"` + ID string `json:"id,omitempty"` + Op ActionOp `json:"operation,omitempty"` + PageToken string `json:"page_token,omitempty"` + ResourceTypeID string `json:"resource_type_id,omitempty"` + // ResourceID == "" on a SyncGrantsOp / SyncEntitlementsOp action is + // the TYPE-SCOPED discriminator: the action enumerates the whole + // resource type (TypeScopedGrants / TypeScopedEntitlements) instead + // of one resource. Deliberately not a separate field — actions are + // JSON-serialized into checkpointed sync tokens, and the planners + // never emit an empty ResourceID for those ops otherwise, so the + // sentinel is unambiguous and keeps old tokens decoding unchanged. + ResourceID string `json:"resource_id,omitempty"` + ParentResourceTypeID string `json:"parent_resource_type_id,omitempty"` + ParentResourceID string `json:"parent_resource_id,omitempty"` // Spawned marks an action enqueued by a connector's EnqueuePageTokens // annotation (a sibling cursor) rather than by the syncer's own diff --git a/pkg/sync/syncer.go b/pkg/sync/syncer.go index 617f2d8dc..f7643a9e0 100644 --- a/pkg/sync/syncer.go +++ b/pkg/sync/syncer.go @@ -1095,9 +1095,15 @@ func (s *syncer) pushChildResourceActions(ctx context.Context, childTypeIDs []st continue } // Monotone evidence for ingestion invariant I4 (see - // ingest_invariants.go): record the pair regardless of which - // ingestion path discovered the parent. - s.childSchedule.record(childTypeID, parentTypeID, parentID) + // ingest_invariants.go), doubling as the per-sync scheduling + // dedupe: a parent discovered by BOTH the replay copy and a + // delta overlay re-emission (or re-emitted within a cold delta + // walk) schedules its children exactly once. In-memory, so a + // resumed process re-schedules — the existing at-least-once + // semantic, just not N-times-per-sync. + if !s.childSchedule.recordIfNew(childTypeID, parentTypeID, parentID) { + continue + } s.state.PushAction(ctx, Action{ Op: SyncResourcesOp, ResourceTypeID: childTypeID, @@ -1604,46 +1610,13 @@ func (s *syncer) SyncEntitlements(ctx context.Context, action *Action) error { // resourceTypeHasTypeScopedEntitlements reports (cached per sync) whether the // resource type carries the TypeScopedEntitlements annotation. func (s *syncer) resourceTypeHasTypeScopedEntitlements(ctx context.Context, resourceTypeID string) (bool, error) { - if v, ok := s.typeScopedEntitlementsForResourceType.Load(resourceTypeID); ok { - return v, nil - } - rt, err := s.store.GetResourceType(ctx, reader_v2.ResourceTypesReaderServiceGetResourceTypeRequest_builder{ - ResourceTypeId: resourceTypeID, - }.Build()) - if err != nil { - return false, err - } - rtAnnos := annotations.Annotations(rt.GetResourceType().GetAnnotations()) - typeScoped := rtAnnos.Contains(&v2.TypeScopedEntitlements{}) - s.typeScopedEntitlementsForResourceType.Store(resourceTypeID, typeScoped) - return typeScoped, nil + return s.resourceTypeCarries(ctx, resourceTypeID, &v2.TypeScopedEntitlements{}, &s.typeScopedEntitlementsForResourceType) } // typeScopedEntitlementsResourceTypes lists every synced resource type annotated // with TypeScopedEntitlements. func (s *syncer) typeScopedEntitlementsResourceTypes(ctx context.Context) ([]string, error) { - var out []string - pageToken := "" - for { - resp, err := s.store.ListResourceTypes(ctx, v2.ResourceTypesServiceListResourceTypesRequest_builder{ - PageToken: pageToken, - }.Build()) - if err != nil { - return nil, err - } - for _, rt := range resp.GetList() { - rtAnnos := annotations.Annotations(rt.GetAnnotations()) - typeScoped := rtAnnos.Contains(&v2.TypeScopedEntitlements{}) - s.typeScopedEntitlementsForResourceType.Store(rt.GetId(), typeScoped) - if typeScoped { - out = append(out, rt.GetId()) - } - } - pageToken = resp.GetNextPageToken() - if pageToken == "" { - return out, nil - } - } + return s.resourceTypesCarrying(ctx, &v2.TypeScopedEntitlements{}, &s.typeScopedEntitlementsForResourceType) } // syncEntitlementsForResource fetches the entitlements for a specific resource from the connector. @@ -1661,16 +1634,7 @@ func (s *syncer) syncEntitlementsForResource(ctx context.Context, action *Action var resource *v2.Resource var reqAnnos annotations.Annotations if typeScoped { - // Wire validation requires a non-empty resource id, so the stub is - // self-referential ({type, type}) and the request carries the - // TypeScopedEntitlements annotation as the routing marker. - resource = v2.Resource_builder{ - Id: v2.ResourceId_builder{ - ResourceType: action.ResourceTypeID, - Resource: action.ResourceTypeID, - }.Build(), - }.Build() - reqAnnos.Update(&v2.TypeScopedEntitlements{}) + resource, reqAnnos = typeScopedRequestStub(action.ResourceTypeID, &v2.TypeScopedEntitlements{}) } else { resourceResponse, err := s.store.GetResource(ctx, reader_v2.ResourcesReaderServiceGetResourceRequest_builder{ ResourceId: resourceID, @@ -1739,45 +1703,11 @@ func (s *syncer) syncEntitlementsForResource(ctx context.Context, action *Action } // EnqueuePageTokens: enqueue sibling cursors (type-scoped shards or - // per-resource parallel pages). Same contracts as syncGrantsForResource. - spawn := &v2.EnqueuePageTokens{} - hasSpawn, err := respAnnos.Pick(spawn) + // per-resource parallel pages). Shared contract: see + // collectEnqueuedPageTokens (type_scoped.go). + spawned, err := s.collectEnqueuedPageTokens(ctx, "sync-entitlements", SyncEntitlementsOp, action, respAnnos) if err != nil { - return fmt.Errorf("sync-entitlements: error parsing spawn-cursors annotation: %w", err) - } - var spawned []Action - if hasSpawn { - if len(spawn.GetPageTokens()) > maxEnqueuePageTokensPerResponse { - return fmt.Errorf( - "sync-entitlements: EnqueuePageTokens carried %d page tokens (max %d per response); chain additional spawns across pages instead", - len(spawn.GetPageTokens()), maxEnqueuePageTokensPerResponse) - } - for _, tok := range spawn.GetPageTokens() { - if tok == "" { - continue - } - if len(tok) > maxSpawnCursorTokenBytes { - return fmt.Errorf( - "sync-entitlements: EnqueuePageTokens page token is %d bytes (max %d)", - len(tok), maxSpawnCursorTokenBytes) - } - spawned = append(spawned, Action{ - Op: SyncEntitlementsOp, - ResourceTypeID: action.ResourceTypeID, - ResourceID: action.ResourceID, // empty on type-scoped actions - PageToken: tok, - Spawned: true, - }) - } - if len(spawned) > 0 { - s.state.AddSourceCacheStats(SourceCacheStats{SpawnedCursors: int64(len(spawned))}) - } - ctxzap.Extract(ctx).Debug("sync-entitlements: spawned sibling entitlement cursors", - zap.String("resource_type_id", action.ResourceTypeID), - zap.String("resource_id", action.ResourceID), - zap.Bool("type_scoped", typeScoped), - zap.Int("cursors", len(spawned)), - zap.Int64("estimated_total", spawn.GetEstimatedTotal())) + return err } return s.nextPageOrFinishAction(ctx, action, resp.GetNextPageToken(), spawned...) @@ -2271,46 +2201,13 @@ func (s *syncer) SyncGrants(ctx context.Context, action *Action) error { // resourceTypeHasTypeScopedGrants reports (cached per sync) whether the // resource type carries the TypeScopedGrants annotation. func (s *syncer) resourceTypeHasTypeScopedGrants(ctx context.Context, resourceTypeID string) (bool, error) { - if v, ok := s.typeScopedGrantsForResourceType.Load(resourceTypeID); ok { - return v, nil - } - rt, err := s.store.GetResourceType(ctx, reader_v2.ResourceTypesReaderServiceGetResourceTypeRequest_builder{ - ResourceTypeId: resourceTypeID, - }.Build()) - if err != nil { - return false, err - } - rtAnnos := annotations.Annotations(rt.GetResourceType().GetAnnotations()) - typeScoped := rtAnnos.Contains(&v2.TypeScopedGrants{}) - s.typeScopedGrantsForResourceType.Store(resourceTypeID, typeScoped) - return typeScoped, nil + return s.resourceTypeCarries(ctx, resourceTypeID, &v2.TypeScopedGrants{}, &s.typeScopedGrantsForResourceType) } // typeScopedGrantsResourceTypes lists every synced resource type annotated // with TypeScopedGrants. func (s *syncer) typeScopedGrantsResourceTypes(ctx context.Context) ([]string, error) { - var out []string - pageToken := "" - for { - resp, err := s.store.ListResourceTypes(ctx, v2.ResourceTypesServiceListResourceTypesRequest_builder{ - PageToken: pageToken, - }.Build()) - if err != nil { - return nil, err - } - for _, rt := range resp.GetList() { - rtAnnos := annotations.Annotations(rt.GetAnnotations()) - typeScoped := rtAnnos.Contains(&v2.TypeScopedGrants{}) - s.typeScopedGrantsForResourceType.Store(rt.GetId(), typeScoped) - if typeScoped { - out = append(out, rt.GetId()) - } - } - pageToken = resp.GetNextPageToken() - if pageToken == "" { - return out, nil - } - } + return s.resourceTypesCarrying(ctx, &v2.TypeScopedGrants{}, &s.typeScopedGrantsForResourceType) } // syncGrantsForResource fetches the grants for a specific resource from the connector. @@ -2328,16 +2225,7 @@ func (s *syncer) syncGrantsForResource(ctx context.Context, action *Action) erro var resource *v2.Resource var reqAnnos annotations.Annotations if typeScoped { - // Wire validation requires a non-empty resource id, so the stub is - // self-referential ({type, type}) and the request carries the - // TypeScopedGrants annotation as the routing marker. - resource = v2.Resource_builder{ - Id: v2.ResourceId_builder{ - ResourceType: action.ResourceTypeID, - Resource: action.ResourceTypeID, - }.Build(), - }.Build() - reqAnnos.Update(&v2.TypeScopedGrants{}) + resource, reqAnnos = typeScopedRequestStub(action.ResourceTypeID, &v2.TypeScopedGrants{}) } else { resourceResponse, err := s.store.GetResource(ctx, reader_v2.ResourcesReaderServiceGetResourceRequest_builder{ ResourceId: resourceID, @@ -2504,66 +2392,16 @@ func (s *syncer) syncGrantsForResource(ctx context.Context, action *Action) erro s.counts.LogGrantsProgress(ctx, resourceID.GetResourceType()) } - // EnqueuePageTokens: the response may enqueue sibling cursors — each runs - // as its own action, scheduled by the worker pool, rate-limited, and - // checkpointed like any other pagination. - // - // - Type-scoped calls: one cursor per connector-defined shard (e.g. - // 50-id delta chunks); actions carry only the resource type. - // - Per-resource calls: parallel page fan-out for page-numbered APIs - // (the connector knows every page URL from page one, so a warm - // sync can revalidate all pages concurrently instead of serially); - // actions carry the resource identity. - // - // Spawned cursors are ORDINARY pages that happen to be enqueued - // eagerly: the SDK assumes nothing about replay — a spawned page may - // hit its lookup and replay, miss and fetch cold (page boundary - // shifted since last sync), and may chain further via NextPageToken. - spawn := &v2.EnqueuePageTokens{} - hasSpawn, err := respAnnos.Pick(spawn) + // EnqueuePageTokens: the response may enqueue sibling cursors — + // type-scoped shards or per-resource parallel pages. Spawned cursors + // are ORDINARY pages that happen to be enqueued eagerly: the SDK + // assumes nothing about replay — a spawned page may hit its lookup + // and replay, miss and fetch cold (page boundary shifted since last + // sync), and may chain further via NextPageToken. Shared contract: + // see collectEnqueuedPageTokens (type_scoped.go). + spawned, err := s.collectEnqueuedPageTokens(ctx, "sync-grants-for-resource", SyncGrantsOp, action, respAnnos) if err != nil { - return fmt.Errorf("sync-grants-for-resource: error parsing spawn-cursors annotation: %w", err) - } - var spawned []Action - if hasSpawn { - // Every spawned action is persisted in the checkpointed state - // token, so an unbounded fan-out (a buggy planner emitting - // millions of tokens, or a spawn-inside-spawn loop) would bloat - // every checkpoint write before anything else fails. Cap - // per-response and fail loudly, mirroring the lookup bounce cap. - // Legitimate patterns sit far below this: Entra spawns ~35 chunk - // cursors per planning page, GitHub one sibling per stored page. - if len(spawn.GetPageTokens()) > maxEnqueuePageTokensPerResponse { - return fmt.Errorf( - "sync-grants-for-resource: EnqueuePageTokens carried %d page tokens (max %d per response); chain additional spawns across pages instead", - len(spawn.GetPageTokens()), maxEnqueuePageTokensPerResponse) - } - for _, tok := range spawn.GetPageTokens() { - if tok == "" { - continue - } - if len(tok) > maxSpawnCursorTokenBytes { - return fmt.Errorf( - "sync-grants-for-resource: EnqueuePageTokens page token is %d bytes (max %d)", - len(tok), maxSpawnCursorTokenBytes) - } - spawned = append(spawned, Action{ - Op: SyncGrantsOp, - ResourceTypeID: action.ResourceTypeID, - ResourceID: action.ResourceID, // empty on type-scoped actions - PageToken: tok, - Spawned: true, - }) - } - if len(spawned) > 0 { - s.state.AddSourceCacheStats(SourceCacheStats{SpawnedCursors: int64(len(spawned))}) - } - l.Debug("sync-grants-for-resource: spawned sibling grant cursors", - zap.String("resource_type_id", action.ResourceTypeID), - zap.String("resource_id", action.ResourceID), - zap.Bool("type_scoped", typeScoped), - zap.Int("cursors", len(spawned)), - zap.Int64("estimated_total", spawn.GetEstimatedTotal())) + return err } return s.nextPageOrFinishAction(ctx, action, resp.GetNextPageToken(), spawned...) @@ -3800,7 +3638,7 @@ func NewSyncer(ctx context.Context, c types.ConnectorClient, opts ...SyncOpt) (S // without ETag replay, never a failed sync. The caller that // maintains the cache replaces it after its next successful // upload, so a bad file self-heals. - ctxzap.Extract(ctx).Warn("previous-sync c1z unusable; syncing without etag replay", + ctxzap.Extract(ctx).Warn("previous-sync c1z unusable; syncing without source-cache replay", zap.String("previous_sync_c1z_path", s.previousSyncC1ZPath), zap.Error(err), ) diff --git a/pkg/sync/type_scoped.go b/pkg/sync/type_scoped.go new file mode 100644 index 000000000..cc89efe8b --- /dev/null +++ b/pkg/sync/type_scoped.go @@ -0,0 +1,148 @@ +package sync //nolint:revive,nolintlint // we can't change the package name for backwards compatibility + +// Shared plumbing for TYPE-SCOPED grants and entitlements (see +// proto/c1/connector/v2/annotation_type_scoped_grants.proto and its +// entitlements sibling). Both phases have identical mechanics — a +// resource-type annotation opts a type out of the per-resource fan-out, +// the syncer issues one planner action whose request carries the marker +// annotation over a {type, type} stub resource, and the connector may +// fan out sibling cursors via EnqueuePageTokens — so the mechanics live +// here once, parameterized by marker/cache/op, and the phase functions +// in syncer.go stay thin. + +import ( + "context" + "fmt" + + "go.uber.org/zap" + "google.golang.org/protobuf/proto" + + "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" + + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + reader_v2 "github.com/conductorone/baton-sdk/pb/c1/reader/v2" + "github.com/conductorone/baton-sdk/pkg/annotations" +) + +// resourceTypeCarries reports (cached per sync in cache) whether the +// resource type's stored annotations contain marker. +func (s *syncer) resourceTypeCarries(ctx context.Context, resourceTypeID string, marker proto.Message, cache *syncMap[string, bool]) (bool, error) { + if v, ok := cache.Load(resourceTypeID); ok { + return v, nil + } + rt, err := s.store.GetResourceType(ctx, reader_v2.ResourceTypesReaderServiceGetResourceTypeRequest_builder{ + ResourceTypeId: resourceTypeID, + }.Build()) + if err != nil { + return false, err + } + rtAnnos := annotations.Annotations(rt.GetResourceType().GetAnnotations()) + carries := rtAnnos.Contains(marker) + cache.Store(resourceTypeID, carries) + return carries, nil +} + +// resourceTypesCarrying lists every synced resource type whose stored +// annotations contain marker, warming cache for every type seen. +func (s *syncer) resourceTypesCarrying(ctx context.Context, marker proto.Message, cache *syncMap[string, bool]) ([]string, error) { + var out []string + pageToken := "" + for { + resp, err := s.store.ListResourceTypes(ctx, v2.ResourceTypesServiceListResourceTypesRequest_builder{ + PageToken: pageToken, + }.Build()) + if err != nil { + return nil, err + } + for _, rt := range resp.GetList() { + rtAnnos := annotations.Annotations(rt.GetAnnotations()) + carries := rtAnnos.Contains(marker) + cache.Store(rt.GetId(), carries) + if carries { + out = append(out, rt.GetId()) + } + } + pageToken = resp.GetNextPageToken() + if pageToken == "" { + return out, nil + } + } +} + +// typeScopedRequestStub builds the wire shape of a type-scoped list +// request: wire validation requires a non-empty resource id, so the stub +// resource is self-referential ({type, type}) and the marker annotation +// on the REQUEST is the routing signal the builder dispatches on. +func typeScopedRequestStub(resourceTypeID string, marker proto.Message) (*v2.Resource, annotations.Annotations) { + stub := v2.Resource_builder{ + Id: v2.ResourceId_builder{ + ResourceType: resourceTypeID, + Resource: resourceTypeID, + }.Build(), + }.Build() + var reqAnnos annotations.Annotations + reqAnnos.Update(marker) + return stub, reqAnnos +} + +// collectEnqueuedPageTokens parses a response's EnqueuePageTokens +// annotation and converts its page tokens into sibling actions — each an +// ORDINARY page that happens to be enqueued eagerly: scheduled by the +// worker pool, rate-limited, and checkpointed like any other pagination. +// +// - Type-scoped calls: one cursor per connector-defined shard (e.g. +// 50-id delta chunks); actions carry only the resource type. +// - Per-resource calls: parallel page fan-out for page-numbered APIs; +// actions carry the resource identity. +// +// Every spawned action is persisted in the checkpointed state token, so +// fan-out is capped per response and oversized/empty tokens fail loudly +// (matching the proto contract) rather than bloating every checkpoint or +// silently dropping a shard. +func (s *syncer) collectEnqueuedPageTokens(ctx context.Context, phase string, op ActionOp, action *Action, respAnnos annotations.Annotations) ([]Action, error) { + spawn := &v2.EnqueuePageTokens{} + hasSpawn, err := respAnnos.Pick(spawn) + if err != nil { + return nil, fmt.Errorf("%s: error parsing enqueue-page-tokens annotation: %w", phase, err) + } + if !hasSpawn { + return nil, nil + } + if len(spawn.GetPageTokens()) > maxEnqueuePageTokensPerResponse { + return nil, fmt.Errorf( + "%s: EnqueuePageTokens carried %d page tokens (max %d per response); chain additional spawns across pages instead", + phase, len(spawn.GetPageTokens()), maxEnqueuePageTokensPerResponse) + } + spawned := make([]Action, 0, len(spawn.GetPageTokens())) + for _, tok := range spawn.GetPageTokens() { + if tok == "" { + // An empty token would spawn a PLANNER call, not a page — + // a connector bug that would loop the fan-out. Loud, like + // the caps. + return nil, fmt.Errorf("%s: EnqueuePageTokens carried an empty page token", phase) + } + if len(tok) > maxEnqueuedPageTokenBytes { + return nil, fmt.Errorf( + "%s: EnqueuePageTokens page token is %d bytes (max %d)", + phase, len(tok), maxEnqueuedPageTokenBytes) + } + spawned = append(spawned, Action{ + Op: op, + ResourceTypeID: action.ResourceTypeID, + ResourceID: action.ResourceID, // empty on type-scoped actions + PageToken: tok, + Spawned: true, + }) + } + if len(spawned) > 0 { + s.state.AddSourceCacheStats(SourceCacheStats{EnqueuedPageTokens: int64(len(spawned))}) + } + ctxzap.Extract(ctx).Debug(phase+": enqueued sibling page-token cursors", + zap.String("resource_type_id", action.ResourceTypeID), + zap.String("resource_id", action.ResourceID), + zap.Int("cursors", len(spawned)), + // estimated_total is advisory (logging/observability only); + // nothing gates on it. + zap.Int64("estimated_total", spawn.GetEstimatedTotal())) + return spawned, nil +} diff --git a/pkg/sync/whale_invariants_cost_test.go b/pkg/sync/whale_invariants_cost_test.go new file mode 100644 index 000000000..372e80dac --- /dev/null +++ b/pkg/sync/whale_invariants_cost_test.go @@ -0,0 +1,129 @@ +package sync //nolint:revive,nolintlint // we can't change the package name for backwards compatibility + +// Throwaway cost measurement for runIngestionInvariants against a real +// (whale-scale, post-expansion) pebble c1z. Times each invariant's +// underlying store work exactly as the syncer performs it. Gated on +// BATON_WHALE_C1Z pointing at the artifact. +// +// TEMPORARY VERIFICATION FILE — safe to delete; nothing references it. + +import ( + "os" + "testing" + "time" + + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + "github.com/conductorone/baton-sdk/pkg/dotc1z" + "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" + "github.com/conductorone/baton-sdk/pkg/logging" + + "github.com/stretchr/testify/require" +) + +func TestWhaleInvariantCost(t *testing.T) { + path := os.Getenv("BATON_WHALE_C1Z") + if path == "" { + t.Skip("set BATON_WHALE_C1Z to the artifact path") + } + ctx, err := logging.Init(t.Context(), logging.WithLogLevel("error")) + require.NoError(t, err) + + store, err := dotc1z.NewStore(ctx, path, + dotc1z.WithEngine(c1zstore.EnginePebble), + dotc1z.WithReadOnly(true), + dotc1z.WithTmpDir(t.TempDir()), + ) + require.NoError(t, err) + defer func() { _ = store.Close(ctx) }() + + run, err := store.SyncMeta().LatestFullSync(ctx) + require.NoError(t, err) + require.NotNil(t, run) + require.NoError(t, store.SetCurrentSync(ctx, run.ID)) + + facts, ok := store.(dotc1z.IngestFactStore) + require.True(t, ok) + sc, isSC := store.(dotc1z.SourceCacheStore) + + // --- I3: grant→resource referential check, exactly as + // checkGrantResourceReferences performs it (skip-scan + point Get per + // distinct entitlement resource; the insert-fact probe only on + // dangling refs). + start := time.Now() + distinct, dangling, annotated := 0, 0, 0 + require.NoError(t, facts.ForEachDistinctGrantEntitlementResource(ctx, func(rt, rid string) error { + distinct++ + exists, err := facts.HasResourceRecord(ctx, rt, rid) + if err != nil { + return err + } + if exists { + return nil + } + dangling++ + carry, err := facts.GrantsForEntResourceCarryInsertFact(ctx, rt, rid) + if err != nil { + return err + } + if carry { + annotated++ + } + return nil + })) + i3 := time.Since(start) + t.Logf("I3 grant→resource: %v (distinct ent-resources=%d dangling=%d insert-annotated=%d)", i3, distinct, dangling, annotated) + + // --- I5: stored-keyspace exclusion-group validation, exactly as + // validateStoredExclusionGroups performs it (full entitlement listing + // + tracker.record per row). + start = time.Now() + tracker := &exclusionGroupTracker{} + ents := 0 + pageToken := "" + for { + resp, err := store.ListEntitlements(ctx, v2.EntitlementsServiceListEntitlementsRequest_builder{PageToken: pageToken}.Build()) + require.NoError(t, err) + for _, ent := range resp.GetList() { + ents++ + require.NoError(t, tracker.record(ent)) + } + if pageToken = resp.GetNextPageToken(); pageToken == "" { + break + } + } + i5 := time.Since(start) + t.Logf("I5 exclusion groups: %v (entitlements=%d)", i5, ents) + + // --- I6: scope-index vs manifest consistency. + if isSC { + start = time.Now() + orphans, err := sc.SourceCacheOrphanScopes(ctx) + require.NoError(t, err) + i6 := time.Since(start) + t.Logf("I6 orphan scopes: %v (orphans=%d kinds)", i6, len(orphans)) + } + + // --- I4 (STRICT-ONLY in production; measured for reference): full + // resource listing + annotation walk, as checkChildScheduling would. + start = time.Now() + resources, withCRT := 0, 0 + pageToken = "" + for { + resp, err := store.ListResources(ctx, v2.ResourcesServiceListResourcesRequest_builder{PageToken: pageToken}.Build()) + require.NoError(t, err) + for _, r := range resp.GetList() { + resources++ + for _, a := range r.GetAnnotations() { + if a.MessageIs((*v2.ChildResourceType)(nil)) { + withCRT++ + break + } + } + } + if pageToken = resp.GetNextPageToken(); pageToken == "" { + break + } + } + i4 := time.Since(start) + t.Logf("I4 (strict-only) child scheduling scan: %v (resources=%d with-child-annotations=%d)", i4, resources, withCRT) +} diff --git a/pkg/tasks/c1api/full_sync.go b/pkg/tasks/c1api/full_sync.go index ed2fe9fca..b6221831a 100644 --- a/pkg/tasks/c1api/full_sync.go +++ b/pkg/tasks/c1api/full_sync.go @@ -44,7 +44,7 @@ type fullSyncTaskHandler struct { workerCount int storageEngine c1zstore.Engine - // previousSyncSparePath is the connector's ETag-replay opt-in: when + // previousSyncSparePath is the connector's source-cache-replay opt-in: when // non-empty, the handler retains one spare c1z (the last successfully // uploaded sync) at exactly this path and feeds it to the next sync // as the replay source. Empty (the default) disables the feature @@ -96,7 +96,7 @@ func previousSyncSparePath(tempDir, clientID string) string { // payload decode) so the spare a future task replays from is positively // identifiable in the logs, and both failure paths warn — a connector // that opted in but whose rotation keeps failing must be visible, not -// silently etag-less forever. +// silently replay-less forever. func promoteOrRemoveC1Z(ctx context.Context, currentPath, sparePath string, keep bool) { l := ctxzap.Extract(ctx) if keep { @@ -107,7 +107,7 @@ func promoteOrRemoveC1Z(ctx context.Context, currentPath, sparePath string, keep zap.String("spare_sync_id", c1zSyncIDBestEffort(sparePath))) return } - l.Warn("failed to retain uploaded c1z as previous-sync spare; removing it instead (etag replay will be inactive next sync)", + l.Warn("failed to retain uploaded c1z as previous-sync spare; removing it instead (source-cache replay will be inactive next sync)", zap.String("current_path", currentPath), zap.String("spare_path", sparePath), zap.Error(err)) @@ -188,22 +188,22 @@ func (c *fullSyncTaskHandler) sync(ctx context.Context, c1zPath string) error { syncOpts = append(syncOpts, sdkSync.WithExternalResourceC1ZPath(c.externalResourceC1ZPath)) } - // ETag replay (opt-in): feed the spare retained from the last + // source-cache replay (opt-in): feed the spare retained from the last // successful upload as the previous-sync replay source. Optional // semantics — a missing/corrupt/stale-format spare degrades to a // plain sync and is replaced by this task's rotation on success, // so a bad spare self-heals and can never fail the sync. Both // branches log: a persistently-absent spare on an opted-in // connector means rotation is broken, and that must be visible - // rather than silently disabling etag replay forever. + // rather than silently disabling source-cache replay forever. if c.previousSyncSparePath != "" { if fileExists(c.previousSyncSparePath) { - l.Info("previous-sync spare found; etag replay enabled for this sync", + l.Info("previous-sync spare found; source-cache replay enabled for this sync", zap.String("spare_path", c.previousSyncSparePath), zap.String("spare_sync_id", c1zSyncIDBestEffort(c.previousSyncSparePath))) syncOpts = append(syncOpts, sdkSync.WithOptionalPreviousSyncC1ZPath(c.previousSyncSparePath)) } else { - l.Info("no previous-sync spare on disk; etag replay inactive for this sync (expected on the first sync after enabling, or after a failed rotation)", + l.Info("no previous-sync spare on disk; source-cache replay inactive for this sync (expected on the first sync after enabling, or after a failed rotation)", zap.String("spare_path", c.previousSyncSparePath)) } } diff --git a/pkg/tasks/c1api/manager.go b/pkg/tasks/c1api/manager.go index c0df2348a..b88b6b096 100644 --- a/pkg/tasks/c1api/manager.go +++ b/pkg/tasks/c1api/manager.go @@ -71,7 +71,7 @@ type c1ApiTaskManager struct { storageEngine c1zstore.Engine // previousSyncSparePath is non-empty when the connector opted into - // ETag replay (keepPreviousSyncC1Z): the fixed, client-id-namespaced + // source-cache replay (keepPreviousSyncC1Z): the fixed, client-id-namespaced // location of the retained previous-sync c1z. Computed once at // construction so every full-sync task agrees on the same spare. previousSyncSparePath string diff --git a/pkg/types/resource/resource.go b/pkg/types/resource/resource.go index 49a01e0f1..4b246a0eb 100644 --- a/pkg/types/resource/resource.go +++ b/pkg/types/resource/resource.go @@ -526,7 +526,7 @@ func NewManagedDeviceResource( type SyncOpAttrs struct { Session sessions.SessionStore - // SourceCache resolves a scope's previous-sync validator (etag / + // SourceCache resolves a scope's previous-sync validator (HTTP ETag / // delta token) for source-cache replay. Never nil for framework-built // connectors: when source cache is disabled or degraded the SDK // supplies sourcecache.NoopLookup, which always misses. diff --git a/proto/c1/connector/v2/annotation_type_scoped_grants.proto b/proto/c1/connector/v2/annotation_type_scoped_grants.proto index 0fdc07907..ef7707817 100644 --- a/proto/c1/connector/v2/annotation_type_scoped_grants.proto +++ b/proto/c1/connector/v2/annotation_type_scoped_grants.proto @@ -24,8 +24,9 @@ option go_package = "github.com/conductorone/baton-sdk/pb/c1/connector/v2"; // for the whole type, across as many paginated cursors as it chooses to // spawn (see EnqueuePageTokens). // -// This is the grants-phase analogue of StaticEntitlements: the connector -// takes over enumeration for the type, and every downstream behavior +// The connector takes over enumeration for the type (the same contract +// as TypeScopedEntitlements in +// annotation_type_scoped_entitlements.proto), and every downstream behavior // (storage, source-cache scopes/replay/tombstones, grant expansion, rate // limiting, page-token checkpointing) is unchanged because grants are // self-describing rows. diff --git a/proto/c1/connectorapi/baton/v1/source_cache.proto b/proto/c1/connectorapi/baton/v1/source_cache.proto index a03b9ebc9..37a510b2c 100644 --- a/proto/c1/connectorapi/baton/v1/source_cache.proto +++ b/proto/c1/connectorapi/baton/v1/source_cache.proto @@ -25,7 +25,7 @@ service BatonSourceCacheService { message LookupRequest { // Row kind: resources / entitlements / grants // (pkg/sourcecache.RowKind values). Entries are partitioned by row - // kind, so one scope hash can carry a different validator per kind. + // kind, so one scope key can carry a different validator per kind. string row_kind = 1 [(validate.rules).string = { min_len: 1 max_len: 64 From 81d71c21f320fe840ef82c6cc8c9ca69ab5e67d8 Mon Sep 17 00:00:00 2001 From: Matt Kaniaris Date: Tue, 14 Jul 2026 23:17:07 -0600 Subject: [PATCH 08/17] referential integrity error handling for replayed grants that point at non-existant resources --- pkg/dotc1z/engine/pebble/source_cache.go | 32 ++++ pkg/dotc1z/source_cache.go | 19 +++ pkg/sync/ingest_invariants.go | 90 ++++++++++- pkg/sync/ingest_invariants_repair_test.go | 184 ++++++++++++++++++++++ pkg/sync/source_cache.go | 10 +- pkg/sync/syncer.go | 13 ++ pkg/sync/whale_invariants_cost_test.go | 18 +++ pkg/tasks/c1api/full_sync.go | 56 +++++-- pkg/tasks/local/syncer.go | 47 ++++-- 9 files changed, 431 insertions(+), 38 deletions(-) create mode 100644 pkg/sync/ingest_invariants_repair_test.go diff --git a/pkg/dotc1z/engine/pebble/source_cache.go b/pkg/dotc1z/engine/pebble/source_cache.go index 73c6fed44..ec30fb146 100644 --- a/pkg/dotc1z/engine/pebble/source_cache.go +++ b/pkg/dotc1z/engine/pebble/source_cache.go @@ -921,6 +921,38 @@ func (e *Engine) ReplaySourceCacheGrants(ctx context.Context, prev *Engine, scop return res, nil } +// CopyResourceRowFrom copies one resource row (primary + secondary +// indexes) from prev into the receiver, byte-verbatim. The repair side of +// ingestion invariant I3: when a grant-inserted resource row was lost by +// a replay-path failure, the previous sync's row is the full-fidelity +// source of truth (it was cold-materialized from the wire grant's +// embedded resource, which grant STORAGE does not retain). Returns false +// when prev has no such row — repair impossible. +func (e *Engine) CopyResourceRowFrom(ctx context.Context, prev *Engine, resourceTypeID, resourceID string) (bool, error) { + copied := false + err := e.withWrite(func() error { + if err := e.requireCurrentSync(); err != nil { + return err + } + batch := e.db.NewBatch() + defer batch.Close() + ok, err := e.replayRelatedResourceRaw(batch, prev, encodeResourceKey(resourceTypeID, resourceID), resourceTypeID, resourceID) + if err != nil { + return err + } + if !ok { + return nil + } + if err := batch.Commit(writeOpts(e.opts.durability)); err != nil { + return err + } + _ = e.takeFreshResourcesEmpty() + copied = true + return nil + }) + return copied, err +} + // replayRelatedResourceRaw copies one resource row from prev into batch // (primary + secondary indexes), byte-verbatim — the engine side of // recreating InsertResourceGrants-inserted resources when their grants diff --git a/pkg/dotc1z/source_cache.go b/pkg/dotc1z/source_cache.go index b8f447197..476caa177 100644 --- a/pkg/dotc1z/source_cache.go +++ b/pkg/dotc1z/source_cache.go @@ -140,6 +140,25 @@ func (s *pebbleStore) SourceCacheOrphanScopes(ctx context.Context) (map[string][ return s.engine.SourceCacheOrphanScopes(ctx) } +// RepairRelatedResource copies one resource row from prev (the previous +// sync's store) into the current sync — ingestion invariant I3's repair +// arm for grant-inserted resources lost by a replay-path failure. +// Returns false when prev holds no such row (repair impossible). +func (s *pebbleStore) RepairRelatedResource(ctx context.Context, prev connectorstore.Reader, resourceTypeID, resourceID string) (bool, error) { + prevEngine, ok := sourceCacheEngine(prev) + if !ok { + return false, errors.New("related-resource repair: previous sync store is not a pebble store") + } + copied, err := s.engine.CopyResourceRowFrom(ctx, prevEngine, resourceTypeID, resourceID) + if err != nil { + return false, err + } + if copied { + s.MarkDirty() + } + return copied, nil +} + // sourceCacheEngine recovers the Pebble engine from an arbitrary store, // nil-safe. Mirrors pebble.AsEngine but accepts any value so the syncer // can probe its previous-sync reader without caring about its static type. diff --git a/pkg/sync/ingest_invariants.go b/pkg/sync/ingest_invariants.go index d53860086..16292d670 100644 --- a/pkg/sync/ingest_invariants.go +++ b/pkg/sync/ingest_invariants.go @@ -53,6 +53,7 @@ package sync //nolint:revive,nolintlint // we can't change the package name for import ( "context" + "errors" "fmt" "sort" stdsync "sync" @@ -154,10 +155,43 @@ func (s *syncer) repairExternalMatchFlag(ctx context.Context) error { return nil } +// ErrReplayIntegrity classifies sync failures where source-cache REPLAY +// state is implicated: a replay copy that lost or dropped rows, or +// store-derived evidence that a replayed artifact is inconsistent. Since +// replay is a pure optimization, every such failure has a safe +// remediation the runner can apply automatically — discard the previous +// sync source and re-run the sync cold (see the runners' cold-retry). +// +// The retry doubles as the attribution experiment: a clean cold run +// means replay was at fault (and the sync self-healed); a cold run that +// STILL violates the invariant is re-attributed to connector data — the +// cold-path failure deliberately does NOT carry this sentinel, so a +// recurring connector bug fails plainly (and cheaply) every sync instead +// of looping through wasted warm-attempt-plus-retry cycles under an SDK +// label. Also deliberately NOT applied to connector-contract violations +// (e.g. a replay requested for a scope that was never looked up): a cold +// retry would mask those without fixing the connector. +var ErrReplayIntegrity = errors.New("source-cache replay integrity failure") + +// relatedResourceRepairer is the optional store capability behind I3's +// repair arm (pebble implements it). +type relatedResourceRepairer interface { + RepairRelatedResource(ctx context.Context, prev connectorstore.Reader, resourceTypeID, resourceID string) (bool, error) +} + // checkGrantResourceReferences is invariant I3: every distinct // entitlement resource referenced by a grant must exist as a resource // row. One seek per distinct resource (the grant keyspace leads with the // entitlement resource), value reads only for dangling refs. +// +// The annotated arm (dangling ref + InsertResourceGrants) is +// REPAIR-flavored in lenient mode: those rows are the syncer's own +// establishment guarantee (no ListResources call ever returns them), and +// the previous sync's row — still open on s.sourceCache.prevReader — is +// a full-fidelity copy source, exactly the copy replay itself performs. +// Strict mode (tests, the harness) skips repair and hard-fails so the +// underlying machinery bug is named rather than papered over. An +// unrepairable annotated ref fails the sync under ErrReplayIntegrity. func (s *syncer) checkGrantResourceReferences(ctx context.Context) error { if s.syncType != connectorstore.SyncTypeFull { return nil @@ -180,9 +214,41 @@ func (s *syncer) checkGrantResourceReferences(ctx context.Context) error { return fmt.Errorf("ingest invariant I3: probing grant annotations for %s/%s: %w", rt, rid, err) } if annotated { + if s.sourceCache.prev == nil { + // COLD sync: replay cannot be the culprit — the cold + // path materializes these rows unconditionally in the + // same page handling, so this state means the connector + // destroyed or never supplied the row (e.g. a tombstone + // naming a grant-inserted resource). Deliberately NOT + // ErrReplayIntegrity: a cold retry can't fix connector + // data, and mislabeling would send operators hunting an + // SDK replay bug. This is also the terminal verdict of + // the runners' cold retry — a warm failure retried cold + // that still fails lands here, correctly re-attributed. + return fmt.Errorf( + "ingest invariant I3 violated on a COLD sync: grants reference resource %s/%s via InsertResourceGrants "+ + "but no resource row exists — the connector's own data is inconsistent "+ + "(check for tombstones deleting grant-inserted resources)", + rt, rid) + } + if !s.strictIngestionInvariants { + if repaired := s.repairRelatedResource(ctx, rt, rid); repaired { + l.Warn("ingest invariant I3: REPAIRED a lost grant-inserted resource by copying the previous sync's row; "+ + "the replay path failed to carry it — investigate before trusting warm syncs", + zap.String("resource_type_id", rt), + zap.String("resource_id", rid), + ) + return nil + } + } + // WARM sync: replay is implicated (possibly alongside a + // connector bug — the runners' cold retry disambiguates: + // a clean cold run means replay was at fault and the sync + // self-healed; a cold failure lands in the branch above. return fmt.Errorf( - "ingest invariant I3 violated: grants reference resource %s/%s via InsertResourceGrants but no resource row exists (related-resource insertion was lost)", - rt, rid) + "ingest invariant I3 violated: grants reference resource %s/%s via InsertResourceGrants but no resource row exists "+ + "(related-resource insertion was lost and could not be repaired from the previous sync): %w", + rt, rid, ErrReplayIntegrity) } // Tolerated today: connectors emitting grants for unlisted // resources without InsertResourceGrants. Visible, not fatal. @@ -194,6 +260,26 @@ func (s *syncer) checkGrantResourceReferences(ctx context.Context) error { }) } +// repairRelatedResource attempts I3's repair: copy the missing +// grant-inserted resource row from the previous sync's store. Best +// effort — a false return falls through to the hard failure. +func (s *syncer) repairRelatedResource(ctx context.Context, rt, rid string) bool { + repairer, ok := s.store.(relatedResourceRepairer) + if !ok || s.sourceCache.prevReader == nil { + return false + } + repaired, err := repairer.RepairRelatedResource(ctx, s.sourceCache.prevReader, rt, rid) + if err != nil { + ctxzap.Extract(ctx).Warn("ingest invariant I3: related-resource repair failed", + zap.String("resource_type_id", rt), + zap.String("resource_id", rid), + zap.Error(err), + ) + return false + } + return repaired +} + // checkChildScheduling is invariant I4: every stored resource carrying a // ChildResourceType annotation (and passing the resource-type filter) // must have had its child action scheduled. Check-only — scheduling diff --git a/pkg/sync/ingest_invariants_repair_test.go b/pkg/sync/ingest_invariants_repair_test.go new file mode 100644 index 000000000..1d175196a --- /dev/null +++ b/pkg/sync/ingest_invariants_repair_test.go @@ -0,0 +1,184 @@ +package sync //nolint:revive,nolintlint // we can't change the package name for backwards compatibility + +// I3 repair-arm tests: a grant-inserted resource lost from the current +// sync (the replay-loss class) is repaired by copying the previous +// sync's row in lenient mode, and hard-fails under ErrReplayIntegrity in +// strict mode (so mutation tests still name the machinery bug) or when +// no previous source can supply the row. + +import ( + "context" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + reader_v2 "github.com/conductorone/baton-sdk/pb/c1/reader/v2" + "github.com/conductorone/baton-sdk/pkg/annotations" + "github.com/conductorone/baton-sdk/pkg/connectorstore" + "github.com/conductorone/baton-sdk/pkg/dotc1z" + "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" + "github.com/conductorone/baton-sdk/pkg/logging" + "github.com/conductorone/baton-sdk/pkg/types/grant" + rs "github.com/conductorone/baton-sdk/pkg/types/resource" +) + +func repairTestGrant(t *testing.T, repo *v2.Resource) *v2.Grant { + t.Helper() + principal := v2.Resource_builder{ + Id: v2.ResourceId_builder{ResourceType: "user", Resource: "alice"}.Build(), + }.Build() + g := grant.NewGrant(repo, "admin", principal.GetId()) + annos := annotations.Annotations(g.GetAnnotations()) + annos.Update(&v2.InsertResourceGrants{}) + g.SetAnnotations(annos) + return g +} + +func newRepairTestStore(ctx context.Context, t *testing.T, path, tmpDir string) c1zstore.Store { + t.Helper() + store, err := dotc1z.NewStore(ctx, path, + dotc1z.WithEngine(c1zstore.EnginePebble), + dotc1z.WithTmpDir(tmpDir), + ) + require.NoError(t, err) + return store +} + +func TestIngestInvariantI3RepairsLostRelatedResource(t *testing.T) { + ctx, err := logging.Init(t.Context()) + require.NoError(t, err) + tmpDir := t.TempDir() + + repo, err := rs.NewResource("Repo r1", equivRepoRT, "r1") + require.NoError(t, err) + g := repairTestGrant(t, repo) + + // Previous sync: healthy state — annotated grant AND its resource. + prev := newRepairTestStore(ctx, t, filepath.Join(tmpDir, "prev.c1z"), tmpDir) + _, err = prev.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + require.NoError(t, prev.PutResourceTypes(ctx, equivRepoRT)) + require.NoError(t, prev.PutResources(ctx, repo)) + require.NoError(t, prev.PutGrants(ctx, g)) + require.NoError(t, prev.EndSync(ctx)) + + // Current sync: the replay-loss shape — annotated grant, NO resource. + cur := newRepairTestStore(ctx, t, filepath.Join(tmpDir, "cur.c1z"), tmpDir) + _, err = cur.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + require.NoError(t, cur.PutResourceTypes(ctx, equivRepoRT)) + require.NoError(t, cur.PutGrants(ctx, g)) + + s := &syncer{store: cur, syncType: connectorstore.SyncTypeFull} + prevSC, ok := any(prev).(dotc1z.SourceCacheStore) + require.True(t, ok) + s.sourceCache.prev = prevSC + s.sourceCache.prevReader = prev + + // Strict mode: no repair — the machinery bug must be NAMED, and the + // failure classified as replay-integrity. + s.strictIngestionInvariants = true + err = s.checkGrantResourceReferences(ctx) + require.Error(t, err) + require.ErrorIs(t, err, ErrReplayIntegrity) + + // Lenient mode: repaired from the previous sync's row, full fidelity. + s.strictIngestionInvariants = false + require.NoError(t, s.checkGrantResourceReferences(ctx)) + got, err := cur.GetResource(ctx, reader_v2.ResourcesReaderServiceGetResourceRequest_builder{ + ResourceId: repo.GetId(), + }.Build()) + require.NoError(t, err) + require.Equal(t, "Repo r1", got.GetResource().GetDisplayName(), + "repair must restore the previous sync's full-fidelity row, not a stub") + + // Idempotent: a second pass finds nothing dangling. + require.NoError(t, s.checkGrantResourceReferences(ctx)) + + require.NoError(t, cur.Close(ctx)) + require.NoError(t, prev.Close(ctx)) +} + +// TestWithoutPreviousSyncForcesCold pins the runners' cold-retry +// mechanism: appending WithoutPreviousSync AFTER a previous-sync path +// option must fully disable replay — the connector sees only lookup +// misses and the sync completes cold. +func TestWithoutPreviousSyncForcesCold(t *testing.T) { + ctx, err := logging.Init(t.Context()) + require.NoError(t, err) + tmpDir := t.TempDir() + + mc := newSourceCacheMockConnector() + res, _, err := mc.AddGroup(ctx, "g1") + require.NoError(t, err) + _, err = mc.AddUser(ctx, "u1") + require.NoError(t, err) + mc.etagByResource[res.GetId().GetResource()] = "v1" + + // Warm-capable chain: sync 1 stamps, sync 2 (control) replays. + sync1 := filepath.Join(tmpDir, "s1.c1z") + runSourceCacheSync(ctx, t, mc, sync1, "", tmpDir) + sync2 := filepath.Join(tmpDir, "s2.c1z") + runSourceCacheSync(ctx, t, mc, sync2, sync1, tmpDir) + require.Positive(t, mc.lookupHits, "control: the warm sync must have replayed") + + // The retry shape: previous path SET, WithoutPreviousSync appended + // last. The connector must see only misses. + hitsBefore := mc.lookupHits + sync3 := filepath.Join(tmpDir, "s3.c1z") + runSourceCacheSync(ctx, t, mc, sync3, sync1, tmpDir, WithoutPreviousSync()) + require.Equal(t, hitsBefore, mc.lookupHits, "WithoutPreviousSync must force every lookup to miss") +} + +func TestIngestInvariantI3UnrepairableFailsWithReplayIntegrity(t *testing.T) { + ctx, err := logging.Init(t.Context()) + require.NoError(t, err) + tmpDir := t.TempDir() + + repo, err := rs.NewResource("Repo r1", equivRepoRT, "r1") + require.NoError(t, err) + g := repairTestGrant(t, repo) + + // Previous sync exists but is ALSO missing the resource: repair has + // no source, so even lenient mode must fail — and classify as + // replay-integrity so the runners' cold retry can take over. + prev := newRepairTestStore(ctx, t, filepath.Join(tmpDir, "prev.c1z"), tmpDir) + _, err = prev.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + require.NoError(t, prev.PutResourceTypes(ctx, equivRepoRT)) + require.NoError(t, prev.EndSync(ctx)) + + cur := newRepairTestStore(ctx, t, filepath.Join(tmpDir, "cur.c1z"), tmpDir) + _, err = cur.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + require.NoError(t, cur.PutResourceTypes(ctx, equivRepoRT)) + require.NoError(t, cur.PutGrants(ctx, g)) + + s := &syncer{store: cur, syncType: connectorstore.SyncTypeFull} + prevSC, ok := any(prev).(dotc1z.SourceCacheStore) + require.True(t, ok) + s.sourceCache.prev = prevSC + s.sourceCache.prevReader = prev + + // WARM + unrepairable: replay is implicated → replay-integrity, so + // the runners retry cold (the attribution experiment). + err = s.checkGrantResourceReferences(ctx) + require.Error(t, err) + require.ErrorIs(t, err, ErrReplayIntegrity) + + // COLD (no previous source): replay cannot be the culprit — the + // error must blame connector data and must NOT carry the replay + // sentinel, or the runners would burn a pointless cold retry every + // sync on a recurring connector bug. + s.sourceCache.prev = nil + s.sourceCache.prevReader = nil + err = s.checkGrantResourceReferences(ctx) + require.Error(t, err) + require.NotErrorIs(t, err, ErrReplayIntegrity) + require.Contains(t, err.Error(), "COLD sync") + + require.NoError(t, cur.Close(ctx)) + require.NoError(t, prev.Close(ctx)) +} diff --git a/pkg/sync/source_cache.go b/pkg/sync/source_cache.go index 41eba8a9d..8adf20198 100644 --- a/pkg/sync/source_cache.go +++ b/pkg/sync/source_cache.go @@ -391,7 +391,11 @@ func (s *syncer) executeScopeReplay(ctx context.Context, kind sourcecache.RowKin res, err := s.sourceCache.current.ReplaySourceCache(ctx, s.sourceCache.prevReader, kind, page.scopeKey) s.state.AddStepDuration("source_cache_replay", time.Since(replayStart)) if err != nil { - return fmt.Errorf("source cache: replay for scope %q failed: %w", page.scopeKey, err) + // Classified as a replay-integrity failure: whether the copy + // tripped its own count check or plain I/O died mid-copy, the + // replay state can't be trusted and a cold re-run is the safe + // remediation (see ErrReplayIntegrity). + return fmt.Errorf("source cache: replay for scope %q failed: %w: %w", page.scopeKey, err, ErrReplayIntegrity) } // Stats are stashed on the page and recorded in // finishSourceCachePage so a page re-run after failing between @@ -420,8 +424,8 @@ func (s *syncer) executeScopeReplay(ctx context.Context, kind sourcecache.RowKin if res.Rows == 0 && res.StaleSkipped > 0 { return fmt.Errorf( "source cache: replay for scope %q copied no rows but the previous file's scope index holds %d entries "+ - "whose rows no longer carry this scope's stamp; refusing to silently drop the scope", - page.scopeKey, res.StaleSkipped) + "whose rows no longer carry this scope's stamp; refusing to silently drop the scope: %w", + page.scopeKey, res.StaleSkipped, ErrReplayIntegrity) } if res.StaleSkipped > 0 { // Partial staleness: some rows copied, some index entries were diff --git a/pkg/sync/syncer.go b/pkg/sync/syncer.go index f7643a9e0..46ce36e8d 100644 --- a/pkg/sync/syncer.go +++ b/pkg/sync/syncer.go @@ -3460,6 +3460,19 @@ func WithOptionalPreviousSyncC1ZPath(path string) SyncOpt { } } +// WithoutPreviousSync clears any previous-sync replay source, forcing a +// cold sync. Applied LAST it overrides an earlier +// With(Optional)PreviousSyncC1ZPath — the runners' cold-retry appends it +// to the original option set after a replay-integrity failure +// (ErrReplayIntegrity) so the re-run cannot consult the implicated +// artifact. +func WithoutPreviousSync() SyncOpt { + return func(s *syncer) { + s.previousSyncC1ZPath = "" + s.previousSyncC1ZPathOptional = false + } +} + func WithExternalResourceEntitlementIdFilter(entitlementId string) SyncOpt { return func(s *syncer) { s.externalResourceEntitlementIdFilter = entitlementId diff --git a/pkg/sync/whale_invariants_cost_test.go b/pkg/sync/whale_invariants_cost_test.go index 372e80dac..749ae6db4 100644 --- a/pkg/sync/whale_invariants_cost_test.go +++ b/pkg/sync/whale_invariants_cost_test.go @@ -103,6 +103,24 @@ func TestWhaleInvariantCost(t *testing.T) { t.Logf("I6 orphan scopes: %v (orphans=%d kinds)", i6, len(orphans)) } + // --- Contrast: a full O(grants) pass over the grant keyspace — what + // I3 would cost if it were per-grant instead of per-distinct-resource. + // Uses the raw listing read path (value decode per row, like any + // full-table invariant would pay). + start = time.Now() + grants := 0 + pageToken = "" + for { + resp, err := store.ListGrants(ctx, v2.GrantsServiceListGrantsRequest_builder{PageToken: pageToken, PageSize: 1000}.Build()) + require.NoError(t, err) + grants += len(resp.GetList()) + if pageToken = resp.GetNextPageToken(); pageToken == "" { + break + } + } + scan := time.Since(start) + t.Logf("CONTRAST full grant scan (O(grants)): %v (grants=%d) — vs I3's %v", scan, grants, i3) + // --- I4 (STRICT-ONLY in production; measured for reference): full // resource listing + annotation walk, as checkChildScheduling would. start = time.Now() diff --git a/pkg/tasks/c1api/full_sync.go b/pkg/tasks/c1api/full_sync.go index b6221831a..935c57b5d 100644 --- a/pkg/tasks/c1api/full_sync.go +++ b/pkg/tasks/c1api/full_sync.go @@ -8,6 +8,7 @@ import ( "io" "os" "path/filepath" + "slices" "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" "go.uber.org/zap" @@ -235,28 +236,49 @@ func (c *fullSyncTaskHandler) sync(ctx context.Context, c1zPath string) error { syncOpts = append(syncOpts, sdkSync.WithSessionStore(setSessionStore)) } - syncer, err := sdkSync.NewSyncer(ctx, c.helpers.ConnectorClient(), syncOpts...) - if err != nil { - l.Error("failed to create syncer", zap.Error(err)) - return err + runSync := func(opts []sdkSync.SyncOpt) error { + syncer, err := sdkSync.NewSyncer(ctx, c.helpers.ConnectorClient(), opts...) + if err != nil { + l.Error("failed to create syncer", zap.Error(err)) + return err + } + err = syncer.Sync(ctx) + if err != nil { + // We don't defer syncer.Close() in order to capture the error without named return values. + if closeErr := syncer.Close(ctx); closeErr != nil { + l.Error("failed to close syncer after sync error", zap.Error(closeErr)) + err = errors.Join(err, closeErr) + } + return err + } + return syncer.Close(ctx) } // TODO(jirwin): Should we attempt to retry at all before failing the task? - err = syncer.Sync(ctx) - if err != nil { - l.Error("failed to sync", zap.Error(err)) - - // We don't defer syncer.Close() in order to capture the error without named return values. - if closeErr := syncer.Close(ctx); closeErr != nil { - l.Error("failed to close syncer after sync error", zap.Error(err)) - err = errors.Join(err, closeErr) + err = runSync(syncOpts) + if errors.Is(err, sdkSync.ErrReplayIntegrity) { + // Replay is a pure optimization: a replay-integrity failure means + // the warm sync's state cannot be trusted, and the safe automatic + // remediation is a cold re-run — discard the partial output (its + // replayed rows may be wrong; resuming would keep them), retire + // the spare (it is implicated until a fresh rotation replaces + // it), and sync again without a previous source. One retry: a + // cold sync cannot fail this way again. + l.Error("source-cache replay integrity failure; discarding replay state and re-running the sync cold", + zap.Error(err), + zap.String("spare_path", c.previousSyncSparePath)) + if rmErr := os.Remove(c1zPath); rmErr != nil && !os.IsNotExist(rmErr) { + return errors.Join(err, rmErr) } - - return err + if c.previousSyncSparePath != "" { + if rmErr := os.Remove(c.previousSyncSparePath); rmErr != nil && !os.IsNotExist(rmErr) { + l.Warn("failed to remove implicated previous-sync spare", zap.Error(rmErr)) + } + } + err = runSync(append(slices.Clone(syncOpts), sdkSync.WithoutPreviousSync())) } - - if err := syncer.Close(ctx); err != nil { - l.Error("failed to close syncer", zap.Error(err)) + if err != nil { + l.Error("failed to sync", zap.Error(err)) return err } diff --git a/pkg/tasks/local/syncer.go b/pkg/tasks/local/syncer.go index 916b166db..a36d65217 100644 --- a/pkg/tasks/local/syncer.go +++ b/pkg/tasks/local/syncer.go @@ -3,10 +3,13 @@ package local import ( "context" "errors" + "os" + "slices" "sync" "time" "go.opentelemetry.io/otel/trace" + "go.uber.org/zap" v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" v1 "github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1" @@ -17,6 +20,7 @@ import ( "github.com/conductorone/baton-sdk/pkg/types" "github.com/conductorone/baton-sdk/pkg/uotel" "github.com/conductorone/baton-sdk/pkg/uotel/uotelzap" + "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" ) type localSyncer struct { @@ -142,24 +146,35 @@ func (m *localSyncer) Process(ctx context.Context, task *v1.Task, cc types.Conne syncOpts = append(syncOpts, sdkSync.WithStorageEngine(m.storageEngine)) } - syncer, err := sdkSync.NewSyncer(ctx, cc, syncOpts...) - if err != nil { - return err - } - - err = syncer.Sync(ctx) - if err != nil { - if closeErr := syncer.Close(ctx); closeErr != nil { - err = errors.Join(err, closeErr) + runSync := func(opts []sdkSync.SyncOpt) error { + syncer, err := sdkSync.NewSyncer(ctx, cc, opts...) + if err != nil { + return err } - return err - } - - if err := syncer.Close(ctx); err != nil { - return err + if err := syncer.Sync(ctx); err != nil { + if closeErr := syncer.Close(ctx); closeErr != nil { + err = errors.Join(err, closeErr) + } + return err + } + return syncer.Close(ctx) + } + + err = runSync(syncOpts) + if errors.Is(err, sdkSync.ErrReplayIntegrity) { + // Replay is a pure optimization: discard the partial output (its + // replayed rows may be wrong; resuming would keep them) and + // re-run cold, without the previous-sync source. One retry: a + // cold sync cannot fail this way again. See ErrReplayIntegrity. + ctxzap.Extract(ctx).Error("source-cache replay integrity failure; discarding replay state and re-running the sync cold", + zap.Error(err), + zap.String("previous_sync_c1z", m.previousSyncC1Z)) + if rmErr := os.Remove(m.dbPath); rmErr != nil && !os.IsNotExist(rmErr) { + return errors.Join(err, rmErr) + } + err = runSync(append(slices.Clone(syncOpts), sdkSync.WithoutPreviousSync())) } - - return nil + return err } // NewSyncer returns a task manager that queues a sync task. From 0b83cab76b533a9321fa2d65bc0deac72bb2a2b2 Mon Sep 17 00:00:00 2001 From: Matt Kaniaris Date: Wed, 15 Jul 2026 10:51:43 -0600 Subject: [PATCH 09/17] drop dangling data --- pkg/dotc1z/engine/pebble/ingest_facts.go | 116 +++++ pkg/dotc1z/engine/pebble/ingest_repair.go | 462 ++++++++++++++++++ pkg/dotc1z/engine/pebble/keys.go | 13 + pkg/dotc1z/engine/pebble/lookup.go | 5 +- pkg/dotc1z/engine/pebble/source_cache.go | 24 - .../engine/pebble/source_cache_engine_test.go | 23 - pkg/dotc1z/source_cache.go | 91 +++- pkg/sync/ingest_invariants.go | 387 ++++++++++++++- .../ingest_invariants_referential_test.go | 334 +++++++++++++ pkg/sync/ingest_invariants_repair_test.go | 64 ++- pkg/sync/ingest_invariants_scale_test.go | 171 +++++++ pkg/sync/ingest_invariants_test.go | 2 + pkg/sync/source_cache.go | 6 +- pkg/sync/source_cache_equivalence_test.go | 2 +- pkg/sync/source_cache_sideeffects_test.go | 2 +- pkg/sync/source_cache_test.go | 2 +- pkg/sync/syncer.go | 19 +- pkg/sync/type_scoped_grants_test.go | 19 +- pkg/sync/whale_invariants_cost_test.go | 70 ++- pkg/synccompactor/source_cache_replay_test.go | 9 + 20 files changed, 1721 insertions(+), 100 deletions(-) create mode 100644 pkg/dotc1z/engine/pebble/ingest_repair.go create mode 100644 pkg/sync/ingest_invariants_referential_test.go create mode 100644 pkg/sync/ingest_invariants_scale_test.go diff --git a/pkg/dotc1z/engine/pebble/ingest_facts.go b/pkg/dotc1z/engine/pebble/ingest_facts.go index 9b2cb66cc..e813cfb5d 100644 --- a/pkg/dotc1z/engine/pebble/ingest_facts.go +++ b/pkg/dotc1z/engine/pebble/ingest_facts.go @@ -163,6 +163,122 @@ func (e *Engine) ForEachDistinctGrantEntitlementResource(ctx context.Context, vi return iter.Error() } +// ForEachDistinctEntitlementResource visits each distinct +// (resource_type_id, resource_id) pair present in the entitlement +// primary keyspace, in key order. Same prefix-skip shape as the grant +// scan: one seek per distinct resource, never O(entitlements). Backs +// the syncer's entitlement→resource referential invariant (I7). +func (e *Engine) ForEachDistinctEntitlementResource(ctx context.Context, visit func(resourceTypeID, resourceID string) error) error { + prefix := encodeEntitlementPrefix() + iter, err := e.db.NewIter(&pebble.IterOptions{ + LowerBound: prefix, + UpperBound: upperBoundOf(prefix), + }) + if err != nil { + return err + } + defer iter.Close() + + const headerLen = 3 // versionV3, typeEntitlement, separator + for valid := iter.First(); valid; { + if err := ctx.Err(); err != nil { + return err + } + key := iter.Key() + if len(key) <= headerLen { + return fmt.Errorf("distinct entitlement-resource scan: malformed entitlement key %x", key) + } + tail := key[headerLen:] + rtBytes, next, ok := codec.DecodeTupleStringAlias(tail, 0) + if !ok || next >= len(tail) { + return fmt.Errorf("distinct entitlement-resource scan: malformed entitlement key tail %x", key) + } + ridBytes, _, ok := codec.DecodeTupleStringAlias(tail, next+1) + if !ok { + return fmt.Errorf("distinct entitlement-resource scan: malformed entitlement key tail %x", key) + } + rt, rid := string(rtBytes), string(ridBytes) + if err := visit(rt, rid); err != nil { + return err + } + valid = iter.SeekGE(upperBoundOf(encodeEntitlementPrimaryResourcePrefix(rt, rid))) + } + return iter.Error() +} + +// ForEachDanglingGrantEntitlement visits each distinct entitlement +// identity referenced by grants for which NO entitlement row exists. +// The grant primary key leads with the full entitlement identity tuple +// (which is byte-identical to the entitlement primary key's tuple), so +// the scan is one seek per distinct entitlement plus one point-Get +// probe each — O(distinct entitlements), never O(grants), and no value +// reads: the identity tuple reconstructs the external id byte-exactly +// (see identity.go). Backs the syncer's grant→entitlement referential +// invariant (I8). +func (e *Engine) ForEachDanglingGrantEntitlement(ctx context.Context, visit func(entitlementID, resourceTypeID, resourceID string) error) error { + prefix := encodeGrantPrefix() + iter, err := e.db.NewIter(&pebble.IterOptions{ + LowerBound: prefix, + UpperBound: upperBoundOf(prefix), + }) + if err != nil { + return err + } + defer iter.Close() + + const headerLen = 3 // versionV3, typeGrant, separator + for valid := iter.First(); valid; { + if err := ctx.Err(); err != nil { + return err + } + key := iter.Key() + if len(key) <= headerLen { + return fmt.Errorf("dangling grant-entitlement scan: malformed grant key %x", key) + } + tail := key[headerLen:] + var comps [4]string + off := 0 + for i := range comps { + b, next, ok := codec.DecodeTupleStringAlias(tail, off) + if !ok || (i < len(comps)-1 && next >= len(tail)) { + return fmt.Errorf("dangling grant-entitlement scan: malformed grant key tail %x", key) + } + comps[i] = string(b) + off = next + 1 + } + id := entitlementIdentity{ + resourceTypeID: comps[0], + resourceID: comps[1], + stripped: comps[2] == idFlagStripped, + tail: comps[3], + } + exists, err := e.hasEntitlementIdentity(id) + if err != nil { + return err + } + if !exists { + if err := visit(id.externalID(), id.resourceTypeID, id.resourceID); err != nil { + return err + } + } + // Skip every remaining grant of this entitlement. + valid = iter.SeekGE(upperBoundOf(encodeGrantPrimaryEntitlementIdentityPrefix(comps[0], comps[1], comps[2], comps[3]))) + } + return iter.Error() +} + +func (e *Engine) hasEntitlementIdentity(id entitlementIdentity) (bool, error) { + _, closer, err := e.db.Get(encodeEntitlementIdentityKey(id)) + if err != nil { + if errors.Is(err, pebble.ErrNotFound) { + return false, nil + } + return false, err + } + closer.Close() + return true, nil +} + // GrantsForEntResourceCarryInsertFact reports whether any grant under the // given entitlement resource carries the InsertResourceGrants annotation. // Reads row values, so it is reserved for DANGLING referential probes — diff --git a/pkg/dotc1z/engine/pebble/ingest_repair.go b/pkg/dotc1z/engine/pebble/ingest_repair.go new file mode 100644 index 000000000..f17a02ca0 --- /dev/null +++ b/pkg/dotc1z/engine/pebble/ingest_repair.go @@ -0,0 +1,462 @@ +package pebble + +// Dangling-reference drops: the engine mechanics behind the syncer's +// lenient-mode referential invariants (I7/I8/I9 in +// pkg/sync/ingest_invariants.go). Connectors ship magic-id construction +// bugs — grants and annotations naming entitlements/resources whose rows +// were never synced — and the platform's ingestion provably discards +// those rows, so the syncer drops them at the post-collection seam and +// reports aggregates. Policy (what to drop, exemptions, strict-mode +// failure) lives in the syncer; these methods are pure mechanics. +// +// All deletes route through the same locked helpers as interactive +// deletes, so secondary indexes and (if already built) digest +// invalidation stay consistent. Populations are small in practice +// (dangling refs are bugs, not steady state), so per-row batches are +// acceptable. + +import ( + "context" + "errors" + "fmt" + + "github.com/cockroachdb/pebble/v2" + + v3 "github.com/conductorone/baton-sdk/pb/c1/storage/v3" + "github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/codec" +) + +// EnsureGrantIndexes runs the deferred grant-index build NOW if one is +// pending. The by_principal index is inline for connector-emitted grants +// but deferred for expansion/synth writes; the dangling-principal scan +// (I9) needs it complete. Calling early is near-free: the same build +// would run at EndSync anyway — this just moves it before the invariant +// seam (rows dropped afterward maintain the indexes inline and stage +// digest invalidation, which EndSync's repair pass heals). +func (e *Engine) EnsureGrantIndexes(ctx context.Context) error { + if !e.deferredIdxPending.Load() { + return nil + } + return e.BuildDeferredGrantIndexes(ctx) +} + +// ForEachDanglingGrantPrincipal visits each distinct principal referenced +// by grants for which NO resource row exists. One seek per distinct +// principal over the by_principal index (which leads with principal +// identity) plus one point probe each — O(distinct principals), never +// O(grants). Callers must EnsureGrantIndexes first or the scan misses +// synth-written grants. +// +// matchAnnotatedOnly reports that EVERY grant of the dangling principal +// carries an ExternalResourceMatch* annotation — the legitimate carrier +// shape when no external resource file was configured (the match op +// deletes carriers when it runs, so annotated survivors always mean the +// op didn't run or didn't match). carrierGrants is the number of grant +// rows under such a principal, so callers can report per-GRANT totals +// (the mixed-principal path reports its own per-grant skip count). +// Value reads happen only for dangling principals, never on the +// healthy path. +func (e *Engine) ForEachDanglingGrantPrincipal(ctx context.Context, visit func(principalRT, principalID string, matchAnnotatedOnly bool, carrierGrants int64) error) error { + prefix := []byte{versionV3, typeIndex, idxGrantByPrincipal} + prefix = codec.AppendTupleSeparator(prefix) + iter, err := e.db.NewIter(&pebble.IterOptions{ + LowerBound: prefix, + UpperBound: upperBoundOf(prefix), + }) + if err != nil { + return err + } + defer iter.Close() + + for valid := iter.First(); valid; { + if err := ctx.Err(); err != nil { + return err + } + key := iter.Key() + tail := key[len(prefix):] + rtBytes, next, ok := codec.DecodeTupleStringAlias(tail, 0) + if !ok || next >= len(tail) { + return fmt.Errorf("dangling grant-principal scan: malformed index key %x", key) + } + ridBytes, _, ok := codec.DecodeTupleStringAlias(tail, next+1) + if !ok { + return fmt.Errorf("dangling grant-principal scan: malformed index key %x", key) + } + rt, rid := string(rtBytes), string(ridBytes) + exists, err := e.HasResourceRecord(ctx, rt, rid) + if err != nil { + return err + } + if !exists { + matchOnly, carrierGrants, err := e.grantsForPrincipalAllMatchAnnotated(ctx, rt, rid) + if err != nil { + return err + } + if err := visit(rt, rid, matchOnly, carrierGrants); err != nil { + return err + } + } + // Skip every remaining grant of this principal. + valid = iter.SeekGE(upperBoundOf(encodeGrantByPrincipalPrefix(rt, rid))) + } + return iter.Error() +} + +// grantsForPrincipalAllMatchAnnotated reports whether every grant under +// the principal carries an ExternalResourceMatch* annotation, and how +// many grant rows that is (valid only when the bool is true — the walk +// stops at the first non-annotated grant). Reserved for dangling +// principals (reads row values). +func (e *Engine) grantsForPrincipalAllMatchAnnotated(ctx context.Context, principalRT, principalID string) (bool, int64, error) { + ids, err := e.grantIdentitiesForPrincipal(ctx, principalRT, principalID) + if err != nil { + return false, 0, err + } + var carrierGrants int64 + for _, id := range ids { + rec, err := e.getGrantRecordByIdentity(id) + if err != nil { + if errors.Is(err, pebble.ErrNotFound) { + continue // index entry without a row; the scan tolerates it + } + return false, 0, err + } + if !grantRecordHasExternalMatch(rec.GetAnnotations()) { + return false, 0, nil + } + carrierGrants++ + } + return true, carrierGrants, nil +} + +// grantIdentitiesForPrincipal collects the grant identities under one +// principal from the by_principal index. Collected before any deletes so +// callers never interleave iteration with writes. +func (e *Engine) grantIdentitiesForPrincipal(ctx context.Context, principalRT, principalID string) ([]grantIdentity, error) { + prefix := encodeGrantByPrincipalPrefix(principalRT, principalID) + iter, err := e.db.NewIter(&pebble.IterOptions{ + LowerBound: prefix, + UpperBound: upperBoundOf(prefix), + }) + if err != nil { + return nil, err + } + defer iter.Close() + + var ids []grantIdentity + for iter.First(); iter.Valid(); iter.Next() { + if err := ctx.Err(); err != nil { + return nil, err + } + tail := iter.Key()[len(prefix):] + var comps [4]string + off := 0 + for i := range comps { + b, next, ok := codec.DecodeTupleStringAlias(tail, off) + if !ok { + return nil, fmt.Errorf("by_principal index: malformed key tail %x", iter.Key()) + } + comps[i] = string(b) + off = next + 1 + } + ids = append(ids, grantIdentity{ + entitlement: entitlementIdentity{ + resourceTypeID: comps[0], + resourceID: comps[1], + stripped: comps[2] == idFlagStripped, + tail: comps[3], + }, + principalTypeID: principalRT, + principalID: principalID, + }) + } + return ids, iter.Error() +} + +func (e *Engine) getGrantRecordByIdentity(id grantIdentity) (*v3.GrantRecord, error) { + val, closer, err := e.db.Get(encodeGrantIdentityKey(id)) + if err != nil { + return nil, err + } + defer closer.Close() + rec := &v3.GrantRecord{} + if err := unmarshalRecord(val, rec); err != nil { + return nil, fmt.Errorf("grant record by identity: unmarshal: %w", err) + } + return rec, nil +} + +// dropBatchRows chunks the streaming drop batches: large enough to +// amortize commit overhead, small enough to bound batch memory. Drops +// during a fresh sync ride NoSync (the seal fsyncs), matching the +// tombstone and replay write paths — WITHOUT this, a dangling +// entitlement with a large grant population would pay one fsync per +// row. +const dropBatchRows = 4096 + +// dropWriteOpts returns the write options for streaming drops. +func (e *Engine) dropWriteOpts() *pebble.WriteOptions { + if e.IsFreshSync() { + return pebble.NoSync + } + return writeOpts(e.opts.durability) +} + +// DeleteGrantsForPrincipal deletes every grant of one principal EXCEPT +// rows carrying ExternalResourceMatch* annotations (unprocessed match +// carriers are evidence of a missing external-resource file, not bad +// data). Streams one chunked batch pass — never one commit per row. +// Returns (deleted, skippedMatchAnnotated). +func (e *Engine) DeleteGrantsForPrincipal(ctx context.Context, principalRT, principalID string) (int64, int64, error) { + indexPrefix := encodeGrantByPrincipalPrefix(principalRT, principalID) + + var deleted, skipped int64 + err := e.withWrite(func() error { + iter, err := e.db.NewIter(&pebble.IterOptions{ + LowerBound: indexPrefix, + UpperBound: upperBoundOf(indexPrefix), + }) + if err != nil { + return err + } + defer iter.Close() + + opts := e.dropWriteOpts() + batch := e.db.NewBatch() + defer func() { _ = batch.Close() }() + rowsInBatch := 0 + + for iter.First(); iter.Valid(); iter.Next() { + if err := ctx.Err(); err != nil { + return err + } + // Index tail: ent identity components (the principal is the + // prefix). Primary key = grant header + identity tail in + // primary order. + tail := iter.Key()[len(indexPrefix):] + var comps [4]string + off := 0 + for i := range comps { + b, next, ok := codec.DecodeTupleStringAlias(tail, off) + if !ok { + return fmt.Errorf("by_principal index: malformed key tail %x", iter.Key()) + } + comps[i] = string(b) + off = next + 1 + } + id := grantIdentity{ + entitlement: entitlementIdentity{ + resourceTypeID: comps[0], + resourceID: comps[1], + stripped: comps[2] == idFlagStripped, + tail: comps[3], + }, + principalTypeID: principalRT, + principalID: principalID, + } + priKey := encodeGrantIdentityKey(id) + val, closer, getErr := e.db.Get(priKey) + if getErr != nil { + if errors.Is(getErr, pebble.ErrNotFound) { + continue // index entry without a row; tolerated + } + return getErr + } + rec := &v3.GrantRecord{} + if err := unmarshalRecord(val, rec); err != nil { + closer.Close() + return fmt.Errorf("dangling principal drop: unmarshal: %w", err) + } + if grantRecordHasExternalMatch(rec.GetAnnotations()) { + closer.Close() + skipped++ + continue + } + if err := e.deleteGrantIndexesRaw(batch, "", val); err != nil { + closer.Close() + return err + } + closer.Close() + if err := batch.Delete(priKey, nil); err != nil { + return err + } + deleted++ + rowsInBatch++ + if rowsInBatch >= dropBatchRows { + if err := batch.Commit(opts); err != nil { + return err + } + _ = batch.Close() + batch = e.db.NewBatch() + rowsInBatch = 0 + } + } + if err := iter.Error(); err != nil { + return err + } + return batch.Commit(opts) + }) + if err != nil { + return 0, 0, err + } + return deleted, skipped, nil +} + +// DeleteGrantsForEntitlement deletes every grant row under one +// entitlement identity — the drop arm for grants referencing an +// entitlement with no row. Streams the primary prefix with values in +// hand (index cleanup derives from the value), chunked batches, no +// per-row commits. +func (e *Engine) DeleteGrantsForEntitlement(ctx context.Context, entitlementID, entResourceTypeID, entResourceID string) (int64, error) { + entID := entitlementIdentityFromParts(entResourceTypeID, entResourceID, entitlementID) + prefix := encodeGrantPrimaryEntitlementIdentityPrefix( + entID.resourceTypeID, entID.resourceID, entID.flagComponent(), entID.tail) + + var deleted int64 + err := e.withWrite(func() error { + iter, err := e.db.NewIter(&pebble.IterOptions{ + LowerBound: prefix, + UpperBound: upperBoundOf(prefix), + }) + if err != nil { + return err + } + defer iter.Close() + + opts := e.dropWriteOpts() + batch := e.db.NewBatch() + defer func() { _ = batch.Close() }() + rowsInBatch := 0 + + for iter.First(); iter.Valid(); iter.Next() { + if err := ctx.Err(); err != nil { + return err + } + if err := e.deleteGrantIndexesRaw(batch, "", iter.Value()); err != nil { + return err + } + key := iter.Key() + k := make([]byte, len(key)) + copy(k, key) + if err := batch.Delete(k, nil); err != nil { + return err + } + deleted++ + rowsInBatch++ + if rowsInBatch >= dropBatchRows { + if err := batch.Commit(opts); err != nil { + return err + } + _ = batch.Close() + batch = e.db.NewBatch() + rowsInBatch = 0 + } + } + if err := iter.Error(); err != nil { + return err + } + return batch.Commit(opts) + }) + if err != nil { + return 0, err + } + return deleted, nil +} + +// DeleteEntitlementsForResource deletes every entitlement row under one +// resource — the drop arm for entitlements referencing a resource with no +// row. Returns the count and up to maxIDs deleted external ids for the +// caller's aggregate report. +func (e *Engine) DeleteEntitlementsForResource(ctx context.Context, resourceTypeID, resourceID string, maxIDs int) (int64, []string, error) { + prefix := encodeEntitlementPrimaryResourcePrefix(resourceTypeID, resourceID) + type entRow struct { + id entitlementIdentity + scopeKey string + primaryKy []byte + } + var rows []entRow + + iter, err := e.db.NewIter(&pebble.IterOptions{ + LowerBound: prefix, + UpperBound: upperBoundOf(prefix), + }) + if err != nil { + return 0, nil, err + } + const headerLen = 3 // versionV3, typeEntitlement, separator + for iter.First(); iter.Valid(); iter.Next() { + if err := ctx.Err(); err != nil { + iter.Close() + return 0, nil, err + } + key := iter.Key() + tail := key[headerLen:] + var comps [4]string + off := 0 + ok := true + for i := range comps { + b, next, decOK := codec.DecodeTupleStringAlias(tail, off) + if !decOK { + ok = false + break + } + comps[i] = string(b) + off = next + 1 + } + if !ok { + iter.Close() + return 0, nil, fmt.Errorf("dangling entitlement drop: malformed entitlement key %x", key) + } + scope, scanErr := scanEntitlementSourceScopeRaw(iter.Value()) + if scanErr != nil { + iter.Close() + return 0, nil, scanErr + } + k := make([]byte, len(key)) + copy(k, key) + rows = append(rows, entRow{ + id: entitlementIdentity{ + resourceTypeID: comps[0], + resourceID: comps[1], + stripped: comps[2] == idFlagStripped, + tail: comps[3], + }, + scopeKey: scope, + primaryKy: k, + }) + } + if err := iter.Error(); err != nil { + iter.Close() + return 0, nil, err + } + iter.Close() + + var deleted int64 + var ids []string + err = e.withWrite(func() error { + batch := e.db.NewBatch() + defer batch.Close() + for _, row := range rows { + if row.scopeKey != "" { + if err := batch.Delete(encodeEntitlementBySourceScopeIndexKey(row.scopeKey, row.id), nil); err != nil { + return err + } + } + if err := batch.Delete(row.primaryKy, nil); err != nil { + return err + } + deleted++ + if len(ids) < maxIDs { + ids = append(ids, row.id.externalID()) + } + } + if err := batch.Commit(e.dropWriteOpts()); err != nil { + return err + } + if deleted > 0 { + e.noteEntitlementKeyspaceWrite() + } + return nil + }) + return deleted, ids, err +} diff --git a/pkg/dotc1z/engine/pebble/keys.go b/pkg/dotc1z/engine/pebble/keys.go index f9d1e92a9..06f611a51 100644 --- a/pkg/dotc1z/engine/pebble/keys.go +++ b/pkg/dotc1z/engine/pebble/keys.go @@ -229,6 +229,19 @@ func encodeGrantPrimaryEntitlementResourcePrefix(resourceTypeID, resourceID stri return codec.AppendTupleSeparator(buf) } +// encodeGrantPrimaryEntitlementIdentityPrefix is the grant primary-key +// prefix covering every grant of one entitlement identity: the four +// leading tuple components (rt, rid, flag, tail) with a trailing +// separator. Components are passed raw (as decoded from a key) and +// re-escaped by the tuple codec, so the round trip is byte-exact. +func encodeGrantPrimaryEntitlementIdentityPrefix(resourceTypeID, resourceID, flag, tail string) []byte { + buf := make([]byte, 0, 96) + buf = append(buf, versionV3, typeGrant) + buf = codec.AppendTupleSeparator(buf) + buf = codec.AppendTupleStrings(buf, resourceTypeID, resourceID, flag, tail) + return codec.AppendTupleSeparator(buf) +} + // encodeGrantByNeedsExpansionIndexKey: index of grants whose // NeedsExpansion flag is true. Pebble equivalent of the SQLite // partial index `WHERE needs_expansion = 1`. The grant is added to diff --git a/pkg/dotc1z/engine/pebble/lookup.go b/pkg/dotc1z/engine/pebble/lookup.go index 282c3a9c2..1058b7b0b 100644 --- a/pkg/dotc1z/engine/pebble/lookup.go +++ b/pkg/dotc1z/engine/pebble/lookup.go @@ -271,8 +271,9 @@ func (e *Engine) resolveGrantIdentityByExternalID(ctx context.Context, grantID s } // Candidate probing proved nothing either way: the id may still be a // connector-custom STORED external id (with or without colons), which - // only the O(all grants) scan can find. Interactive edges pay it; - // bounded callers (DeleteGrantRecordBounded) stop before this line. + // only the O(all grants) scan can find. Only interactive edges reach + // this line; bulk paths (source-cache tombstones) resolve rows via + // the scope index instead and never pay the scan. return e.scanGrantIdentityByStoredExternalID(ctx, grantID) } diff --git a/pkg/dotc1z/engine/pebble/source_cache.go b/pkg/dotc1z/engine/pebble/source_cache.go index ec30fb146..5fb3bbf10 100644 --- a/pkg/dotc1z/engine/pebble/source_cache.go +++ b/pkg/dotc1z/engine/pebble/source_cache.go @@ -318,30 +318,6 @@ func (e *Engine) GetSourceCacheEntry(ctx context.Context, rowKind, scopeKey stri return rec, nil } -// DeleteGrantRecordBounded deletes a grant by canonical public id WITHOUT -// the O(all grants) stored-external-id scan fallback that the interactive -// DeleteGrantRecord path is allowed to take. Used by the source-cache -// tombstone path, where a mass-removal round would otherwise pay a full -// keyspace scan PER already-absent id. -// -// Consequence, by design: a grant stored under a connector-CUSTOM id (one -// that isn't the SDK concat shape) is unreachable here and the delete -// no-ops. Connectors with custom grant ids must use principal-scoped -// tombstones (SourceCacheRecord.deleted_principal_ids) instead — documented -// in the annotation proto. -func (e *Engine) DeleteGrantRecordBounded(ctx context.Context, externalID string) error { - return e.withWrite(func() error { - id, err := e.resolveGrantIdentityByCandidates(ctx, externalID) - if err != nil { - if errors.Is(err, errNoGrantCandidateHits) || errors.Is(err, pebble.ErrNotFound) { - return nil // absent (or custom-id) — tombstone no-op - } - return err - } - return e.deleteGrantByIdentityLocked(id) - }) -} - // DeleteGrantsByPrincipalsInScope deletes every grant row in the CURRENT // store stamped with scopeKey whose principal id is in principalIDs — // the engine side of principal-scoped delta tombstones diff --git a/pkg/dotc1z/engine/pebble/source_cache_engine_test.go b/pkg/dotc1z/engine/pebble/source_cache_engine_test.go index 6ce970d5d..aeffaab12 100644 --- a/pkg/dotc1z/engine/pebble/source_cache_engine_test.go +++ b/pkg/dotc1z/engine/pebble/source_cache_engine_test.go @@ -383,29 +383,6 @@ func TestDeleteResourcesByIDsInScope(t *testing.T) { require.NoError(t, err, "tombstone must be scope-relative") } -// TestDeleteGrantRecordBounded pins the no-scan contract: SDK-shaped ids -// resolve and delete; absent ids no-op without error (and without the -// O(all grants) fallback scan, by construction). -func TestDeleteGrantRecordBounded(t *testing.T) { - ctx := context.Background() - a := newAdapter(t) - _, err := a.StartNewSync(ctx, connectorstore.SyncTypeFull, "") - require.NoError(t, err) - - g := scGrant("member", "alice", false) - require.NoError(t, a.PutGrants(ctx, g)) - - // Absent id (well-formed, no row): no-op, no error. - require.NoError(t, a.PebbleEngine().DeleteGrantRecordBounded(ctx, "group:g1:custom:member:user:ghost")) - _, err = a.PebbleEngine().GetGrantRecord(ctx, g.GetId()) - require.NoError(t, err) - - // Present id: deleted. - require.NoError(t, a.PebbleEngine().DeleteGrantRecordBounded(ctx, g.GetId())) - _, err = a.PebbleEngine().GetGrantRecord(ctx, g.GetId()) - require.ErrorIs(t, err, pebble.ErrNotFound) -} - func TestSourceCacheReplayResourcesAndEntitlements(t *testing.T) { ctx := context.Background() diff --git a/pkg/dotc1z/source_cache.go b/pkg/dotc1z/source_cache.go index 476caa177..961177423 100644 --- a/pkg/dotc1z/source_cache.go +++ b/pkg/dotc1z/source_cache.go @@ -93,12 +93,15 @@ func (s *pebbleStore) SourceScopeIndexSnapshot(ctx context.Context) (map[string] return s.engine.SourceScopeIndexSnapshot(ctx) } -// IngestFactStore is the optional store capability backing the syncer's -// ingestion invariants (see docs/tasks/source-cache-ingestion-invariants.md): -// dense row facts tracked as existence bits, and the ordered-key -// inspection reads the referential invariant uses. Pebble-only; the -// syncer degrades per-invariant when the store lacks it. -type IngestFactStore interface { +// IngestInvariantStore is the optional store capability backing the +// syncer's ingestion invariants (see +// docs/tasks/source-cache-ingestion-invariants.md): dense row facts +// tracked as existence bits, the ordered-key scans the referential +// invariants (I3/I7/I8/I9) ride, and the drop/repair mechanics their +// default-mode verdicts apply. Pebble-only; the syncer degrades +// per-invariant when the store lacks it (SQLite artifacts are never +// scanned or mutated). +type IngestInvariantStore interface { // HasExternalMatchGrants reports whether the open sync holds at // least one grant carrying an ExternalResourceMatch* annotation — // the store-derived truth behind HasExternalResourcesGrants. @@ -116,9 +119,43 @@ type IngestFactStore interface { // HasResourceRecord reports whether a resource row exists. HasResourceRecord(ctx context.Context, resourceTypeID, resourceID string) (bool, error) + + // ForEachDistinctEntitlementResource visits each distinct resource + // referenced by any entitlement row: one seek per distinct + // resource, never O(entitlements). + ForEachDistinctEntitlementResource(ctx context.Context, visit func(resourceTypeID, resourceID string) error) error + + // ForEachDanglingGrantEntitlement visits each distinct entitlement + // referenced by grants that has no entitlement row: one seek plus + // one point probe per distinct entitlement, never O(grants). + ForEachDanglingGrantEntitlement(ctx context.Context, visit func(entitlementID, resourceTypeID, resourceID string) error) error + + // EnsureGrantIndexes forces a pending deferred grant-index build to + // run now, making by_principal complete for the dangling-principal + // scan. Near-free: the same build would run at EndSync. + EnsureGrantIndexes(ctx context.Context) error + + // ForEachDanglingGrantPrincipal visits each distinct principal + // referenced by grants that has no resource row. matchAnnotatedOnly + // reports that every grant of the principal is an unprocessed + // ExternalResourceMatch* carrier (exempt shape); carrierGrants is + // the per-grant count of that population. + ForEachDanglingGrantPrincipal(ctx context.Context, visit func(principalRT, principalID string, matchAnnotatedOnly bool, carrierGrants int64) error) error + + // DeleteEntitlementsForResource drops every entitlement row under a + // resource; returns the count and up to maxIDs deleted ids. + DeleteEntitlementsForResource(ctx context.Context, resourceTypeID, resourceID string, maxIDs int) (int64, []string, error) + + // DeleteGrantsForEntitlement drops every grant row under one + // entitlement identity; returns the count. + DeleteGrantsForEntitlement(ctx context.Context, entitlementID, entResourceTypeID, entResourceID string) (int64, error) + + // DeleteGrantsForPrincipal drops every grant of one principal except + // ExternalResourceMatch* carriers; returns (deleted, skipped). + DeleteGrantsForPrincipal(ctx context.Context, principalRT, principalID string) (int64, int64, error) } -var _ IngestFactStore = (*pebbleStore)(nil) +var _ IngestInvariantStore = (*pebbleStore)(nil) func (s *pebbleStore) HasExternalMatchGrants(ctx context.Context) (bool, error) { return s.engine.HasExternalMatchGrants(), nil @@ -136,6 +173,46 @@ func (s *pebbleStore) HasResourceRecord(ctx context.Context, resourceTypeID, res return s.engine.HasResourceRecord(ctx, resourceTypeID, resourceID) } +func (s *pebbleStore) ForEachDistinctEntitlementResource(ctx context.Context, visit func(resourceTypeID, resourceID string) error) error { + return s.engine.ForEachDistinctEntitlementResource(ctx, visit) +} + +func (s *pebbleStore) ForEachDanglingGrantEntitlement(ctx context.Context, visit func(entitlementID, resourceTypeID, resourceID string) error) error { + return s.engine.ForEachDanglingGrantEntitlement(ctx, visit) +} + +func (s *pebbleStore) EnsureGrantIndexes(ctx context.Context) error { + return s.engine.EnsureGrantIndexes(ctx) +} + +func (s *pebbleStore) ForEachDanglingGrantPrincipal(ctx context.Context, visit func(principalRT, principalID string, matchAnnotatedOnly bool, carrierGrants int64) error) error { + return s.engine.ForEachDanglingGrantPrincipal(ctx, visit) +} + +func (s *pebbleStore) DeleteEntitlementsForResource(ctx context.Context, resourceTypeID, resourceID string, maxIDs int) (int64, []string, error) { + n, ids, err := s.engine.DeleteEntitlementsForResource(ctx, resourceTypeID, resourceID, maxIDs) + if n > 0 { + s.MarkDirty() + } + return n, ids, err +} + +func (s *pebbleStore) DeleteGrantsForEntitlement(ctx context.Context, entitlementID, entResourceTypeID, entResourceID string) (int64, error) { + n, err := s.engine.DeleteGrantsForEntitlement(ctx, entitlementID, entResourceTypeID, entResourceID) + if n > 0 { + s.MarkDirty() + } + return n, err +} + +func (s *pebbleStore) DeleteGrantsForPrincipal(ctx context.Context, principalRT, principalID string) (int64, int64, error) { + n, skipped, err := s.engine.DeleteGrantsForPrincipal(ctx, principalRT, principalID) + if n > 0 { + s.MarkDirty() + } + return n, skipped, err +} + func (s *pebbleStore) SourceCacheOrphanScopes(ctx context.Context) (map[string][]string, error) { return s.engine.SourceCacheOrphanScopes(ctx) } diff --git a/pkg/sync/ingest_invariants.go b/pkg/sync/ingest_invariants.go index 16292d670..572363917 100644 --- a/pkg/sync/ingest_invariants.go +++ b/pkg/sync/ingest_invariants.go @@ -29,9 +29,9 @@ package sync //nolint:revive,nolintlint // we can't change the package name for // ChildResourceType must have had its (childType, parent) action // scheduled. Check-only; skipped when the resources phase did not // run in this process (a zero-row child listing leaves no store -// evidence that its action ran). Full syncs only. STRICT MODE ONLY: +// evidence that its action ran). Full syncs only. FAIL-FAST ONLY: // the check costs a full post-collection resource scan and its -// lenient outcome was only ever a warning, so production (lenient) +// default outcome was only ever a warning, so production (default) // skips it entirely — tests and the equivalence harness enforce it. // - I5 exclusion-group validation: the stored entitlement keyspace is // validated post-collection (one default per group, one resource @@ -45,6 +45,44 @@ package sync //nolint:revive,nolintlint // we can't change the package name for // Pebble-only (the only engine with source-cache state); the // replay-time counterpart (copied-row count vs the scope index) // lives in the engine's ReplaySourceCache* and always hard-fails. +// - I7 entitlement→resource referential integrity: post-collection +// prefix skip-scan over the entitlement keyspace (one seek per +// distinct resource). Per-resource entitlement scopes cannot dangle +// by construction (the scope is only visited when the resource was +// synced); the exposure is TYPE-scoped listings and magic-id +// construction bugs in connectors. +// - I8 grant→entitlement referential integrity: post-collection +// skip-scan over the grant keyspace at entitlement-identity +// granularity plus one point probe per distinct entitlement. +// Exposures: independent scope freshness under replay (grants scope +// 304s while the entitlement listing refreshed away a row) and, +// dominantly, connector magic-id bugs — annotations/grants naming +// entitlement ids the entitlement enumeration never produced +// (production expansion drops millions of edges per week to this +// class). EXEMPT: grants under InsertResourceGrants pages (the +// machinery inserts the resource; no listing returns an entitlement +// for it — an established shape). +// - I9 grant→principal referential integrity: dangling-principal scan +// over the by_principal index (one seek + probe per distinct +// principal; a pending deferred index build is forced first, which +// just moves EndSync's own pass earlier). EXEMPT: grants carrying +// ExternalResourceMatch* annotations — unprocessed match carriers +// are evidence that no external resource file was configured, not +// bad data (the match op deletes carriers when it runs). +// +// I7/I8/I9 violations are connector-attributable bad data. The +// platform's ingestion provably discards such rows (grants that resolve +// no principal or entitlement are skipped at uplift; grants under a +// missing entitlement are never even read), so by DEFAULT the +// syncer DROPS the rows and emits one aggregated warning per invariant +// (total dropped, distinct dangling refs, capped id examples — never +// per-row logs; the expansion path's per-edge warnings produce millions +// of lines a week in production). FAIL-FAST mode (tests, the harness) +// hard-fails so engineered violations are named. Deliberately NOT +// ErrReplayIntegrity: a dishonest validator makes the warm run fail but +// the cold run CLEAN, so the cold-retry ladder would never re-attribute +// to the connector and every sync would pay warm-fail-plus-retry +// forever. // // Evidence is monotone (existence bits, set-union scheduling records), // so parallel workers and mixed stream/replay arrival orders yield @@ -62,6 +100,7 @@ import ( "go.uber.org/zap" v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + reader_v2 "github.com/conductorone/baton-sdk/pb/c1/reader/v2" "github.com/conductorone/baton-sdk/pkg/annotations" "github.com/conductorone/baton-sdk/pkg/connectorstore" "github.com/conductorone/baton-sdk/pkg/dotc1z" @@ -83,6 +122,8 @@ var sideEffectAnnotationCoverage = map[string]string{ "c1.connector.v2.InsertResourceGrants": "I3: grant→resource referential check post-collection; replay carries the row copy", "c1.connector.v2.ChildResourceType": "I4: scheduled-set completeness check post-collection; replay carries the scheduling", "c1.connector.v2.EntitlementExclusionGroup": "I5: stored-keyspace validation post-collection", + "c1.connector.v2.TypeScopedEntitlements": "I7: entitlement→resource referential check post-collection (type-granularity scopes can carry rows for vanished resources)", + "c1.connector.v2.TypeScopedGrants": "I8: grant→entitlement referential check post-collection (independently-fresh scopes can strand grant references)", } // childScheduleSet is the monotone record of scheduled child-resource @@ -140,12 +181,16 @@ func (s *syncer) repairExternalMatchFlag(ctx context.Context) error { if s.state.HasExternalResourcesGrants() { return nil } - facts, ok := s.store.(dotc1z.IngestFactStore) + facts, ok := s.store.(dotc1z.IngestInvariantStore) if !ok { return nil // SQLite: the response-loop arm is the mechanism. } has, err := facts.HasExternalMatchGrants(ctx) if err != nil { + // Deliberately NOT ErrReplayIntegrity: this reads engine-meta on + // the CURRENT store — a failure here is store health, not replay + // state, and a cold retry against a sick store just fails + // elsewhere. Fail plainly so the real problem is investigated. return fmt.Errorf("ingest invariant I2: reading external-match fact: %w", err) } if has { @@ -185,18 +230,18 @@ type relatedResourceRepairer interface { // entitlement resource), value reads only for dangling refs. // // The annotated arm (dangling ref + InsertResourceGrants) is -// REPAIR-flavored in lenient mode: those rows are the syncer's own +// REPAIR-flavored in default mode: those rows are the syncer's own // establishment guarantee (no ListResources call ever returns them), and // the previous sync's row — still open on s.sourceCache.prevReader — is // a full-fidelity copy source, exactly the copy replay itself performs. -// Strict mode (tests, the harness) skips repair and hard-fails so the +// Fail-fast mode (tests, the harness) skips repair and hard-fails so the // underlying machinery bug is named rather than papered over. An // unrepairable annotated ref fails the sync under ErrReplayIntegrity. func (s *syncer) checkGrantResourceReferences(ctx context.Context) error { if s.syncType != connectorstore.SyncTypeFull { return nil } - facts, ok := s.store.(dotc1z.IngestFactStore) + facts, ok := s.store.(dotc1z.IngestInvariantStore) if !ok { return nil // engine without the inspection surface } @@ -231,7 +276,7 @@ func (s *syncer) checkGrantResourceReferences(ctx context.Context) error { "(check for tombstones deleting grant-inserted resources)", rt, rid) } - if !s.strictIngestionInvariants { + if !s.failFastInvariants { if repaired := s.repairRelatedResource(ctx, rt, rid); repaired { l.Warn("ingest invariant I3: REPAIRED a lost grant-inserted resource by copying the previous sync's row; "+ "the replay path failed to carry it — investigate before trusting warm syncs", @@ -286,11 +331,11 @@ func (s *syncer) repairRelatedResource(ctx context.Context, rt, rid string) bool // cannot be derived after the fact (an executed child action that // returned zero rows leaves no store evidence). func (s *syncer) checkChildScheduling(ctx context.Context) error { - if !s.strictIngestionInvariants { - // Strict mode only: the check costs a full post-collection + if !s.failFastInvariants { + // Fail-fast mode only: the check costs a full post-collection // resource scan (value decode + annotation walk per row) and in - // lenient mode a violation was only ever a warning — a bad - // trade at whale scale. Tests and the harness run strict and + // default mode a violation was only ever a warning — a bad + // trade at whale scale. Tests and the harness run fail-fast and // enforce it there. return nil } @@ -441,6 +486,287 @@ func (s *syncer) validateStoredExclusionGroups(ctx context.Context) error { return nil } +// maxDanglingIDExamples caps the id samples carried on the aggregated +// dangling-reference warnings. Production data shows a handful of +// distinct ids explain millions of dangling rows (magic-id construction +// bugs repeat), so a small sample is enough to pivot on. +const maxDanglingIDExamples = 25 + +// danglingTypeProbe memoizes "does a resource-type row exist" — the +// attribution split for dangling references. A reference into a type +// with NO type row means the type was never synced (typically a type +// added to the connector after the instance was configured and left +// disabled by default — a CONFIG gap); a missing row under a type that +// EXISTS means the id was constructed for a row the enumeration never +// produced (a magic-id BUG). +type danglingTypeProbe struct { + s *syncer + known map[string]bool +} + +func (p *danglingTypeProbe) typeExists(ctx context.Context, resourceTypeID string) bool { + if v, ok := p.known[resourceTypeID]; ok { + return v + } + if p.known == nil { + p.known = map[string]bool{} + } + _, err := p.s.store.GetResourceType(ctx, reader_v2.ResourceTypesReaderServiceGetResourceTypeRequest_builder{ + ResourceTypeId: resourceTypeID, + }.Build()) + exists := err == nil + p.known[resourceTypeID] = exists + return exists +} + +// danglingAttribution renders the config-gap vs bug split for the +// aggregated warnings. +func danglingAttribution(unsyncedTypeCount, missingRowCount int) string { + switch { + case unsyncedTypeCount > 0 && missingRowCount == 0: + return "referenced resource type(s) never synced — likely disabled on the connector (config gap), not a code bug" + case unsyncedTypeCount == 0 && missingRowCount > 0: + return "referenced type(s) ARE synced but the rows are missing — check the connector's magic-id construction" + default: + return "mixed: some references target never-synced types (config gap), some target missing rows in synced types (magic-id bug)" + } +} + +// checkEntitlementResourceReferences is invariant I7: every distinct +// resource referenced by an entitlement row must exist as a resource +// row. One seek per distinct resource among entitlements — bounded by +// the entitlement count. Default mode DROPS the dangling entitlement +// rows (uplift skips them anyway — see the header) and warns once with +// aggregates; fail-fast mode hard-fails. Never ErrReplayIntegrity. +func (s *syncer) checkEntitlementResourceReferences(ctx context.Context) error { + if s.syncType != connectorstore.SyncTypeFull { + return nil + } + facts, ok := s.store.(dotc1z.IngestInvariantStore) + if !ok { + return nil // engine without the inspection surface + } + probe := &danglingTypeProbe{s: s} + danglingResources, unsyncedType, missingRow := 0, 0, 0 + var droppedRows int64 + var resourceExamples, entIDExamples []string + err := facts.ForEachDistinctEntitlementResource(ctx, func(rt, rid string) error { + exists, err := facts.HasResourceRecord(ctx, rt, rid) + if err != nil { + return fmt.Errorf("ingest invariant I7: probing resource %s/%s: %w", rt, rid, err) + } + if exists { + return nil + } + if s.failFastInvariants { + return fmt.Errorf( + "ingest invariant I7 violated: entitlements reference resource %s/%s but no resource row exists "+ + "(resource type synced: %t — false means a config gap, true a magic-id construction bug)", + rt, rid, probe.typeExists(ctx, rt)) + } + n, ids, err := facts.DeleteEntitlementsForResource(ctx, rt, rid, maxDanglingIDExamples-len(entIDExamples)) + if err != nil { + return fmt.Errorf("ingest invariant I7: dropping entitlements for %s/%s: %w", rt, rid, err) + } + danglingResources++ + droppedRows += n + if probe.typeExists(ctx, rt) { + missingRow++ + } else { + unsyncedType++ + } + entIDExamples = append(entIDExamples, ids...) + if len(resourceExamples) < maxDanglingIDExamples { + resourceExamples = append(resourceExamples, rt+"/"+rid) + } + return nil + }) + if err != nil { + return err + } + if droppedRows > 0 { + ctxzap.Extract(ctx).Warn("ingest invariant I7: DROPPED entitlements referencing resources with no resource row", + zap.Int64("dropped_entitlements", droppedRows), + zap.Int("dangling_resources", danglingResources), + zap.Int("refs_into_unsynced_types", unsyncedType), + zap.Int("refs_missing_rows_in_synced_types", missingRow), + zap.String("attribution", danglingAttribution(unsyncedType, missingRow)), + zap.Strings("resource_examples", resourceExamples), + zap.Strings("entitlement_id_examples", entIDExamples), + ) + } + return nil +} + +// checkGrantEntitlementReferences is invariant I8: every distinct +// entitlement referenced by a grant must exist as an entitlement row. +// One seek plus one point probe per distinct entitlement — O(distinct +// entitlements), never O(grants). +// +// InsertResourceGrants pages are EXEMPT in every mode: that pattern has +// always produced grants whose entitlements have no row (the machinery +// inserts the grant's embedded resource; no listing ever returns an +// entitlement for it), and downstream consumers synthesize the +// entitlement from the grant. The insert-fact probe is the same one I3 +// uses, value-reading and reserved for dangling refs. +// +// Default mode DROPS the dangling grants (uplift reads grants BY +// entitlement, so rows under a missing entitlement are never even read +// platform-side) and warns once with aggregates; fail-fast mode +// hard-fails. Never ErrReplayIntegrity (see the header). +func (s *syncer) checkGrantEntitlementReferences(ctx context.Context) error { + if s.syncType != connectorstore.SyncTypeFull { + return nil + } + facts, ok := s.store.(dotc1z.IngestInvariantStore) + if !ok { + return nil // engine without the inspection surface + } + // Dangling entitlements cluster under shared resources; memoize the + // per-resource insert-fact probe (it reads row values). + insertFactByResource := map[string]bool{} + probe := &danglingTypeProbe{s: s} + danglingEnts, unsyncedType, missingRow := 0, 0, 0 + var droppedRows int64 + var entIDExamples []string + err := facts.ForEachDanglingGrantEntitlement(ctx, func(entitlementID, rt, rid string) error { + resKey := rt + "\x00" + rid + annotated, probed := insertFactByResource[resKey] + if !probed { + var err error + annotated, err = facts.GrantsForEntResourceCarryInsertFact(ctx, rt, rid) + if err != nil { + return fmt.Errorf("ingest invariant I8: probing grant annotations for %s/%s: %w", rt, rid, err) + } + insertFactByResource[resKey] = annotated + } + if annotated { + return nil // the established InsertResourceGrants shape + } + if s.failFastInvariants { + return fmt.Errorf( + "ingest invariant I8 violated: grants reference entitlement %q (resource %s/%s) but no entitlement row exists "+ + "(resource type synced: %t — false means a config gap, true a magic-id construction bug "+ + "or a grants scope replayed against a refreshed entitlement set)", + entitlementID, rt, rid, probe.typeExists(ctx, rt)) + } + n, err := facts.DeleteGrantsForEntitlement(ctx, entitlementID, rt, rid) + if err != nil { + return fmt.Errorf("ingest invariant I8: dropping grants for entitlement %q: %w", entitlementID, err) + } + danglingEnts++ + droppedRows += n + if probe.typeExists(ctx, rt) { + missingRow++ + } else { + unsyncedType++ + } + if len(entIDExamples) < maxDanglingIDExamples { + entIDExamples = append(entIDExamples, entitlementID) + } + return nil + }) + if err != nil { + return err + } + if droppedRows > 0 { + ctxzap.Extract(ctx).Warn("ingest invariant I8: DROPPED grants referencing entitlements with no entitlement row", + zap.Int64("dropped_grants", droppedRows), + zap.Int("dangling_entitlements", danglingEnts), + zap.Int("refs_into_unsynced_types", unsyncedType), + zap.Int("refs_missing_rows_in_synced_types", missingRow), + zap.String("attribution", danglingAttribution(unsyncedType, missingRow)), + zap.Strings("entitlement_id_examples", entIDExamples), + ) + } + return nil +} + +// checkGrantPrincipalReferences is invariant I9: every distinct +// principal referenced by a grant must exist as a resource row. Scans +// the by_principal index (one seek + probe per distinct principal); a +// pending deferred index build is forced first so synth-written grants +// are covered — that build is EndSync's own pass moved earlier, so the +// net cost is ~zero. +// +// Grants carrying ExternalResourceMatch* annotations are EXEMPT in +// every mode: unprocessed match carriers mean no external resource file +// was configured (the match op deletes carriers when it runs) — they +// are evidence of a config gap, not bad data, and dropping them would +// make a misconfigured deployment look clean. +// +// Default mode DROPS the dangling grants (uplift skips grants whose +// principal resolves no platform object) and warns once with +// aggregates; fail-fast mode hard-fails. Never ErrReplayIntegrity. +func (s *syncer) checkGrantPrincipalReferences(ctx context.Context) error { + if s.syncType != connectorstore.SyncTypeFull { + return nil + } + facts, ok := s.store.(dotc1z.IngestInvariantStore) + if !ok { + return nil // engine without the inspection surface + } + if err := facts.EnsureGrantIndexes(ctx); err != nil { + return fmt.Errorf("ingest invariant I9: ensuring grant indexes: %w", err) + } + probe := &danglingTypeProbe{s: s} + danglingPrincipals, unsyncedType, missingRow := 0, 0, 0 + var droppedRows, matchCarriersKept int64 + var principalExamples []string + // matchCarriersKept counts GRANTS uniformly: fully-annotated + // principals contribute their carrier-grant count, mixed principals + // contribute the per-grant skip count from the drop. + err := facts.ForEachDanglingGrantPrincipal(ctx, func(rt, rid string, matchAnnotatedOnly bool, carrierGrants int64) error { + if matchAnnotatedOnly { + matchCarriersKept += carrierGrants + return nil + } + if s.failFastInvariants { + return fmt.Errorf( + "ingest invariant I9 violated: grants reference principal %s/%s but no resource row exists "+ + "(resource type synced: %t — false means a config gap, true means the connector "+ + "emitted grants for a principal it never synced)", + rt, rid, probe.typeExists(ctx, rt)) + } + deleted, skipped, err := facts.DeleteGrantsForPrincipal(ctx, rt, rid) + if err != nil { + return fmt.Errorf("ingest invariant I9: dropping grants for principal %s/%s: %w", rt, rid, err) + } + danglingPrincipals++ + droppedRows += deleted + matchCarriersKept += skipped + if probe.typeExists(ctx, rt) { + missingRow++ + } else { + unsyncedType++ + } + if len(principalExamples) < maxDanglingIDExamples { + principalExamples = append(principalExamples, rt+"/"+rid) + } + return nil + }) + if err != nil { + return err + } + l := ctxzap.Extract(ctx) + if droppedRows > 0 { + l.Warn("ingest invariant I9: DROPPED grants referencing principals with no resource row", + zap.Int64("dropped_grants", droppedRows), + zap.Int("dangling_principals", danglingPrincipals), + zap.Int("refs_into_unsynced_types", unsyncedType), + zap.Int("refs_missing_rows_in_synced_types", missingRow), + zap.String("attribution", danglingAttribution(unsyncedType, missingRow)), + zap.Strings("principal_examples", principalExamples), + ) + } + if matchCarriersKept > 0 { + l.Warn("ingest invariant I9: kept unprocessed external-match carrier grants with dangling principals — was the external resource file configured?", + zap.Int64("match_carriers_kept", matchCarriersKept), + ) + } + return nil +} + // checkSourceCacheScopeConsistency is invariant I6: at the sealed sync's // quiesce point, every scope present in a by_source_scope index must have // a manifest entry. Post-processing only re-stamps rows with scopes that @@ -460,23 +786,31 @@ func (s *syncer) checkSourceCacheScopeConsistency(ctx context.Context) error { if len(orphans) == 0 { return nil } + // Classified as replay-integrity: orphaned scope state in the sealed + // output is exactly the inconsistency that poisons a future replay, + // and a cold re-run either produces a clean artifact (replay-path + // fault, self-healed) or reproduces the orphan and fails terminally + // (a put-path bug, correctly fatal). err = fmt.Errorf( - "ingest invariant I6 violated: scope index entries exist with no manifest entry (lost manifest write or stamp leak): %v", - orphans) - if s.strictIngestionInvariants { + "ingest invariant I6 violated: scope index entries exist with no manifest entry (lost manifest write or stamp leak): %v: %w", + orphans, ErrReplayIntegrity) + if s.failFastInvariants { return err } - ctxzap.Extract(ctx).Warn("ingest invariant I6 violated (lenient mode)", zap.Error(err)) + ctxzap.Extract(ctx).Warn("ingest invariant I6 violated (default mode)", zap.Error(err)) return nil } // runIngestionInvariants evaluates the post-collection (check-flavored) -// invariants: I5, I6, I4, I3, in that order (cheapest and most -// behavior-critical first). Runs after the action loop drains and before -// EndSync, so every ingestion path — stream, replay, resume re-runs, -// expansion, external-match processing — has finished writing. -// Idempotent: a resumed sync that re-reaches this point re-evaluates -// with the same verdict. +// invariants: I5, I6, I4, I7, I3, I8, I9, in that order — cheapest and +// most behavior-critical first, and drop CASCADES respected: I7 drops +// dangling entitlements, which orphans their grants, which I8 then +// catches and drops; I9 runs last over whatever grants survive. Runs +// after the action loop drains and before EndSync, so every ingestion +// path — stream, replay, resume re-runs, expansion, external-match +// processing — has finished writing. Idempotent: a resumed sync that +// re-reaches this point re-evaluates with the same verdict (drops +// already applied leave nothing dangling on re-run). func (s *syncer) runIngestionInvariants(ctx context.Context) error { if err := s.validateStoredExclusionGroups(ctx); err != nil { return err @@ -487,5 +821,14 @@ func (s *syncer) runIngestionInvariants(ctx context.Context) error { if err := s.checkChildScheduling(ctx); err != nil { return err } - return s.checkGrantResourceReferences(ctx) + if err := s.checkEntitlementResourceReferences(ctx); err != nil { + return err + } + if err := s.checkGrantResourceReferences(ctx); err != nil { + return err + } + if err := s.checkGrantEntitlementReferences(ctx); err != nil { + return err + } + return s.checkGrantPrincipalReferences(ctx) } diff --git a/pkg/sync/ingest_invariants_referential_test.go b/pkg/sync/ingest_invariants_referential_test.go new file mode 100644 index 000000000..c4fe51f04 --- /dev/null +++ b/pkg/sync/ingest_invariants_referential_test.go @@ -0,0 +1,334 @@ +package sync //nolint:revive,nolintlint // we can't change the package name for backwards compatibility + +// I7/I8/I9 referential-invariant tests: entitlements referencing +// vanished resources (I7), grants referencing vanished entitlements +// (I8), and grants referencing vanished principals (I9) — connector +// magic-id bugs and the holes type-granularity scopes open. All are +// connector-attributable: default mode DROPS the rows (platform uplift +// discards them anyway) with one aggregated warning, fail-fast mode fails, +// and none carry ErrReplayIntegrity (a cold retry is clean on a +// dishonest validator, so the sentinel would loop forever). + +import ( + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + reader_v2 "github.com/conductorone/baton-sdk/pb/c1/reader/v2" + "github.com/conductorone/baton-sdk/pkg/annotations" + "github.com/conductorone/baton-sdk/pkg/connectorstore" + "github.com/conductorone/baton-sdk/pkg/dotc1z" + "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" + "github.com/conductorone/baton-sdk/pkg/logging" + et "github.com/conductorone/baton-sdk/pkg/types/entitlement" + "github.com/conductorone/baton-sdk/pkg/types/grant" + rs "github.com/conductorone/baton-sdk/pkg/types/resource" +) + +func TestIngestInvariantI7DropsDanglingEntitlements(t *testing.T) { + ctx, err := logging.Init(t.Context()) + require.NoError(t, err) + tmpDir := t.TempDir() + + repo, err := rs.NewResource("Repo r1", equivRepoRT, "r1") + require.NoError(t, err) + ent := et.NewAssignmentEntitlement(repo, "admin") + + // The violating shape: an entitlement row whose resource has no row. + cur := newRepairTestStore(ctx, t, filepath.Join(tmpDir, "cur.c1z"), tmpDir) + _, err = cur.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + require.NoError(t, cur.PutResourceTypes(ctx, equivRepoRT)) + require.NoError(t, cur.PutEntitlements(ctx, ent)) + + s := &syncer{store: cur, syncType: connectorstore.SyncTypeFull} + + // Fail-fast: named failure, no sentinel, no drop. + s.failFastInvariants = true + err = s.checkEntitlementResourceReferences(ctx) + require.Error(t, err) + require.Contains(t, err.Error(), "ingest invariant I7") + require.NotErrorIs(t, err, ErrReplayIntegrity) + _, err = cur.GetEntitlement(ctx, reader_v2.EntitlementsReaderServiceGetEntitlementRequest_builder{ + EntitlementId: ent.GetId(), + }.Build()) + require.NoError(t, err, "fail-fast mode must not drop rows") + + // Default mode: the row is DROPPED, and the pass is idempotent. + s.failFastInvariants = false + require.NoError(t, s.checkEntitlementResourceReferences(ctx)) + _, err = cur.GetEntitlement(ctx, reader_v2.EntitlementsReaderServiceGetEntitlementRequest_builder{ + EntitlementId: ent.GetId(), + }.Build()) + require.Error(t, err, "default mode must drop the dangling entitlement row") + require.NoError(t, s.checkEntitlementResourceReferences(ctx)) + + // After the drop, fail-fast finds nothing. + s.failFastInvariants = true + require.NoError(t, s.checkEntitlementResourceReferences(ctx)) + + require.NoError(t, cur.Close(ctx)) +} + +func TestIngestInvariantI8DropsDanglingGrants(t *testing.T) { + ctx, err := logging.Init(t.Context()) + require.NoError(t, err) + tmpDir := t.TempDir() + + repo, err := rs.NewResource("Repo r1", equivRepoRT, "r1") + require.NoError(t, err) + ent := et.NewAssignmentEntitlement(repo, "admin") + principal := v2.Resource_builder{ + Id: v2.ResourceId_builder{ResourceType: "user", Resource: "alice"}.Build(), + }.Build() + g := grant.NewGrant(repo, "admin", principal.GetId()) + + // The violating shape: a grant row whose entitlement has no row. + cur := newRepairTestStore(ctx, t, filepath.Join(tmpDir, "cur.c1z"), tmpDir) + _, err = cur.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + require.NoError(t, cur.PutResourceTypes(ctx, equivRepoRT)) + require.NoError(t, cur.PutResources(ctx, repo)) + require.NoError(t, cur.PutGrants(ctx, g)) + + s := &syncer{store: cur, syncType: connectorstore.SyncTypeFull} + + // Fail-fast: named failure carrying the exact reconstructed id; no drop. + s.failFastInvariants = true + err = s.checkGrantEntitlementReferences(ctx) + require.Error(t, err) + require.Contains(t, err.Error(), "ingest invariant I8") + require.Contains(t, err.Error(), ent.GetId()) + require.NotErrorIs(t, err, ErrReplayIntegrity) + + // Default mode: the grant is DROPPED; the pass is idempotent; fail-fast then + // finds nothing. + s.failFastInvariants = false + require.NoError(t, s.checkGrantEntitlementReferences(ctx)) + _, err = cur.GetGrant(ctx, reader_v2.GrantsReaderServiceGetGrantRequest_builder{ + GrantId: g.GetId(), + }.Build()) + require.Error(t, err, "default mode must drop the dangling grant row") + require.NoError(t, s.checkGrantEntitlementReferences(ctx)) + s.failFastInvariants = true + require.NoError(t, s.checkGrantEntitlementReferences(ctx)) + + require.NoError(t, cur.Close(ctx)) +} + +func TestIngestInvariantI8ExemptsInsertResourceGrants(t *testing.T) { + ctx, err := logging.Init(t.Context()) + require.NoError(t, err) + tmpDir := t.TempDir() + + repo, err := rs.NewResource("Repo r1", equivRepoRT, "r1") + require.NoError(t, err) + g := repairTestGrant(t, repo) // carries InsertResourceGrants + + cur := newRepairTestStore(ctx, t, filepath.Join(tmpDir, "cur.c1z"), tmpDir) + _, err = cur.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + require.NoError(t, cur.PutResourceTypes(ctx, equivRepoRT)) + require.NoError(t, cur.PutResources(ctx, repo)) + require.NoError(t, cur.PutGrants(ctx, g)) + + s := &syncer{store: cur, syncType: connectorstore.SyncTypeFull} + + // Exempt in both modes: no failure, no drop. + s.failFastInvariants = true + require.NoError(t, s.checkGrantEntitlementReferences(ctx)) + s.failFastInvariants = false + require.NoError(t, s.checkGrantEntitlementReferences(ctx)) + _, err = cur.GetGrant(ctx, reader_v2.GrantsReaderServiceGetGrantRequest_builder{ + GrantId: g.GetId(), + }.Build()) + require.NoError(t, err, "InsertResourceGrants shape must never be dropped") + + require.NoError(t, cur.Close(ctx)) +} + +func TestIngestInvariantI9DropsDanglingPrincipalGrants(t *testing.T) { + ctx, err := logging.Init(t.Context()) + require.NoError(t, err) + tmpDir := t.TempDir() + + repo, err := rs.NewResource("Repo r1", equivRepoRT, "r1") + require.NoError(t, err) + ent := et.NewAssignmentEntitlement(repo, "admin") + ghost := v2.Resource_builder{ + Id: v2.ResourceId_builder{ResourceType: "user", Resource: "ghost"}.Build(), + }.Build() + g := grant.NewGrant(repo, "admin", ghost.GetId()) + + // The violating shape: resource + entitlement exist, principal not. + cur := newRepairTestStore(ctx, t, filepath.Join(tmpDir, "cur.c1z"), tmpDir) + _, err = cur.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + require.NoError(t, cur.PutResourceTypes(ctx, equivRepoRT)) + require.NoError(t, cur.PutResources(ctx, repo)) + require.NoError(t, cur.PutEntitlements(ctx, ent)) + require.NoError(t, cur.PutGrants(ctx, g)) + + s := &syncer{store: cur, syncType: connectorstore.SyncTypeFull} + + // Fail-fast: named failure, no drop. + s.failFastInvariants = true + err = s.checkGrantPrincipalReferences(ctx) + require.Error(t, err) + require.Contains(t, err.Error(), "ingest invariant I9") + require.Contains(t, err.Error(), "user/ghost") + require.NotErrorIs(t, err, ErrReplayIntegrity) + + // Default mode: the grant is DROPPED; the pass is idempotent; fail-fast then + // finds nothing. + s.failFastInvariants = false + require.NoError(t, s.checkGrantPrincipalReferences(ctx)) + _, err = cur.GetGrant(ctx, reader_v2.GrantsReaderServiceGetGrantRequest_builder{ + GrantId: g.GetId(), + }.Build()) + require.Error(t, err, "default mode must drop the dangling-principal grant") + require.NoError(t, s.checkGrantPrincipalReferences(ctx)) + s.failFastInvariants = true + require.NoError(t, s.checkGrantPrincipalReferences(ctx)) + + require.NoError(t, cur.Close(ctx)) +} + +func TestIngestInvariantI9ExemptsExternalMatchCarriers(t *testing.T) { + ctx, err := logging.Init(t.Context()) + require.NoError(t, err) + tmpDir := t.TempDir() + + repo, err := rs.NewResource("Repo r1", equivRepoRT, "r1") + require.NoError(t, err) + ent := et.NewAssignmentEntitlement(repo, "admin") + viewerEnt := et.NewAssignmentEntitlement(repo, "viewer") + carrier := v2.Resource_builder{ + Id: v2.ResourceId_builder{ResourceType: "external-user", Resource: "someone@example.com"}.Build(), + }.Build() + // Two carrier grants for the SAME dangling principal, so the + // carrier-grant count is distinguishable from the principal count. + newCarrierGrant := func(slug string) *v2.Grant { + g := grant.NewGrant(repo, slug, carrier.GetId()) + annos := annotations.Annotations(g.GetAnnotations()) + annos.Update(v2.ExternalResourceMatch_builder{Key: "email", Value: "someone@example.com"}.Build()) + g.SetAnnotations(annos) + return g + } + g := newCarrierGrant("admin") + g2 := newCarrierGrant("viewer") + + // The carrier shape: a match-annotated grant whose stub principal has + // no row because no external resource file was configured (the match + // op never ran to transform-and-delete it). + cur := newRepairTestStore(ctx, t, filepath.Join(tmpDir, "cur.c1z"), tmpDir) + _, err = cur.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + require.NoError(t, cur.PutResourceTypes(ctx, equivRepoRT)) + require.NoError(t, cur.PutResources(ctx, repo)) + require.NoError(t, cur.PutEntitlements(ctx, ent, viewerEnt)) + require.NoError(t, cur.PutGrants(ctx, g, g2)) + + s := &syncer{store: cur, syncType: connectorstore.SyncTypeFull} + + // Exempt in both modes: evidence of a config gap, not bad data. + s.failFastInvariants = true + require.NoError(t, s.checkGrantPrincipalReferences(ctx)) + s.failFastInvariants = false + require.NoError(t, s.checkGrantPrincipalReferences(ctx)) + for _, kept := range []string{g.GetId(), g2.GetId()} { + _, err = cur.GetGrant(ctx, reader_v2.GrantsReaderServiceGetGrantRequest_builder{ + GrantId: kept, + }.Build()) + require.NoError(t, err, "unprocessed external-match carriers must never be dropped") + } + + // The carrier count is per GRANT, not per principal: one dangling + // principal with two carrier grants must report carrierGrants == 2 + // (units consistent with the mixed-principal path's skip count). + facts, ok := cur.(dotc1z.IngestInvariantStore) + require.True(t, ok) + require.NoError(t, facts.EnsureGrantIndexes(ctx)) + visits := 0 + require.NoError(t, facts.ForEachDanglingGrantPrincipal(ctx, func(rt, rid string, matchOnly bool, carrierGrants int64) error { + visits++ + require.True(t, matchOnly) + require.Equal(t, int64(2), carrierGrants) + return nil + })) + require.Equal(t, 1, visits, "one dangling principal expected") + + require.NoError(t, cur.Close(ctx)) +} + +// TestDishonestValidatorStrandsGrantEntitlements is the end-to-end I8 +// scenario: a connector whose grants validator says "unchanged" (so the +// scope replays) while its entitlement listing was refreshed and dropped +// the entitlement. Fail-fast mode names it without the replay sentinel; +// default mode drops the stranded grants and the sync converges to what +// a cold sync would have produced. +func TestDishonestValidatorStrandsGrantEntitlements(t *testing.T) { + ctx, err := logging.Init(t.Context()) + require.NoError(t, err) + tmpDir := t.TempDir() + + mc := newSourceCacheMockConnector() + group, memberEnt, err := mc.AddGroup(ctx, "g1") + require.NoError(t, err) + user, err := mc.AddUser(ctx, "u1") + require.NoError(t, err) + stranded := mc.AddGroupMember(ctx, group, user) + mc.etagByResource[group.GetId().GetResource()] = "v1" + + sync1 := filepath.Join(tmpDir, "s1.c1z") + runSourceCacheSync(ctx, t, mc, sync1, "", tmpDir) + + // The dishonesty: the entitlement listing drops the member + // entitlement while the grants validator still reports "unchanged". + mc.entDB[group.GetId().GetResource()] = nil + + // Fail-fast warm sync: fails, named, not replay-classified. + store2, err := dotc1z.NewStore(ctx, filepath.Join(tmpDir, "s2.c1z"), + dotc1z.WithEngine(c1zstore.EnginePebble), + dotc1z.WithTmpDir(tmpDir), + ) + require.NoError(t, err) + warmStrict, err := NewSyncer(ctx, mc, + WithConnectorStore(store2), + WithTmpDir(tmpDir), + WithPreviousSyncC1ZPath(sync1), + WithFailFastInvariants(), + ) + require.NoError(t, err) + err = warmStrict.Sync(ctx) + require.Error(t, err) + require.Contains(t, err.Error(), "ingest invariant I8") + require.NotErrorIs(t, err, ErrReplayIntegrity, + "a dishonest validator must not trigger the cold-retry ladder: the cold run is clean, "+ + "so the retry would mask the connector bug on every sync forever") + require.NoError(t, warmStrict.Close(ctx)) + + // Default-mode warm sync: succeeds, and the stranded grants are DROPPED — + // the artifact converges to cold-sync semantics. + sync3 := filepath.Join(tmpDir, "s3.c1z") + store3, err := dotc1z.NewStore(ctx, sync3, + dotc1z.WithEngine(c1zstore.EnginePebble), + dotc1z.WithTmpDir(tmpDir), + ) + require.NoError(t, err) + warmLenient, err := NewSyncer(ctx, mc, + WithConnectorStore(store3), + WithTmpDir(tmpDir), + WithPreviousSyncC1ZPath(sync1), + ) + require.NoError(t, err) + require.NoError(t, warmLenient.Sync(ctx)) + require.NoError(t, warmLenient.Close(ctx)) + + grants3 := listGrantsInFile(ctx, t, sync3) + require.NotContains(t, grants3, stranded.GetId(), + "default mode must drop grants stranded by the dishonest validator") + _ = memberEnt +} diff --git a/pkg/sync/ingest_invariants_repair_test.go b/pkg/sync/ingest_invariants_repair_test.go index 1d175196a..8a0e06eb3 100644 --- a/pkg/sync/ingest_invariants_repair_test.go +++ b/pkg/sync/ingest_invariants_repair_test.go @@ -2,12 +2,13 @@ package sync //nolint:revive,nolintlint // we can't change the package name for // I3 repair-arm tests: a grant-inserted resource lost from the current // sync (the replay-loss class) is repaired by copying the previous -// sync's row in lenient mode, and hard-fails under ErrReplayIntegrity in -// strict mode (so mutation tests still name the machinery bug) or when +// sync's row in default mode, and hard-fails under ErrReplayIntegrity in +// fail-fast mode (so mutation tests still name the machinery bug) or when // no previous source can supply the row. import ( "context" + "fmt" "path/filepath" "testing" @@ -77,15 +78,15 @@ func TestIngestInvariantI3RepairsLostRelatedResource(t *testing.T) { s.sourceCache.prev = prevSC s.sourceCache.prevReader = prev - // Strict mode: no repair — the machinery bug must be NAMED, and the + // Fail-fast mode: no repair — the machinery bug must be NAMED, and the // failure classified as replay-integrity. - s.strictIngestionInvariants = true + s.failFastInvariants = true err = s.checkGrantResourceReferences(ctx) require.Error(t, err) require.ErrorIs(t, err, ErrReplayIntegrity) - // Lenient mode: repaired from the previous sync's row, full fidelity. - s.strictIngestionInvariants = false + // Default mode: repaired from the previous sync's row, full fidelity. + s.failFastInvariants = false require.NoError(t, s.checkGrantResourceReferences(ctx)) got, err := cur.GetResource(ctx, reader_v2.ResourcesReaderServiceGetResourceRequest_builder{ ResourceId: repo.GetId(), @@ -101,6 +102,55 @@ func TestIngestInvariantI3RepairsLostRelatedResource(t *testing.T) { require.NoError(t, prev.Close(ctx)) } +// TestReplayIntegrityErrorPropagatesFromSync pins the propagation the +// runners' cold-retry depends on: an ErrReplayIntegrity-classified +// failure raised at the replay seam must survive the action loop and +// parallel workers and satisfy errors.Is at syncer.Sync()'s return. +func TestReplayIntegrityErrorPropagatesFromSync(t *testing.T) { + ctx, err := logging.Init(t.Context()) + require.NoError(t, err) + tmpDir := t.TempDir() + + mc := newSourceCacheMockConnector() + res, _, err := mc.AddGroup(ctx, "g1") + require.NoError(t, err) + _, err = mc.AddUser(ctx, "u1") + require.NoError(t, err) + mc.etagByResource[res.GetId().GetResource()] = "v1" + + sync1 := filepath.Join(tmpDir, "s1.c1z") + runSourceCacheSync(ctx, t, mc, sync1, "", tmpDir) + + // Warm sync whose replay seam raises a classified failure — the same + // return path a real replay copy/count-check failure takes. + store, err := dotc1z.NewStore(ctx, filepath.Join(tmpDir, "s2.c1z"), + dotc1z.WithEngine(c1zstore.EnginePebble), + dotc1z.WithTmpDir(tmpDir), + ) + require.NoError(t, err) + syncer, err := NewSyncer(ctx, mc, + WithConnectorStore(store), + WithTmpDir(tmpDir), + WithPreviousSyncC1ZPath(sync1), + func(s *syncer) { + s.testSourceCacheHaltHook = func(stage, scopeKey string) error { + if stage != "replay-copied" { + return nil + } + return fmt.Errorf("injected replay-seam failure for scope %q: %w", scopeKey, ErrReplayIntegrity) + } + }, + ) + require.NoError(t, err) + err = syncer.Sync(ctx) + require.Error(t, err) + require.ErrorIs(t, err, ErrReplayIntegrity, + "the sentinel must survive propagation to Sync()'s return, or the runners' cold retry never fires") + cctx, cancel := context.WithCancel(ctx) + cancel() + _ = syncer.Close(cctx) +} + // TestWithoutPreviousSyncForcesCold pins the runners' cold-retry // mechanism: appending WithoutPreviousSync AFTER a previous-sync path // option must fully disable replay — the connector sees only lookup @@ -142,7 +192,7 @@ func TestIngestInvariantI3UnrepairableFailsWithReplayIntegrity(t *testing.T) { g := repairTestGrant(t, repo) // Previous sync exists but is ALSO missing the resource: repair has - // no source, so even lenient mode must fail — and classify as + // no source, so even default mode must fail — and classify as // replay-integrity so the runners' cold retry can take over. prev := newRepairTestStore(ctx, t, filepath.Join(tmpDir, "prev.c1z"), tmpDir) _, err = prev.StartNewSync(ctx, connectorstore.SyncTypeFull, "") diff --git a/pkg/sync/ingest_invariants_scale_test.go b/pkg/sync/ingest_invariants_scale_test.go new file mode 100644 index 000000000..856db9036 --- /dev/null +++ b/pkg/sync/ingest_invariants_scale_test.go @@ -0,0 +1,171 @@ +package sync //nolint:revive,nolintlint // we can't change the package name for backwards compatibility + +// Scale and inertness pins for the dangling-reference drops: +// +// - the drop paths must stream chunked batches (a large dangling +// population costs seconds, not one fsync per row); +// - the invariants must be COMPLETELY inert on SQLite stores and on +// partial syncs, in both modes. + +import ( + "fmt" + "path/filepath" + "testing" + "time" + + "github.com/stretchr/testify/require" + + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + reader_v2 "github.com/conductorone/baton-sdk/pb/c1/reader/v2" + "github.com/conductorone/baton-sdk/pkg/connectorstore" + "github.com/conductorone/baton-sdk/pkg/dotc1z" + "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" + "github.com/conductorone/baton-sdk/pkg/logging" + et "github.com/conductorone/baton-sdk/pkg/types/entitlement" + "github.com/conductorone/baton-sdk/pkg/types/grant" + rs "github.com/conductorone/baton-sdk/pkg/types/resource" +) + +// TestIngestInvariantI8DropScale drops 100k grants under one dangling +// entitlement. With the streaming chunked-batch implementation this is +// seconds; a per-row-fsync implementation would blow the test timeout +// (100k synced commits), so completion itself pins the batching. +func TestIngestInvariantI8DropScale(t *testing.T) { + if testing.Short() { + t.Skip("scale test") + } + ctx, err := logging.Init(t.Context()) + require.NoError(t, err) + tmpDir := t.TempDir() + + repo, err := rs.NewResource("Repo r1", equivRepoRT, "r1") + require.NoError(t, err) + + cur := newRepairTestStore(ctx, t, filepath.Join(tmpDir, "cur.c1z"), tmpDir) + _, err = cur.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + require.NoError(t, cur.PutResourceTypes(ctx, equivRepoRT)) + require.NoError(t, cur.PutResources(ctx, repo)) + + const grantCount = 100_000 + batch := make([]*v2.Grant, 0, 1000) + for i := 0; i < grantCount; i++ { + principal := v2.Resource_builder{ + Id: v2.ResourceId_builder{ResourceType: "user", Resource: fmt.Sprintf("u-%06d", i)}.Build(), + }.Build() + // All grants reference the same entitlement, which has no row. + batch = append(batch, grant.NewGrant(repo, "ghost", principal.GetId())) + if len(batch) == cap(batch) { + require.NoError(t, cur.PutGrants(ctx, batch...)) + batch = batch[:0] + } + } + require.NoError(t, cur.PutGrants(ctx, batch...)) + + s := &syncer{store: cur, syncType: connectorstore.SyncTypeFull} + start := time.Now() + require.NoError(t, s.checkGrantEntitlementReferences(ctx)) + elapsed := time.Since(start) + t.Logf("dropped %d dangling grants in %v", grantCount, elapsed) + + resp, err := cur.ListGrants(ctx, v2.GrantsServiceListGrantsRequest_builder{PageSize: 1}.Build()) + require.NoError(t, err) + require.Empty(t, resp.GetList(), "every dangling grant must be dropped") + + // Idempotent second pass, and fail-fast agrees nothing remains. + require.NoError(t, s.checkGrantEntitlementReferences(ctx)) + s.failFastInvariants = true + require.NoError(t, s.checkGrantEntitlementReferences(ctx)) + + require.NoError(t, cur.Close(ctx)) +} + +// TestReferentialInvariantsInertOnSQLite pins the SQLite gate: the +// referential invariants require IngestInvariantStore (Pebble-only), so on a +// SQLite store they are complete no-ops in both modes — no error, no +// drop — even with flagrantly dangling data present. +func TestReferentialInvariantsInertOnSQLite(t *testing.T) { + ctx, err := logging.Init(t.Context()) + require.NoError(t, err) + tmpDir := t.TempDir() + + store, err := dotc1z.NewStore(ctx, filepath.Join(tmpDir, "sqlite.c1z"), + dotc1z.WithEngine(c1zstore.EngineSQLite), + dotc1z.WithTmpDir(tmpDir), + ) + require.NoError(t, err) + _, ok := any(store).(dotc1z.IngestInvariantStore) + require.False(t, ok, "SQLite store must not implement IngestInvariantStore; if it grows the capability, revisit the drop gates") + + _, err = store.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + repo, err := rs.NewResource("Repo r1", equivRepoRT, "r1") + require.NoError(t, err) + require.NoError(t, store.PutResourceTypes(ctx, equivRepoRT)) + // Dangling everything: entitlement without its resource, grant + // without its entitlement or principal. + danglingEnt := et.NewAssignmentEntitlement(repo, "admin") + require.NoError(t, store.PutEntitlements(ctx, danglingEnt)) + ghost := v2.Resource_builder{ + Id: v2.ResourceId_builder{ResourceType: "user", Resource: "ghost"}.Build(), + }.Build() + g := grant.NewGrant(repo, "ghost-slug", ghost.GetId()) + require.NoError(t, store.PutGrants(ctx, g)) + + for _, failFast := range []bool{false, true} { + s := &syncer{store: store, syncType: connectorstore.SyncTypeFull, failFastInvariants: failFast} + require.NoError(t, s.checkEntitlementResourceReferences(ctx)) + require.NoError(t, s.checkGrantResourceReferences(ctx)) + require.NoError(t, s.checkGrantEntitlementReferences(ctx)) + require.NoError(t, s.checkGrantPrincipalReferences(ctx)) + } + + // Nothing was dropped. + _, err = store.GetEntitlement(ctx, reader_v2.EntitlementsReaderServiceGetEntitlementRequest_builder{ + EntitlementId: danglingEnt.GetId(), + }.Build()) + require.NoError(t, err) + _, err = store.GetGrant(ctx, reader_v2.GrantsReaderServiceGetGrantRequest_builder{ + GrantId: g.GetId(), + }.Build()) + require.NoError(t, err) + + require.NoError(t, store.Close(ctx)) +} + +// TestReferentialInvariantsInertOnPartialSync pins the sync-type gate: +// a partial sync's store legitimately lacks rows that live in the base +// full sync, so the referential invariants must not evaluate at all — +// no error even in fail-fast mode, no drops. +func TestReferentialInvariantsInertOnPartialSync(t *testing.T) { + ctx, err := logging.Init(t.Context()) + require.NoError(t, err) + tmpDir := t.TempDir() + + cur := newRepairTestStore(ctx, t, filepath.Join(tmpDir, "cur.c1z"), tmpDir) + _, err = cur.StartNewSync(ctx, connectorstore.SyncTypePartial, "") + require.NoError(t, err) + repo, err := rs.NewResource("Repo r1", equivRepoRT, "r1") + require.NoError(t, err) + require.NoError(t, cur.PutResourceTypes(ctx, equivRepoRT)) + ghost := v2.Resource_builder{ + Id: v2.ResourceId_builder{ResourceType: "user", Resource: "ghost"}.Build(), + }.Build() + g := grant.NewGrant(repo, "ghost-slug", ghost.GetId()) + require.NoError(t, cur.PutGrants(ctx, g)) + + for _, failFast := range []bool{false, true} { + s := &syncer{store: cur, syncType: connectorstore.SyncTypePartial, failFastInvariants: failFast} + require.NoError(t, s.checkEntitlementResourceReferences(ctx)) + require.NoError(t, s.checkGrantResourceReferences(ctx)) + require.NoError(t, s.checkGrantEntitlementReferences(ctx)) + require.NoError(t, s.checkGrantPrincipalReferences(ctx)) + } + + _, err = cur.GetGrant(ctx, reader_v2.GrantsReaderServiceGetGrantRequest_builder{ + GrantId: g.GetId(), + }.Build()) + require.NoError(t, err, "partial syncs must never drop") + + require.NoError(t, cur.Close(ctx)) +} diff --git a/pkg/sync/ingest_invariants_test.go b/pkg/sync/ingest_invariants_test.go index b57b5acf0..e8218047b 100644 --- a/pkg/sync/ingest_invariants_test.go +++ b/pkg/sync/ingest_invariants_test.go @@ -61,6 +61,8 @@ func TestIngestInvariantAnnotationCoverage(t *testing.T) { &v2.InsertResourceGrants{}, &v2.ChildResourceType{}, &v2.EntitlementExclusionGroup{}, + &v2.TypeScopedEntitlements{}, + &v2.TypeScopedGrants{}, } for _, msg := range sideEffectAnnotations { name := string(msg.ProtoReflect().Descriptor().FullName()) diff --git a/pkg/sync/source_cache.go b/pkg/sync/source_cache.go index 8adf20198..39f8413a9 100644 --- a/pkg/sync/source_cache.go +++ b/pkg/sync/source_cache.go @@ -381,7 +381,11 @@ func (s *syncer) executeScopeReplay(ctx context.Context, kind sourcecache.RowKin // produces nothing. _, entryFound, err := s.sourceCache.prev.LookupSourceCacheEntry(ctx, kind, page.scopeKey) if err != nil { - return fmt.Errorf("source cache: error reading previous manifest for scope %q: %w", page.scopeKey, err) + // The PREVIOUS file failing to serve reads mid-sync means the + // replay source itself is sick; retiring it and re-running cold + // (the runners' ErrReplayIntegrity handling) is the definitive + // remediation. + return fmt.Errorf("source cache: error reading previous manifest for scope %q: %w: %w", page.scopeKey, err, ErrReplayIntegrity) } if !entryFound { ctxzap.Extract(ctx).Warn("source cache: replay requested for scope with no previous manifest entry", diff --git a/pkg/sync/source_cache_equivalence_test.go b/pkg/sync/source_cache_equivalence_test.go index 919290bd9..10ec3e118 100644 --- a/pkg/sync/source_cache_equivalence_test.go +++ b/pkg/sync/source_cache_equivalence_test.go @@ -1792,7 +1792,7 @@ func runEquivSync(ctx context.Context, t *testing.T, cc types.ConnectorClient, p // Check-only ingestion invariants hard-fail in the harness: a // violation must name itself here before the differential // comparison would catch its downstream divergence. - WithStrictIngestionInvariants(), + WithFailFastInvariants(), } if prevPath != "" { opts = append(opts, WithPreviousSyncC1ZPath(prevPath)) diff --git a/pkg/sync/source_cache_sideeffects_test.go b/pkg/sync/source_cache_sideeffects_test.go index 32f373848..598f86c18 100644 --- a/pkg/sync/source_cache_sideeffects_test.go +++ b/pkg/sync/source_cache_sideeffects_test.go @@ -38,7 +38,7 @@ func runSourceCacheSyncClient(ctx context.Context, t *testing.T, cc types.Connec dotc1z.WithTmpDir(tmpDir), ) require.NoError(t, err) - opts := []SyncOpt{WithConnectorStore(store), WithTmpDir(tmpDir), WithStrictIngestionInvariants()} + opts := []SyncOpt{WithConnectorStore(store), WithTmpDir(tmpDir), WithFailFastInvariants()} if prevPath != "" { opts = append(opts, WithPreviousSyncC1ZPath(prevPath)) } diff --git a/pkg/sync/source_cache_test.go b/pkg/sync/source_cache_test.go index a58bf3970..4df760679 100644 --- a/pkg/sync/source_cache_test.go +++ b/pkg/sync/source_cache_test.go @@ -169,7 +169,7 @@ func runSourceCacheSync(ctx context.Context, t *testing.T, mc *sourceCacheMockCo dotc1z.WithTmpDir(tmpDir), ) require.NoError(t, err) - opts := []SyncOpt{WithConnectorStore(store), WithTmpDir(tmpDir), WithStrictIngestionInvariants()} + opts := []SyncOpt{WithConnectorStore(store), WithTmpDir(tmpDir), WithFailFastInvariants()} if prevPath != "" { opts = append(opts, WithPreviousSyncC1ZPath(prevPath)) } diff --git a/pkg/sync/syncer.go b/pkg/sync/syncer.go index 46ce36e8d..b455aa7d9 100644 --- a/pkg/sync/syncer.go +++ b/pkg/sync/syncer.go @@ -146,11 +146,12 @@ type syncer struct { // Ingestion-invariant state (see ingest_invariants.go): // childSchedule is the monotone record backing invariant I4; // resourcesPhaseRanHere gates I4 to processes that actually ran the - // resources phase; strictIngestionInvariants promotes I4 from - // warn to hard-fail (tests and the equivalence harness set it). - childSchedule childScheduleSet - resourcesPhaseRanHere bool - strictIngestionInvariants bool + // resources phase; failFastInvariants promotes every invariant + // verdict from its default handling (warn, drop, or repair) to hard + // sync failure (tests and the equivalence harness set it). + childSchedule childScheduleSet + resourcesPhaseRanHere bool + failFastInvariants bool // testSourceCacheHaltHook, when non-nil, fires at named seams of the // scope lifecycle (replay-copied, rows-committed, tombstones-applied, // manifest-written); returning an error fails the sync at exactly @@ -3400,13 +3401,13 @@ func WithStorageEngine(engine c1zstore.Engine) SyncOpt { } } -// WithStrictIngestionInvariants promotes check-only ingestion invariants +// WithFailFastInvariants promotes check-only ingestion invariants // (see ingest_invariants.go) from warn-with-log to hard sync failure. // Tests and the replay-equivalence harness enable it; production default -// is lenient until the predicates have a clean release behind them. -func WithStrictIngestionInvariants() SyncOpt { +// stays warn-and-drop until the predicates have a clean release behind them. +func WithFailFastInvariants() SyncOpt { return func(s *syncer) { - s.strictIngestionInvariants = true + s.failFastInvariants = true } } diff --git a/pkg/sync/type_scoped_grants_test.go b/pkg/sync/type_scoped_grants_test.go index 6f3c3930f..dcc7a920c 100644 --- a/pkg/sync/type_scoped_grants_test.go +++ b/pkg/sync/type_scoped_grants_test.go @@ -354,6 +354,23 @@ func (mc *chunkedGrantsMockConnector) ListGrants(ctx context.Context, in *v2.Gra }.Build(), nil } +// enableUnstampedExtra turns on the unstamped trailing page AND registers +// the "admin" entitlement its grants reference: ingest invariant I8 drops +// grants whose entitlement has no row at seal, so the fixture must stay +// referentially honest (as the real connector this scenario pins is). +func (mc *chunkedGrantsMockConnector) enableUnstampedExtra() { + mc.unstampedExtra = true + gid := mc.groupIDs[0] + for _, ent := range mc.entDB[gid] { + if ent.GetSlug() == "admin" { + return + } + } + adminEnt := et.NewAssignmentEntitlement(mc.resByID[gid], "admin", et.WithGrantableTo(userResourceType)) + adminEnt.SetSlug("admin") + mc.entDB[gid] = append(mc.entDB[gid], adminEnt) +} + // unstampedExtraGrants are the rows served by the unstamped trailing page: // "admin" grants on group-000 for two fixed users. They exist only while // the connector emits them — nothing replays them. @@ -422,7 +439,7 @@ func TestTypeScopedGrantsMixedStampedUnstampedCursor(t *testing.T) { // 10 groups => 1 chunk, 3 members each => 30 member grants, plus 2 // unstamped "admin" grants from the trailing extra page. mc := newChunkedGrantsMockConnector(t, 10, 3) - mc.unstampedExtra = true + mc.enableUnstampedExtra() require.Equal(t, 1, mc.chunkCount()) // --- Sync 1: cold — scoped member pages then the unstamped page ---- diff --git a/pkg/sync/whale_invariants_cost_test.go b/pkg/sync/whale_invariants_cost_test.go index 749ae6db4..0bb1ff5e0 100644 --- a/pkg/sync/whale_invariants_cost_test.go +++ b/pkg/sync/whale_invariants_cost_test.go @@ -13,6 +13,7 @@ import ( "time" v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + reader_v2 "github.com/conductorone/baton-sdk/pb/c1/reader/v2" "github.com/conductorone/baton-sdk/pkg/dotc1z" "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" "github.com/conductorone/baton-sdk/pkg/logging" @@ -41,7 +42,7 @@ func TestWhaleInvariantCost(t *testing.T) { require.NotNil(t, run) require.NoError(t, store.SetCurrentSync(ctx, run.ID)) - facts, ok := store.(dotc1z.IngestFactStore) + facts, ok := store.(dotc1z.IngestInvariantStore) require.True(t, ok) sc, isSC := store.(dotc1z.SourceCacheStore) @@ -103,6 +104,73 @@ func TestWhaleInvariantCost(t *testing.T) { t.Logf("I6 orphan scopes: %v (orphans=%d kinds)", i6, len(orphans)) } + // --- I7: entitlement→resource referential check, exactly as + // checkEntitlementResourceReferences performs it (skip-scan + point + // Get per distinct entitlement resource). + start = time.Now() + i7Distinct, i7Dangling := 0, 0 + require.NoError(t, facts.ForEachDistinctEntitlementResource(ctx, func(rt, rid string) error { + i7Distinct++ + exists, err := facts.HasResourceRecord(ctx, rt, rid) + if err != nil { + return err + } + if !exists { + i7Dangling++ + } + return nil + })) + i7 := time.Since(start) + t.Logf("I7 entitlement→resource: %v (distinct resources=%d dangling=%d)", i7, i7Distinct, i7Dangling) + + // --- I8: grant→entitlement referential check, exactly as + // checkGrantEntitlementReferences performs it (skip-scan at + // entitlement-identity granularity + point probe per distinct + // entitlement; the engine only surfaces dangling ones). + typeKnown := map[string]bool{} + typeExists := func(rt string) bool { + if v, ok := typeKnown[rt]; ok { + return v + } + _, err := store.GetResourceType(ctx, reader_v2.ResourceTypesReaderServiceGetResourceTypeRequest_builder{ + ResourceTypeId: rt, + }.Build()) + typeKnown[rt] = err == nil + return typeKnown[rt] + } + + start = time.Now() + i8Dangling, i8UnsyncedType := 0, 0 + require.NoError(t, facts.ForEachDanglingGrantEntitlement(ctx, func(entID, rt, rid string) error { + i8Dangling++ + if !typeExists(rt) { + i8UnsyncedType++ + } + return nil + })) + i8 := time.Since(start) + t.Logf("I8 grant→entitlement: %v (dangling=%d of which unsynced-type=%d)", i8, i8Dangling, i8UnsyncedType) + + // --- I9: grant→principal dangling scan over by_principal. On a + // sealed artifact the deferred build already ran, so + // EnsureGrantIndexes no-ops and the scan is one seek + probe per + // distinct principal. + start = time.Now() + i9Dangling, i9MatchOnly, i9UnsyncedType := 0, 0, 0 + require.NoError(t, facts.EnsureGrantIndexes(ctx)) + require.NoError(t, facts.ForEachDanglingGrantPrincipal(ctx, func(rt, rid string, matchOnly bool, _ int64) error { + i9Dangling++ + if matchOnly { + i9MatchOnly++ + } + if !typeExists(rt) { + i9UnsyncedType++ + } + return nil + })) + i9 := time.Since(start) + t.Logf("I9 grant→principal: %v (dangling principals=%d match-carrier-only=%d unsynced-type=%d)", i9, i9Dangling, i9MatchOnly, i9UnsyncedType) + // --- Contrast: a full O(grants) pass over the grant keyspace — what // I3 would cost if it were per-grant instead of per-distinct-resource. // Uses the raw listing read path (value decode per row, like any diff --git a/pkg/synccompactor/source_cache_replay_test.go b/pkg/synccompactor/source_cache_replay_test.go index 3200dfda5..a4bb74eac 100644 --- a/pkg/synccompactor/source_cache_replay_test.go +++ b/pkg/synccompactor/source_cache_replay_test.go @@ -39,6 +39,7 @@ import ( "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" pebbleengine "github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble" "github.com/conductorone/baton-sdk/pkg/sourcecache" + et "github.com/conductorone/baton-sdk/pkg/types/entitlement" gt "github.com/conductorone/baton-sdk/pkg/types/grant" rs "github.com/conductorone/baton-sdk/pkg/types/resource" ) @@ -66,6 +67,14 @@ func writeScopedSyncFile(ctx context.Context, t *testing.T, path, tmpDir, scope, require.NoError(t, err) require.NoError(t, store.PutResourceTypes(ctx, scGroupRT, scUserRT)) require.NoError(t, store.PutResources(ctx, group)) + // Referential honesty: the grants below reference the member + // entitlement, and compaction's expand-grants pass runs the + // ingestion invariants over the merged output — grants whose + // entitlement has no row would be dropped (I8), as they would be + // for a real connector with this bug. + memberEnt := et.NewAssignmentEntitlement(group, "member") + memberEnt.SetSlug("member") + require.NoError(t, store.PutEntitlements(ctx, memberEnt)) grants := make([]*v2.Grant, 0, len(memberIDs)) for _, uid := range memberIDs { From 7daaa9acf9a06fcff020572c6219df9cd5145348 Mon Sep 17 00:00:00 2001 From: Matt Kaniaris Date: Wed, 15 Jul 2026 14:13:45 -0600 Subject: [PATCH 10/17] more fixes --- pkg/connectorbuilder/resource_syncer.go | 10 ++++ pkg/connectorbuilder/span_error_test.go | 58 +++++++++++++++++------ pkg/dotc1z/engine/pebble/ingest_repair.go | 3 +- pkg/dotc1z/source_cache.go | 3 +- 4 files changed, 58 insertions(+), 16 deletions(-) diff --git a/pkg/connectorbuilder/resource_syncer.go b/pkg/connectorbuilder/resource_syncer.go index bcaf3d166..4f0ff6e6b 100644 --- a/pkg/connectorbuilder/resource_syncer.go +++ b/pkg/connectorbuilder/resource_syncer.go @@ -302,6 +302,10 @@ func (b *builder) ListResources(ctx context.Context, request *v2.ResourcesServic } if deferred { b.m.RecordTaskSuccess(ctx, tt, b.nowFunc().Sub(start)) + // A deferral is a protocol turn, not a failure: clear the + // handler's ErrLookupDeferred so the span defer above doesn't + // record every normal ask/answer bounce as a trace error. + err = nil // Session usage rides the ask too: the phase-1 handler may have // touched the session before deferring, and the protocol turn is // the only response that work will ever produce. @@ -486,6 +490,9 @@ func (b *builder) ListEntitlements(ctx context.Context, request *v2.Entitlements } if deferred { b.m.RecordTaskSuccess(ctx, tt, b.nowFunc().Sub(start)) + // Protocol turn, not a failure — clear ErrLookupDeferred for the + // span defer; see the ListResources deferred path. + err = nil // Session usage rides the ask; see the ListResources deferred path. return v2.EntitlementsServiceListEntitlementsResponse_builder{Annotations: appendSessionUsage(askAnnos, sessionUsage)}.Build(), nil } @@ -580,6 +587,9 @@ func (b *builder) ListGrants(ctx context.Context, request *v2.GrantsServiceListG } if deferred { b.m.RecordTaskSuccess(ctx, tt, b.nowFunc().Sub(start)) + // Protocol turn, not a failure — clear ErrLookupDeferred for the + // span defer; see the ListResources deferred path. + err = nil // Session usage rides the ask; see the ListResources deferred path. return v2.GrantsServiceListGrantsResponse_builder{Annotations: appendSessionUsage(askAnnos, sessionUsage)}.Build(), nil } diff --git a/pkg/connectorbuilder/span_error_test.go b/pkg/connectorbuilder/span_error_test.go index 4a01e1e38..6a55ebe9b 100644 --- a/pkg/connectorbuilder/span_error_test.go +++ b/pkg/connectorbuilder/span_error_test.go @@ -6,6 +6,7 @@ import ( "testing" v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + "github.com/conductorone/baton-sdk/pkg/annotations" "github.com/stretchr/testify/require" "go.opentelemetry.io/otel" otelcodes "go.opentelemetry.io/otel/codes" @@ -40,26 +41,30 @@ func (r *recordingExporter) spansByName(name string) []sdktrace.ReadOnlySpan { return out } -// installSpanRecorder swaps the global otel tracer provider for an in-memory -// recorder for the duration of the test. The package-level tracer var in -// connectorbuilder.go is captured at init via otel.Tracer(name), which -// returns a delegating wrapper that lazily routes through the current global -// provider — so this swap reaches the existing tracer references. +// installSpanRecorder installs an in-memory span recorder as the global +// otel tracer provider, ONCE per test process, and returns it. A singleton +// because otel's global delegation is one-shot: the package-level tracer in +// connectorbuilder.go (captured at init via otel.Tracer) wires itself to +// the FIRST SetTracerProvider and never re-delegates — a second install +// would silently record nothing, and restoring the previous provider on +// cleanup would shut the only wired exporter down for every later test. // -// Mutates global state; callers must not call t.Parallel(). +// The recorder accumulates spans across tests; assert on the LAST span of +// a name. Mutates global state; callers must not call t.Parallel(). func installSpanRecorder(t *testing.T) *recordingExporter { t.Helper() - exp := &recordingExporter{} - tp := sdktrace.NewTracerProvider(sdktrace.WithSyncer(exp)) - prev := otel.GetTracerProvider() - otel.SetTracerProvider(tp) - t.Cleanup(func() { - otel.SetTracerProvider(prev) - _ = tp.Shutdown(context.Background()) + spanRecorderOnce.Do(func() { + spanRecorderExp = &recordingExporter{} + otel.SetTracerProvider(sdktrace.NewTracerProvider(sdktrace.WithSyncer(spanRecorderExp))) }) - return exp + return spanRecorderExp } +var ( + spanRecorderOnce sync.Once + spanRecorderExp *recordingExporter +) + // TestSpanErrorRecorded_EarlyReturns is the regression test for OPS-1543. // Pre-fix, ~30 early-return error paths in pkg/connectorbuilder shadowed // the function-scope err that the deferred uotel.EndSpanWithError reads, @@ -69,6 +74,31 @@ func installSpanRecorder(t *testing.T) *recordingExporter { // one representative path per bug-pattern class: err := inside if-block // with InvalidArgument, err := inside if-block with NotFound, and direct // return (no outer assignment) with NotFound. +// TestSpanOKOnDeferredProtocolTurn is the inverse regression: a deferred +// source-cache lookup (ask/answer bounce) is a PROTOCOL TURN — the RPC +// succeeds — but the handler's ErrLookupDeferred used to linger in the +// function-scoped err that the deferred EndSpanWithError reads, marking +// every normal bounce as a trace error. The span must end status:ok. +func TestSpanOKOnDeferredProtocolTurn(t *testing.T) { + ctx := context.Background() + exp := installSpanRecorder(t) + + ts := &continuationTestSyncer{testResourceSyncerV2Simple: testResourceSyncerV2Simple{resourceType: "group"}, scope: "groups/g1/members"} + b := newContinuationTestBuilder(t, ts) + + offer := annotations.New(&v2.SourceCacheLookupOffer{}) + resp, err := b.ListGrants(ctx, continuationGrantsRequest(offer)) + require.NoError(t, err, "a deferral is a protocol turn, not a failure") + respAnnos := annotations.Annotations(resp.GetAnnotations()) + require.True(t, respAnnos.Contains(&v2.SourceCacheLookupAsk{}), "sanity: this must be the ask turn") + + spans := exp.spansByName("builder.ListGrants") + require.NotEmpty(t, spans) + s := spans[len(spans)-1] + require.NotEqual(t, otelcodes.Error, s.Status().Code, + "a deferred ask/answer turn must not record a span error; ErrLookupDeferred must be cleared before the deferred EndSpanWithError reads err") +} + func TestSpanErrorRecorded_EarlyReturns(t *testing.T) { ctx := context.Background() exp := installSpanRecorder(t) diff --git a/pkg/dotc1z/engine/pebble/ingest_repair.go b/pkg/dotc1z/engine/pebble/ingest_repair.go index f17a02ca0..45b102211 100644 --- a/pkg/dotc1z/engine/pebble/ingest_repair.go +++ b/pkg/dotc1z/engine/pebble/ingest_repair.go @@ -366,7 +366,8 @@ func (e *Engine) DeleteGrantsForEntitlement(ctx context.Context, entitlementID, // DeleteEntitlementsForResource deletes every entitlement row under one // resource — the drop arm for entitlements referencing a resource with no // row. Returns the count and up to maxIDs deleted external ids for the -// caller's aggregate report. +// caller's aggregate report; maxIDs <= 0 collects none (callers pass a +// shrinking example budget, so the guard below doubles as the clamp). func (e *Engine) DeleteEntitlementsForResource(ctx context.Context, resourceTypeID, resourceID string, maxIDs int) (int64, []string, error) { prefix := encodeEntitlementPrimaryResourcePrefix(resourceTypeID, resourceID) type entRow struct { diff --git a/pkg/dotc1z/source_cache.go b/pkg/dotc1z/source_cache.go index 961177423..5afca6cbb 100644 --- a/pkg/dotc1z/source_cache.go +++ b/pkg/dotc1z/source_cache.go @@ -143,7 +143,8 @@ type IngestInvariantStore interface { ForEachDanglingGrantPrincipal(ctx context.Context, visit func(principalRT, principalID string, matchAnnotatedOnly bool, carrierGrants int64) error) error // DeleteEntitlementsForResource drops every entitlement row under a - // resource; returns the count and up to maxIDs deleted ids. + // resource; returns the count and up to maxIDs deleted ids + // (maxIDs <= 0 collects none — callers pass a shrinking budget). DeleteEntitlementsForResource(ctx context.Context, resourceTypeID, resourceID string, maxIDs int) (int64, []string, error) // DeleteGrantsForEntitlement drops every grant row under one From 71ac77def8f3c4d4d89821ff28d01c08d8c54fb2 Mon Sep 17 00:00:00 2001 From: Matt Kaniaris Date: Wed, 15 Jul 2026 15:46:22 -0600 Subject: [PATCH 11/17] review fixes and test oracles for replay hardening MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Replay: scope pre-clear makes resumed copies idempotent (count check stays exact); lookup slot is sync-aware so concurrent syncs can't cross-wire validators; compaction artifacts from pre-flag binaries are gated by their provenance token; I6 is gated off for compacted runs. Drops: I8/I9 exemptions judged per grant (machinery rows can't shield connector rows); deletes stream chunked batches instead of per-row fsyncs; IngestFactStore renamed IngestInvariantStore to match what it now does. Local runner keeps the user-supplied previous-sync file and names it in an actionable error instead of deleting it. Tests: warning-silence oracle in the equivalence harness (invariant warnings are verdicts nobody asserts on — now they fail clean scenarios); mixed-population exemption fixtures; replay-resume, mixed-version, and drop-scale tests; deferred ask/answer turns no longer mark trace spans as errors. --- pkg/dotc1z/engine/pebble/ingest_facts.go | 36 ++++ pkg/dotc1z/engine/pebble/ingest_repair.go | 27 ++- pkg/dotc1z/engine/pebble/source_cache.go | 196 +++++++++++++++++- .../pebble/source_cache_replay_resume_test.go | 167 +++++++++++++++ pkg/dotc1z/source_cache.go | 21 +- pkg/sourcecache/grpc_server.go | 57 +++-- pkg/sourcecache/grpc_server_test.go | 106 ++++++++++ pkg/sourcecache/sourcecache.go | 6 +- pkg/sync/ingest_invariants.go | 59 +++--- .../ingest_invariants_granularity_test.go | 170 +++++++++++++++ pkg/sync/ingest_invariants_i9_mixed_test.go | 96 +++++++++ pkg/sync/source_cache.go | 44 +++- pkg/sync/source_cache_equivalence_test.go | 71 +++++++ pkg/sync/source_cache_mixed_version_test.go | 90 ++++++++ pkg/tasks/local/syncer.go | 10 +- 15 files changed, 1099 insertions(+), 57 deletions(-) create mode 100644 pkg/dotc1z/engine/pebble/source_cache_replay_resume_test.go create mode 100644 pkg/sourcecache/grpc_server_test.go create mode 100644 pkg/sync/ingest_invariants_granularity_test.go create mode 100644 pkg/sync/ingest_invariants_i9_mixed_test.go create mode 100644 pkg/sync/source_cache_mixed_version_test.go diff --git a/pkg/dotc1z/engine/pebble/ingest_facts.go b/pkg/dotc1z/engine/pebble/ingest_facts.go index e813cfb5d..28c00630e 100644 --- a/pkg/dotc1z/engine/pebble/ingest_facts.go +++ b/pkg/dotc1z/engine/pebble/ingest_facts.go @@ -279,6 +279,42 @@ func (e *Engine) hasEntitlementIdentity(id entitlementIdentity) (bool, error) { return true, nil } +// GrantsForEntitlementAllCarryInsertFact reports whether EVERY grant +// under the given entitlement identity carries the InsertResourceGrants +// annotation — the fail-fast probe for the per-grant I8 exemption: a +// dangling entitlement whose grants all carry the fact is the +// machinery-owned shape (exempt); one non-carrying grant makes it a +// connector-owned dangling reference. Vacuously true with no grants +// (never the case for a dangling entitlement, which is only visited +// because grants reference it). Reads row values, so it is reserved for +// DANGLING referential probes — rare to zero on healthy syncs. +func (e *Engine) GrantsForEntitlementAllCarryInsertFact(ctx context.Context, entitlementID, entResourceTypeID, entResourceID string) (bool, error) { + entID := entitlementIdentityFromParts(entResourceTypeID, entResourceID, entitlementID) + prefix := encodeGrantPrimaryEntitlementIdentityPrefix( + entID.resourceTypeID, entID.resourceID, entID.flagComponent(), entID.tail) + iter, err := e.db.NewIter(&pebble.IterOptions{ + LowerBound: prefix, + UpperBound: upperBoundOf(prefix), + }) + if err != nil { + return false, err + } + defer iter.Close() + for iter.First(); iter.Valid(); iter.Next() { + if err := ctx.Err(); err != nil { + return false, err + } + flags, err := scanGrantSourceCacheFlagsRaw(iter.Value()) + if err != nil { + return false, err + } + if !flags.insertResourceGrants { + return false, nil + } + } + return true, iter.Error() +} + // GrantsForEntResourceCarryInsertFact reports whether any grant under the // given entitlement resource carries the InsertResourceGrants annotation. // Reads row values, so it is reserved for DANGLING referential probes — diff --git a/pkg/dotc1z/engine/pebble/ingest_repair.go b/pkg/dotc1z/engine/pebble/ingest_repair.go index 45b102211..dffbb72f6 100644 --- a/pkg/dotc1z/engine/pebble/ingest_repair.go +++ b/pkg/dotc1z/engine/pebble/ingest_repair.go @@ -303,16 +303,21 @@ func (e *Engine) DeleteGrantsForPrincipal(ctx context.Context, principalRT, prin } // DeleteGrantsForEntitlement deletes every grant row under one -// entitlement identity — the drop arm for grants referencing an -// entitlement with no row. Streams the primary prefix with values in +// entitlement identity EXCEPT rows carrying the InsertResourceGrants +// annotation — the drop arm for grants referencing an entitlement with no +// row. The exemption is per GRANT, matching the ownership rule +// (docs/tasks/dangling-reference-drops.md): a machinery-owned IRG grant +// legitimately has no entitlement row (downstream synthesizes it), while +// a connector-owned grant under the same missing entitlement is a +// dangling reference and drops. Streams the primary prefix with values in // hand (index cleanup derives from the value), chunked batches, no -// per-row commits. -func (e *Engine) DeleteGrantsForEntitlement(ctx context.Context, entitlementID, entResourceTypeID, entResourceID string) (int64, error) { +// per-row commits. Returns (deleted, skippedInsertFact). +func (e *Engine) DeleteGrantsForEntitlement(ctx context.Context, entitlementID, entResourceTypeID, entResourceID string) (int64, int64, error) { entID := entitlementIdentityFromParts(entResourceTypeID, entResourceID, entitlementID) prefix := encodeGrantPrimaryEntitlementIdentityPrefix( entID.resourceTypeID, entID.resourceID, entID.flagComponent(), entID.tail) - var deleted int64 + var deleted, skipped int64 err := e.withWrite(func() error { iter, err := e.db.NewIter(&pebble.IterOptions{ LowerBound: prefix, @@ -332,6 +337,14 @@ func (e *Engine) DeleteGrantsForEntitlement(ctx context.Context, entitlementID, if err := ctx.Err(); err != nil { return err } + flags, flagsErr := scanGrantSourceCacheFlagsRaw(iter.Value()) + if flagsErr != nil { + return fmt.Errorf("dangling entitlement drop: scan flags: %w", flagsErr) + } + if flags.insertResourceGrants { + skipped++ + continue + } if err := e.deleteGrantIndexesRaw(batch, "", iter.Value()); err != nil { return err } @@ -358,9 +371,9 @@ func (e *Engine) DeleteGrantsForEntitlement(ctx context.Context, entitlementID, return batch.Commit(opts) }) if err != nil { - return 0, err + return 0, 0, err } - return deleted, nil + return deleted, skipped, nil } // DeleteEntitlementsForResource deletes every entitlement row under one diff --git a/pkg/dotc1z/engine/pebble/source_cache.go b/pkg/dotc1z/engine/pebble/source_cache.go index 5fb3bbf10..0d08d6099 100644 --- a/pkg/dotc1z/engine/pebble/source_cache.go +++ b/pkg/dotc1z/engine/pebble/source_cache.go @@ -74,6 +74,12 @@ type SourceCacheReplayResult struct { // recreate them or the current sync holds grants whose resources // do not exist. RelatedResourceRows int64 + // ResumedRowsCleared counts destination rows deleted before the copy + // because they were already stamped with the scope — evidence of a + // resumed re-run over a crashed attempt's committed replay/overlay + // rows (see clearReplayScopeRowsLocked). Always zero on a page's + // first run. + ResumedRowsCleared int64 } // SourceCacheManifestSnapshot returns every manifest entry as @@ -177,11 +183,13 @@ func (e *Engine) countSourceScopeIndexRange(ctx context.Context, prefix []byte) // verifyReplayedScopeCount is the replay-time count check: immediately // after a replay's final commit, the destination's scope-index range must // hold exactly the rows the copy claims to have written. Sound because -// the copy is the scope's first writer in the sync (fresh rows for a -// replayed scope arrive only via the overlay, after this) and a resumed -// re-copy overwrites the identical identity set. A mismatch means a -// partial copy landed — the silent-bad-data class — so it fails the -// replay rather than letting the manifest entry seal it as complete. +// the copy is the scope's first writer in the sync — fresh rows for a +// replayed scope arrive only via the overlay, after this, and a resumed +// re-run first clears whatever a crashed attempt committed under the +// scope (clearReplayScopeRowsLocked), including overlay rows whose +// identities the re-copy would not cover. A mismatch means a partial +// copy landed — the silent-bad-data class — so it fails the replay +// rather than letting the manifest entry seal it as complete. func (e *Engine) verifyReplayedScopeCount(ctx context.Context, kind, scopeKey string, prefix []byte, copied int64) error { got, err := e.countSourceScopeIndexRange(ctx, prefix) if err != nil { @@ -678,6 +686,127 @@ func replayPrimaryFromIndexKey(indexKey, indexPrefix []byte, primaryHeader [2]by return append(key, tail...), nil } +// clearReplayScopeRowsLocked deletes every row in the CURRENT store whose +// by_source_scope index entry lives under scopePrefix, restoring the +// replay copy's "first writer for the scope" precondition when a resumed +// action re-runs a replay page. +// +// Why this must exist: a crashed earlier attempt of the same page may +// have committed the page's own overlay rows — net-new identities stamped +// with this scope — AFTER its replay copy landed. Those identities are +// not in the previous file, so a re-run's copy neither overwrites nor +// removes them, and verifyReplayedScopeCount's post-copy recount would +// find copied-base + overlay entries and fail the replay even though no +// data was lost. Deleting up front is safe because the resumed page +// re-delivers everything the crashed attempt wrote: the copy re-copies +// the base and the page's own rows are re-put after the copy. +// +// Rows whose VALUE stamp names a different scope are left in place (they +// belong to that other scope); only their orphaned index entry under this +// scope is removed. On the first run of a page the scope's index range is +// empty and this is a single bounded no-op scan. +// +// stampOf extracts the row's source-scope stamp from its raw value; +// deleteRowIndexes stages the row's secondary-index deletes derived from +// its raw value (including its scope entry, when the kind has more +// indexes than the scope entry itself). The caller must hold the write +// lock (this runs inside the replay's withWrite closure). +func (e *Engine) clearReplayScopeRowsLocked( + ctx context.Context, + scopeKey string, + scopePrefix []byte, + primaryHeader [2]byte, + stampOf func(val []byte) (string, error), + deleteRowIndexes func(batch *pebble.Batch, priKey []byte, val []byte) error, +) (int64, error) { + iter, err := e.db.NewIter(&pebble.IterOptions{ + LowerBound: scopePrefix, + UpperBound: upperBoundOf(scopePrefix), + }) + if err != nil { + return 0, err + } + defer iter.Close() + + opts := writeOpts(e.opts.durability) + if e.IsFreshSync() { + opts = pebble.NoSync + } + batch := e.db.NewBatch() + defer func() { _ = batch.Close() }() + + var deleted int64 + stagedInBatch := 0 + for iter.First(); iter.Valid(); iter.Next() { + if err := ctx.Err(); err != nil { + return deleted, err + } + priKey, err := replayPrimaryFromIndexKey(iter.Key(), scopePrefix, primaryHeader) + if err != nil { + return deleted, err + } + val, closer, getErr := e.db.Get(priKey) + switch { + case errors.Is(getErr, pebble.ErrNotFound): + // Orphan index entry with no row: drop the entry alone. + if err := batch.Delete(iter.Key(), nil); err != nil { + return deleted, err + } + stagedInBatch++ + case getErr != nil: + return deleted, fmt.Errorf("source cache replay: pre-clear get: %w", getErr) + default: + stamp, stampErr := stampOf(val) + if stampErr != nil { + closer.Close() + return deleted, stampErr + } + if stamp != scopeKey { + // The row was rewritten under another scope; this index + // entry is stale. Remove the entry, keep the row. + closer.Close() + if err := batch.Delete(iter.Key(), nil); err != nil { + return deleted, err + } + stagedInBatch++ + break + } + if err := deleteRowIndexes(batch, priKey, val); err != nil { + closer.Close() + return deleted, err + } + closer.Close() + if err := batch.Delete(priKey, nil); err != nil { + return deleted, err + } + // deleteRowIndexes covers the scope entry via the value's + // stamp; delete the iterated key too so the range is clean + // even if the derivations ever disagree (deleting a missing + // key is a pebble no-op). + if err := batch.Delete(iter.Key(), nil); err != nil { + return deleted, err + } + deleted++ + stagedInBatch++ + } + if stagedInBatch >= replayBatchRows { + if err := batch.Commit(opts); err != nil { + return deleted, err + } + _ = batch.Close() + batch = e.db.NewBatch() + stagedInBatch = 0 + } + } + if err := iter.Error(); err != nil { + return deleted, err + } + if err := batch.Commit(opts); err != nil { + return deleted, err + } + return deleted, nil +} + // ReplaySourceCacheGrants copies every grant stamped with scopeKey from // prev into the receiver: raw primary copy plus index synthesis from the // raw value (principal, needs_expansion, source-scope families). Mirrors @@ -692,6 +821,24 @@ func (e *Engine) ReplaySourceCacheGrants(ctx context.Context, prev *Engine, scop if err := e.requireCurrentSync(); err != nil { return err } + // Resume idempotency: drop any destination rows a crashed earlier + // attempt already committed under this scope (replay copy and/or + // overlay rows), so the copy below is the scope's first writer + // again and the post-copy recount stays exact. + cleared, err := e.clearReplayScopeRowsLocked(ctx, scopeKey, prefix, primaryHeader, + func(val []byte) (string, error) { + _, _, _, _, _, _, srcScope, scanErr := scanGrantIndexFieldsRaw(val) + return srcScope, scanErr + }, + func(batch *pebble.Batch, priKey []byte, val []byte) error { + return e.deleteGrantIndexesRaw(batch, "", val) + }, + ) + if err != nil { + return fmt.Errorf("source cache replay: pre-clear grants scope %q: %w", scopeKey, err) + } + res.ResumedRowsCleared = cleared + iter, err := prev.db.NewIter(&pebble.IterOptions{ LowerBound: prefix, UpperBound: upperBoundOf(prefix), @@ -986,6 +1133,22 @@ func (e *Engine) ReplaySourceCacheEntitlements(ctx context.Context, prev *Engine if err := e.requireCurrentSync(); err != nil { return err } + // Resume idempotency: see the grants replay. The entitlement + // bare-id lookup map is invalidated for the deletes exactly as + // for the copies (the defer below keys on either landing). + cleared, err := e.clearReplayScopeRowsLocked(ctx, scopeKey, prefix, primaryHeader, + scanEntitlementSourceScopeRaw, + func(batch *pebble.Batch, priKey []byte, val []byte) error { + // The scope entry is the only entitlement secondary + // index, and the caller deletes the iterated entry. + return nil + }, + ) + if err != nil { + return fmt.Errorf("source cache replay: pre-clear entitlements scope %q: %w", scopeKey, err) + } + res.ResumedRowsCleared = cleared + iter, err := prev.db.NewIter(&pebble.IterOptions{ LowerBound: prefix, UpperBound: upperBoundOf(prefix), @@ -1009,9 +1172,11 @@ func (e *Engine) ReplaySourceCacheEntitlements(ctx context.Context, prev *Engine // invalidated and the empty-keyspace fast path disarmed even when // the replay itself fails — draining workers can still resolve // tombstones and write entitlements while the failure unwinds. + // Pre-clear deletes mutate the keyspace the same way, so they + // arm the invalidation too. committedRows := 0 defer func() { - if committedRows > 0 { + if committedRows > 0 || res.ResumedRowsCleared > 0 { e.noteEntitlementKeyspaceWrite() _ = e.takeFreshEntitlementsEmpty() } @@ -1129,6 +1294,25 @@ func (e *Engine) ReplaySourceCacheResources(ctx context.Context, prev *Engine, s if err := e.requireCurrentSync(); err != nil { return err } + // Resume idempotency: see the grants replay. + cleared, err := e.clearReplayScopeRowsLocked(ctx, scopeKey, prefix, primaryHeader, + func(val []byte) (string, error) { + _, _, srcScope, scanErr := scanResourceIndexFieldsRaw(val) + return srcScope, scanErr + }, + func(batch *pebble.Batch, priKey []byte, val []byte) error { + rt, rid, decodeErr := decodeResourcePrimaryTail(priKey) + if decodeErr != nil { + return decodeErr + } + return e.deleteResourceIndexesRaw(batch, rt, rid, val) + }, + ) + if err != nil { + return fmt.Errorf("source cache replay: pre-clear resources scope %q: %w", scopeKey, err) + } + res.ResumedRowsCleared = cleared + iter, err := prev.db.NewIter(&pebble.IterOptions{ LowerBound: prefix, UpperBound: upperBoundOf(prefix), diff --git a/pkg/dotc1z/engine/pebble/source_cache_replay_resume_test.go b/pkg/dotc1z/engine/pebble/source_cache_replay_resume_test.go new file mode 100644 index 000000000..76a99c867 --- /dev/null +++ b/pkg/dotc1z/engine/pebble/source_cache_replay_resume_test.go @@ -0,0 +1,167 @@ +package pebble + +// Resumed-replay regression tests: a crashed sync attempt can commit a +// replay page's OWN overlay rows (net-new identities stamped with the +// scope) after the replay copy lands and before the action finishes. The +// resumed action re-runs the same page, and the re-run's replay must +// converge — clear the earlier attempt's rows, re-copy the base, and pass +// its post-copy recount — instead of false-failing verifyReplayedScopeCount +// with ErrReplayIntegrity (which would burn the whole warm sync on a cold +// retry). + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + v3 "github.com/conductorone/baton-sdk/pb/c1/storage/v3" + "github.com/conductorone/baton-sdk/pkg/connectorstore" + "github.com/conductorone/baton-sdk/pkg/sourcecache" +) + +// TestSourceCacheReplayResumeAfterOverlayCommit pins the grants shape: +// replay copy + committed overlay row with a net-new identity, then a +// re-run of the same replay. +func TestSourceCacheReplayResumeAfterOverlayCommit(t *testing.T) { + ctx := context.Background() + + prev := newAdapter(t) + _, err := prev.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + base1 := scGrant("member", "alice", false) + base2 := scGrant("member", "bob", false) + require.NoError(t, prev.PutGrants(sourcecache.WithScope(ctx, scopeA), base1, base2)) + + cur := newAdapter(t) + _, err = cur.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + + // First attempt: the replay copies the base... + res1, err := cur.PebbleEngine().ReplaySourceCacheGrants(ctx, prev.PebbleEngine(), scopeA) + require.NoError(t, err) + require.Equal(t, int64(2), res1.Rows) + require.Zero(t, res1.ResumedRowsCleared) + + // ...then the page's own overlay row commits under the same scope — + // a NET-NEW identity the previous file does not contain... + overlay := scGrant("member", "carol", false) + require.NoError(t, cur.PutGrants(sourcecache.WithScope(ctx, scopeA), overlay)) + + // ...plus a row another scope legitimately owns, and a stale scopeA + // index entry pointing at it: the re-run's pre-clear must keep the + // row and drop only the orphaned entry. + otherScopeRow := scGrant("owner", "dave", false) + require.NoError(t, cur.PutGrants(sourcecache.WithScope(ctx, scopeB), otherScopeRow)) + otherRec, err := cur.PebbleEngine().GetGrantRecord(ctx, otherScopeRow.GetId()) + require.NoError(t, err) + staleIdx := replayTestGrantScopeIndexKey(t, scopeA, otherRec) + require.NoError(t, cur.PebbleEngine().DB().Set(staleIdx, nil, nil)) + + // The sync crashes before the action finishes; the resumed action + // re-runs the same page. The replay must converge, not false-fail + // its post-copy recount against the overlay row. + res2, err := cur.PebbleEngine().ReplaySourceCacheGrants(ctx, prev.PebbleEngine(), scopeA) + require.NoError(t, err, "resumed replay over committed overlay rows must not false-fail") + require.Equal(t, int64(2), res2.Rows) + require.Equal(t, int64(3), res2.ResumedRowsCleared, + "the crashed attempt's two base copies plus its overlay row") + + // The other scope's row survived the pre-clear. + _, err = cur.PebbleEngine().GetGrantRecord(ctx, otherScopeRow.GetId()) + require.NoError(t, err, "a row stamped with another scope must survive the pre-clear") + + // The resumed page re-delivers its rows after the replay. + require.NoError(t, cur.PutGrants(sourcecache.WithScope(ctx, scopeA), overlay)) + + // Converged state: base + overlay + the other scope's row, once each. + for _, g := range []*v2.Grant{base1, base2, overlay, otherScopeRow} { + _, err := cur.PebbleEngine().GetGrantRecord(ctx, g.GetId()) + require.NoError(t, err) + } + var seen int + require.NoError(t, cur.PebbleEngine().IterateGrants(ctx, func(*v3.GrantRecord) bool { + seen++ + return true + })) + require.Equal(t, 4, seen) + + // The scope index is exact again: a replay FROM this store (as the + // next sync's previous) finds exactly the scope's three rows. + next := newAdapter(t) + _, err = next.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + res3, err := next.PebbleEngine().ReplaySourceCacheGrants(ctx, cur.PebbleEngine(), scopeA) + require.NoError(t, err) + require.Equal(t, int64(3), res3.Rows) + require.Zero(t, res3.StaleSkipped) +} + +// TestSourceCacheReplayResumeAfterOverlayCommitResourcesAndEntitlements +// covers the same crash shape for the other two row kinds, including the +// entitlement bare-id lookup invalidation for pre-clear deletes. +func TestSourceCacheReplayResumeAfterOverlayCommitResourcesAndEntitlements(t *testing.T) { + ctx := context.Background() + + prev := newAdapter(t) + _, err := prev.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + + baseRes := v2.Resource_builder{ + Id: v2.ResourceId_builder{ResourceType: "group", Resource: "g1"}.Build(), + DisplayName: "Group One", + }.Build() + baseEntID := "group:g1:member" + baseEnt := v2.Entitlement_builder{Id: baseEntID, Resource: baseRes, DisplayName: "member"}.Build() + scoped := sourcecache.WithScope(ctx, scopeA) + require.NoError(t, prev.PutResources(scoped, baseRes)) + require.NoError(t, prev.PutEntitlements(scoped, baseEnt)) + + cur := newAdapter(t) + _, err = cur.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + + // First attempt: replay copies, then overlay rows commit. + _, err = cur.PebbleEngine().ReplaySourceCacheResources(ctx, prev.PebbleEngine(), scopeA) + require.NoError(t, err) + _, err = cur.PebbleEngine().ReplaySourceCacheEntitlements(ctx, prev.PebbleEngine(), scopeA) + require.NoError(t, err) + + overlayRes := v2.Resource_builder{ + Id: v2.ResourceId_builder{ResourceType: "group", Resource: "g2"}.Build(), + DisplayName: "Group Two", + }.Build() + overlayEntID := "group:g2:member" + overlayEnt := v2.Entitlement_builder{Id: overlayEntID, Resource: overlayRes, DisplayName: "member"}.Build() + curScoped := sourcecache.WithScope(ctx, scopeA) + require.NoError(t, cur.PutResources(curScoped, overlayRes)) + require.NoError(t, cur.PutEntitlements(curScoped, overlayEnt)) + + // Resumed re-run: both kinds converge instead of false-failing. + rres, err := cur.PebbleEngine().ReplaySourceCacheResources(ctx, prev.PebbleEngine(), scopeA) + require.NoError(t, err, "resumed resources replay must not false-fail") + require.Equal(t, int64(1), rres.Rows) + require.Equal(t, int64(2), rres.ResumedRowsCleared) + + eres, err := cur.PebbleEngine().ReplaySourceCacheEntitlements(ctx, prev.PebbleEngine(), scopeA) + require.NoError(t, err, "resumed entitlements replay must not false-fail") + require.Equal(t, int64(1), eres.Rows) + require.Equal(t, int64(2), eres.ResumedRowsCleared) + + // The pre-clear's deletes invalidated the bare-id lookup: the overlay + // entitlement is gone until the resumed page re-delivers it. + _, err = cur.PebbleEngine().GetEntitlementRecord(ctx, overlayEntID) + require.Error(t, err, "cleared overlay entitlement must not resolve through a stale bare-id map") + + // The resumed page re-delivers its rows; state converges. + require.NoError(t, cur.PutResources(curScoped, overlayRes)) + require.NoError(t, cur.PutEntitlements(curScoped, overlayEnt)) + + gotRes, err := cur.PebbleEngine().GetResourceRecord(ctx, "group", "g2") + require.NoError(t, err) + require.Equal(t, "Group Two", gotRes.GetDisplayName()) + gotEnt, err := cur.PebbleEngine().GetEntitlementRecord(ctx, overlayEntID) + require.NoError(t, err) + require.Equal(t, scopeA, gotEnt.GetSourceScopeKey()) +} diff --git a/pkg/dotc1z/source_cache.go b/pkg/dotc1z/source_cache.go index 5afca6cbb..6134397e1 100644 --- a/pkg/dotc1z/source_cache.go +++ b/pkg/dotc1z/source_cache.go @@ -117,6 +117,12 @@ type IngestInvariantStore interface { // Value-reading; reserved for dangling-reference probes. GrantsForEntResourceCarryInsertFact(ctx context.Context, resourceTypeID, resourceID string) (bool, error) + // GrantsForEntitlementAllCarryInsertFact reports whether every grant + // under the entitlement identity carries InsertResourceGrants — the + // fail-fast side of I8's per-grant exemption. Value-reading; + // reserved for dangling-reference probes. + GrantsForEntitlementAllCarryInsertFact(ctx context.Context, entitlementID, entResourceTypeID, entResourceID string) (bool, error) + // HasResourceRecord reports whether a resource row exists. HasResourceRecord(ctx context.Context, resourceTypeID, resourceID string) (bool, error) @@ -148,8 +154,9 @@ type IngestInvariantStore interface { DeleteEntitlementsForResource(ctx context.Context, resourceTypeID, resourceID string, maxIDs int) (int64, []string, error) // DeleteGrantsForEntitlement drops every grant row under one - // entitlement identity; returns the count. - DeleteGrantsForEntitlement(ctx context.Context, entitlementID, entResourceTypeID, entResourceID string) (int64, error) + // entitlement identity except InsertResourceGrants carriers (the + // machinery-owned shape); returns (deleted, skippedInsertFact). + DeleteGrantsForEntitlement(ctx context.Context, entitlementID, entResourceTypeID, entResourceID string) (int64, int64, error) // DeleteGrantsForPrincipal drops every grant of one principal except // ExternalResourceMatch* carriers; returns (deleted, skipped). @@ -170,6 +177,10 @@ func (s *pebbleStore) GrantsForEntResourceCarryInsertFact(ctx context.Context, r return s.engine.GrantsForEntResourceCarryInsertFact(ctx, resourceTypeID, resourceID) } +func (s *pebbleStore) GrantsForEntitlementAllCarryInsertFact(ctx context.Context, entitlementID, entResourceTypeID, entResourceID string) (bool, error) { + return s.engine.GrantsForEntitlementAllCarryInsertFact(ctx, entitlementID, entResourceTypeID, entResourceID) +} + func (s *pebbleStore) HasResourceRecord(ctx context.Context, resourceTypeID, resourceID string) (bool, error) { return s.engine.HasResourceRecord(ctx, resourceTypeID, resourceID) } @@ -198,12 +209,12 @@ func (s *pebbleStore) DeleteEntitlementsForResource(ctx context.Context, resourc return n, ids, err } -func (s *pebbleStore) DeleteGrantsForEntitlement(ctx context.Context, entitlementID, entResourceTypeID, entResourceID string) (int64, error) { - n, err := s.engine.DeleteGrantsForEntitlement(ctx, entitlementID, entResourceTypeID, entResourceID) +func (s *pebbleStore) DeleteGrantsForEntitlement(ctx context.Context, entitlementID, entResourceTypeID, entResourceID string) (int64, int64, error) { + n, skipped, err := s.engine.DeleteGrantsForEntitlement(ctx, entitlementID, entResourceTypeID, entResourceID) if n > 0 { s.MarkDirty() } - return n, err + return n, skipped, err } func (s *pebbleStore) DeleteGrantsForPrincipal(ctx context.Context, principalRT, principalID string) (int64, int64, error) { diff --git a/pkg/sourcecache/grpc_server.go b/pkg/sourcecache/grpc_server.go index ea4356004..c47dee384 100644 --- a/pkg/sourcecache/grpc_server.go +++ b/pkg/sourcecache/grpc_server.go @@ -3,6 +3,7 @@ package sourcecache import ( "context" "fmt" + "sync" "sync/atomic" "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" @@ -24,6 +25,21 @@ import ( type GRPCServer struct { v1.UnimplementedBatonSourceCacheServiceServer lookup atomic.Pointer[Lookup] + + // installMu guards the install accounting below; the lookup pointer + // stays atomic so the RPC hot path never takes the mutex. + installMu sync.Mutex + // installs counts live installs (non-nil SetSourceCache calls not yet + // cleared). The wire carries no sync id, so an RPC cannot be + // attributed to one of several concurrent syncs; the slot is only + // servable while exactly one owner holds it. + installs int + // contended latches when a second concurrent install lands. From that + // point every lookup misses (found=false — always safe: the connector + // fetches cold) until the slot fully drains, because the server keeps + // no per-owner state and cannot know which lookup a surviving sync + // owns after the others clear. + contended bool } var _ v1.BatonSourceCacheServiceServer = (*GRPCServer)(nil) @@ -34,25 +50,42 @@ func NewGRPCServer() *GRPCServer { return &GRPCServer{} } -// SetSourceCache replaces the active lookup. Safe to call concurrently with -// in-flight RPCs: existing RPCs continue against the value they read at -// entry; new RPCs see the swapped value. +// SetSourceCache installs (non-nil) or clears (nil) a sync's lookup. Safe +// to call concurrently with in-flight RPCs: existing RPCs continue against +// the value they read at entry; new RPCs see the swapped value. // -// The slot is still SINGLE — the wire carries no sync id — so two live -// syncs installing lookups cross-wire each other (one sync's connector -// resolves scopes against the other's previous store) and the first to -// finish blinds the survivor when it clears. That topology isn't -// supported; an install landing on an occupied slot warns loudly so it is -// visible instead of silently wrong. +// The slot is SINGLE — the wire carries no sync id — so with two live +// syncs the server cannot attribute a Lookup RPC to either one. Serving +// the last-installed lookup would cross-wire them: one sync's connector +// would resolve validators against the other's previous store, and its +// syncer would then replay rows from its OWN store under a validator the +// other artifact vouched for — stale rows sealed as current. A miss, by +// contrast, is always safe (the connector fetches cold). So on contention +// the server blinds every caller until all concurrent owners have +// cleared, and warns loudly so the degraded overlap is visible. func (s *GRPCServer) SetSourceCache(ctx context.Context, lookup Lookup) { + s.installMu.Lock() + defer s.installMu.Unlock() if lookup == nil { + if s.installs > 0 { + s.installs-- + } + if s.installs == 0 { + // Slot fully drained: the next sync starts clean. + s.contended = false + } s.lookup.Store(nil) return } - if prev := s.lookup.Swap(&lookup); prev != nil { - ctxzap.Extract(ctx).Warn("source cache: installing a lookup over an existing one; " + - "concurrent syncs against one connector share a single lookup slot and will cross-wire or blind each other") + s.installs++ + if s.installs > 1 || s.contended { + s.contended = true + s.lookup.Store(nil) + ctxzap.Extract(ctx).Warn("source cache: concurrent syncs share this connector's single lookup slot; " + + "serving misses to every sync (cold fetches) until the overlap drains, to avoid cross-wiring validators between previous-sync artifacts") + return } + s.lookup.Store(&lookup) } func (s *GRPCServer) Lookup(ctx context.Context, req *v1.LookupRequest) (*v1.LookupResponse, error) { diff --git a/pkg/sourcecache/grpc_server_test.go b/pkg/sourcecache/grpc_server_test.go new file mode 100644 index 000000000..6859ab80a --- /dev/null +++ b/pkg/sourcecache/grpc_server_test.go @@ -0,0 +1,106 @@ +package sourcecache + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + + v1 "github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1" +) + +// staticLookup answers every scope with a fixed validator, tagging which +// sync's store it stands in for. +type staticLookup struct{ validator string } + +func (l staticLookup) Lookup(context.Context, RowKind, string) (Entry, bool, error) { + return Entry{CacheValidator: l.validator}, true, nil +} + +func lookupReq(t *testing.T) *v1.LookupRequest { + t.Helper() + return v1.LookupRequest_builder{ + RowKind: string(RowKindGrants), + ScopeKey: HashScope("scope"), + }.Build() +} + +func requireServes(t *testing.T, s *GRPCServer, wantValidator string) { + t.Helper() + resp, err := s.Lookup(context.Background(), lookupReq(t)) + require.NoError(t, err) + require.True(t, resp.GetFound()) + require.Equal(t, wantValidator, resp.GetCacheValidator()) +} + +func requireMisses(t *testing.T, s *GRPCServer) { + t.Helper() + resp, err := s.Lookup(context.Background(), lookupReq(t)) + require.NoError(t, err) + require.False(t, resp.GetFound()) +} + +// TestGRPCServerSingleOwnerLifecycle pins the uncontended path: install +// serves, clear blinds, a fresh install serves again. +func TestGRPCServerSingleOwnerLifecycle(t *testing.T) { + ctx := context.Background() + s := NewGRPCServer() + + requireMisses(t, s) + s.SetSourceCache(ctx, staticLookup{validator: "etag-a"}) + requireServes(t, s, "etag-a") + s.SetSourceCache(ctx, nil) + requireMisses(t, s) + s.SetSourceCache(ctx, staticLookup{validator: "etag-b"}) + requireServes(t, s, "etag-b") +} + +// TestGRPCServerContentionBlindsAllOwners pins the cross-wiring defense: +// the wire carries no sync id, so with two live syncs the server cannot +// attribute an RPC to either — serving the last-installed lookup would +// hand sync A validators from sync B's previous artifact while A replays +// rows from its own. On contention every lookup must MISS (cold fetch, +// always safe) until the slot fully drains. +func TestGRPCServerContentionBlindsAllOwners(t *testing.T) { + ctx := context.Background() + s := NewGRPCServer() + + // Sync A installs; sync B overlaps. + s.SetSourceCache(ctx, staticLookup{validator: "etag-a"}) + requireServes(t, s, "etag-a") + s.SetSourceCache(ctx, staticLookup{validator: "etag-b"}) + requireMisses(t, s) + + // A finishes and clears. B is the sole survivor, but the server kept + // no per-owner state — it must stay blind rather than guess. + s.SetSourceCache(ctx, nil) + requireMisses(t, s) + + // A third sync overlapping the still-live B must not be served either + // (this is the exact validator-from-S1/rows-from-S0 shape). + s.SetSourceCache(ctx, staticLookup{validator: "etag-c"}) + requireMisses(t, s) + s.SetSourceCache(ctx, nil) + requireMisses(t, s) + + // Only when the overlap fully drains does the slot reset. + s.SetSourceCache(ctx, nil) + requireMisses(t, s) + s.SetSourceCache(ctx, staticLookup{validator: "etag-d"}) + requireServes(t, s, "etag-d") +} + +// TestGRPCServerUnbalancedClearIsClamped pins the defensive clamp: a +// spurious extra clear (a sync that never installed) must not wedge the +// counter below zero and break the next legitimate install. +func TestGRPCServerUnbalancedClearIsClamped(t *testing.T) { + ctx := context.Background() + s := NewGRPCServer() + + s.SetSourceCache(ctx, nil) + s.SetSourceCache(ctx, nil) + s.SetSourceCache(ctx, staticLookup{validator: "etag-a"}) + requireServes(t, s, "etag-a") + s.SetSourceCache(ctx, nil) + requireMisses(t, s) +} diff --git a/pkg/sourcecache/sourcecache.go b/pkg/sourcecache/sourcecache.go index 514a6668f..fcd0003a9 100644 --- a/pkg/sourcecache/sourcecache.go +++ b/pkg/sourcecache/sourcecache.go @@ -132,7 +132,11 @@ func (NoopLookup) Lookup(context.Context, RowKind, string) (Entry, bool, error) // SourceCacheSetter is implemented by connector clients/servers that can receive a // source-cache lookup implementation from the sync runner. The SDK calls // SetSourceCache(lookup) at the start of each sync and SetSourceCache(nil) -// when the sync ends so a late RPC can't read stale state. +// when the sync ends so a late RPC can't read stale state. Calls must be +// balanced per sync — exactly one nil clear for each non-nil install, and +// no clear from a sync that never installed — because the shared +// subprocess slot counts installs to detect concurrent-sync contention +// (see GRPCServer.SetSourceCache). type SourceCacheSetter interface { SetSourceCache(ctx context.Context, lookup Lookup) } diff --git a/pkg/sync/ingest_invariants.go b/pkg/sync/ingest_invariants.go index 572363917..b308b3d0b 100644 --- a/pkg/sync/ingest_invariants.go +++ b/pkg/sync/ingest_invariants.go @@ -603,12 +603,16 @@ func (s *syncer) checkEntitlementResourceReferences(ctx context.Context) error { // One seek plus one point probe per distinct entitlement — O(distinct // entitlements), never O(grants). // -// InsertResourceGrants pages are EXEMPT in every mode: that pattern has -// always produced grants whose entitlements have no row (the machinery -// inserts the grant's embedded resource; no listing ever returns an -// entitlement for it), and downstream consumers synthesize the -// entitlement from the grant. The insert-fact probe is the same one I3 -// uses, value-reading and reserved for dangling refs. +// InsertResourceGrants grants are EXEMPT in every mode, per GRANT: that +// pattern has always produced grants whose entitlements have no row (the +// machinery inserts the grant's embedded resource; no listing ever +// returns an entitlement for it), and downstream consumers synthesize +// the entitlement from the grant. The exemption follows the ownership +// rule (docs/tasks/dangling-reference-drops.md) at the grant's own +// granularity: a connector-owned grant dangling under the same missing +// entitlement as machinery-owned IRG grants is still a violation — a +// resource- or entitlement-level exemption would let one IRG grant +// launder every dangling reference that shares its resource. // // Default mode DROPS the dangling grants (uplift reads grants BY // entitlement, so rows under a missing entitlement are never even read @@ -622,38 +626,34 @@ func (s *syncer) checkGrantEntitlementReferences(ctx context.Context) error { if !ok { return nil // engine without the inspection surface } - // Dangling entitlements cluster under shared resources; memoize the - // per-resource insert-fact probe (it reads row values). - insertFactByResource := map[string]bool{} probe := &danglingTypeProbe{s: s} danglingEnts, unsyncedType, missingRow := 0, 0, 0 - var droppedRows int64 + var droppedRows, insertFactKept int64 var entIDExamples []string err := facts.ForEachDanglingGrantEntitlement(ctx, func(entitlementID, rt, rid string) error { - resKey := rt + "\x00" + rid - annotated, probed := insertFactByResource[resKey] - if !probed { - var err error - annotated, err = facts.GrantsForEntResourceCarryInsertFact(ctx, rt, rid) + if s.failFastInvariants { + allCarry, err := facts.GrantsForEntitlementAllCarryInsertFact(ctx, entitlementID, rt, rid) if err != nil { - return fmt.Errorf("ingest invariant I8: probing grant annotations for %s/%s: %w", rt, rid, err) + return fmt.Errorf("ingest invariant I8: probing grant annotations for entitlement %q: %w", entitlementID, err) + } + if allCarry { + return nil // the established InsertResourceGrants shape } - insertFactByResource[resKey] = annotated - } - if annotated { - return nil // the established InsertResourceGrants shape - } - if s.failFastInvariants { return fmt.Errorf( - "ingest invariant I8 violated: grants reference entitlement %q (resource %s/%s) but no entitlement row exists "+ + "ingest invariant I8 violated: grants without InsertResourceGrants reference entitlement %q (resource %s/%s) but no entitlement row exists "+ "(resource type synced: %t — false means a config gap, true a magic-id construction bug "+ "or a grants scope replayed against a refreshed entitlement set)", entitlementID, rt, rid, probe.typeExists(ctx, rt)) } - n, err := facts.DeleteGrantsForEntitlement(ctx, entitlementID, rt, rid) + n, skipped, err := facts.DeleteGrantsForEntitlement(ctx, entitlementID, rt, rid) if err != nil { return fmt.Errorf("ingest invariant I8: dropping grants for entitlement %q: %w", entitlementID, err) } + insertFactKept += skipped + if n == 0 { + // Pure machinery shape: every grant carried the insert fact. + return nil + } danglingEnts++ droppedRows += n if probe.typeExists(ctx, rt) { @@ -673,6 +673,7 @@ func (s *syncer) checkGrantEntitlementReferences(ctx context.Context) error { ctxzap.Extract(ctx).Warn("ingest invariant I8: DROPPED grants referencing entitlements with no entitlement row", zap.Int64("dropped_grants", droppedRows), zap.Int("dangling_entitlements", danglingEnts), + zap.Int64("insert_resource_grants_kept", insertFactKept), zap.Int("refs_into_unsynced_types", unsyncedType), zap.Int("refs_missing_rows_in_synced_types", missingRow), zap.String("attribution", danglingAttribution(unsyncedType, missingRow)), @@ -775,6 +776,16 @@ func (s *syncer) checkGrantPrincipalReferences(ctx context.Context) error { // lost manifest write or a stamp leak — and the damage is deferred: THIS // sync reads clean, the NEXT sync replays from the damaged state. func (s *syncer) checkSourceCacheScopeConsistency(ctx context.Context) error { + if s.onlyExpandGrants { + // Expansion-only runs ingest no pages, so scope/manifest state + // was settled (and I6-checked) by whoever produced the store. + // The load-bearing case is compaction's expand-grants pass: a + // fold output deliberately violates I6's premise — the manifest + // was cleared (no input's validator describes the merged rows) + // while the base copy's scope stamps remain — so every stamped + // scope would be reported as an orphan on a healthy artifact. + return nil + } sc, ok := s.store.(dotc1z.SourceCacheStore) if !ok { return nil // engine without source-cache state diff --git a/pkg/sync/ingest_invariants_granularity_test.go b/pkg/sync/ingest_invariants_granularity_test.go new file mode 100644 index 000000000..72f65a98b --- /dev/null +++ b/pkg/sync/ingest_invariants_granularity_test.go @@ -0,0 +1,170 @@ +package sync //nolint:revive,nolintlint // we can't change the package name for backwards compatibility + +// Granularity tests for the invariant exemptions and gates: +// +// - I8's InsertResourceGrants exemption is per GRANT (the ownership +// rule's unit), not per entitlement resource — one machinery-owned +// IRG grant must not launder connector-owned dangling references +// that share its resource. +// - I6 (scope/manifest consistency) is skipped for expand-grants-only +// syncs: compaction's fold output deliberately clears the manifest +// while keeping scope stamps, so on that path every stamped scope of +// a healthy artifact would read as an orphan. + +import ( + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + reader_v2 "github.com/conductorone/baton-sdk/pb/c1/reader/v2" + "github.com/conductorone/baton-sdk/pkg/connectorstore" + "github.com/conductorone/baton-sdk/pkg/logging" + "github.com/conductorone/baton-sdk/pkg/sourcecache" + "github.com/conductorone/baton-sdk/pkg/types/grant" + rs "github.com/conductorone/baton-sdk/pkg/types/resource" +) + +// TestIngestInvariantI8ExemptionIsPerGrant pins the granularity: a +// resource holding BOTH a machinery-owned IRG grant (dangling by design) +// and a connector-owned grant referencing a different missing entitlement +// must keep the former and drop (or fail on) the latter. +func TestIngestInvariantI8ExemptionIsPerGrant(t *testing.T) { + ctx, err := logging.Init(t.Context()) + require.NoError(t, err) + tmpDir := t.TempDir() + + repo, err := rs.NewResource("Repo r1", equivRepoRT, "r1") + require.NoError(t, err) + irgGrant := repairTestGrant(t, repo) // "admin" slug, carries InsertResourceGrants + ghost := v2.Resource_builder{ + Id: v2.ResourceId_builder{ResourceType: "user", Resource: "bob"}.Build(), + }.Build() + // Same resource, different missing entitlement, NO annotation: + // connector-owned dangling reference. + plainGrant := grant.NewGrant(repo, "viewer", ghost.GetId()) + + cur := newRepairTestStore(ctx, t, filepath.Join(tmpDir, "cur.c1z"), tmpDir) + _, err = cur.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + require.NoError(t, cur.PutResourceTypes(ctx, equivRepoRT)) + require.NoError(t, cur.PutResources(ctx, repo)) + require.NoError(t, cur.PutGrants(ctx, irgGrant, plainGrant)) + + s := &syncer{store: cur, syncType: connectorstore.SyncTypeFull} + + // Fail-fast: the connector-owned dangling grant must fail the sync — + // the sibling IRG grant on the same resource must not exempt it. + s.failFastInvariants = true + err = s.checkGrantEntitlementReferences(ctx) + require.Error(t, err, "an IRG grant on the resource must not exempt an unrelated dangling reference") + require.Contains(t, err.Error(), "ingest invariant I8") + require.Contains(t, err.Error(), "viewer") + require.NotErrorIs(t, err, ErrReplayIntegrity) + + // Default mode: the connector-owned grant drops, the IRG grant stays. + s.failFastInvariants = false + require.NoError(t, s.checkGrantEntitlementReferences(ctx)) + _, err = cur.GetGrant(ctx, reader_v2.GrantsReaderServiceGetGrantRequest_builder{ + GrantId: plainGrant.GetId(), + }.Build()) + require.Error(t, err, "the connector-owned dangling grant must be dropped") + _, err = cur.GetGrant(ctx, reader_v2.GrantsReaderServiceGetGrantRequest_builder{ + GrantId: irgGrant.GetId(), + }.Build()) + require.NoError(t, err, "the machinery-owned IRG grant must never be dropped") + + // Idempotent: after the drop, both modes find nothing. + require.NoError(t, s.checkGrantEntitlementReferences(ctx)) + s.failFastInvariants = true + require.NoError(t, s.checkGrantEntitlementReferences(ctx)) + + require.NoError(t, cur.Close(ctx)) +} + +// TestIngestInvariantI8MixedGrantsUnderOneEntitlement pins the per-grant +// rule within a single missing entitlement: IRG carriers and plain grants +// can share the SAME entitlement identity (an IRG page and a normal page +// both emitted grants for it); only the plain grants drop. +func TestIngestInvariantI8MixedGrantsUnderOneEntitlement(t *testing.T) { + ctx, err := logging.Init(t.Context()) + require.NoError(t, err) + tmpDir := t.TempDir() + + repo, err := rs.NewResource("Repo r1", equivRepoRT, "r1") + require.NoError(t, err) + irgGrant := repairTestGrant(t, repo) // "admin" entitlement, alice, annotated + ghost := v2.Resource_builder{ + Id: v2.ResourceId_builder{ResourceType: "user", Resource: "bob"}.Build(), + }.Build() + plainGrant := grant.NewGrant(repo, "admin", ghost.GetId()) // same entitlement, no annotation + + cur := newRepairTestStore(ctx, t, filepath.Join(tmpDir, "cur.c1z"), tmpDir) + _, err = cur.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + require.NoError(t, cur.PutResourceTypes(ctx, equivRepoRT)) + require.NoError(t, cur.PutResources(ctx, repo)) + require.NoError(t, cur.PutGrants(ctx, irgGrant, plainGrant)) + + s := &syncer{store: cur, syncType: connectorstore.SyncTypeFull} + + s.failFastInvariants = true + err = s.checkGrantEntitlementReferences(ctx) + require.Error(t, err, "a mixed entitlement (IRG + plain grants) must fail fast on the plain grant") + + s.failFastInvariants = false + require.NoError(t, s.checkGrantEntitlementReferences(ctx)) + _, err = cur.GetGrant(ctx, reader_v2.GrantsReaderServiceGetGrantRequest_builder{ + GrantId: plainGrant.GetId(), + }.Build()) + require.Error(t, err, "the plain grant under the mixed entitlement must be dropped") + _, err = cur.GetGrant(ctx, reader_v2.GrantsReaderServiceGetGrantRequest_builder{ + GrantId: irgGrant.GetId(), + }.Build()) + require.NoError(t, err, "the IRG carrier under the mixed entitlement must be kept") + + require.NoError(t, cur.Close(ctx)) +} + +// TestIngestInvariantI6SkippedForExpandOnlySyncs pins the expand-grants +// gate: an orphan scope (stamped rows, no manifest entry) fails a normal +// sealed sync's I6, but an expansion-only run — compaction's pass over a +// fold output whose manifest was deliberately cleared — must not evaluate +// it at all. +func TestIngestInvariantI6SkippedForExpandOnlySyncs(t *testing.T) { + ctx, err := logging.Init(t.Context()) + require.NoError(t, err) + tmpDir := t.TempDir() + + repo, err := rs.NewResource("Repo r1", equivRepoRT, "r1") + require.NoError(t, err) + ghost := v2.Resource_builder{ + Id: v2.ResourceId_builder{ResourceType: "user", Resource: "alice"}.Build(), + }.Build() + g := grant.NewGrant(repo, "admin", ghost.GetId()) + + // Stamped rows with NO manifest entry: the orphan shape. + cur := newRepairTestStore(ctx, t, filepath.Join(tmpDir, "cur.c1z"), tmpDir) + _, err = cur.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + require.NoError(t, cur.PutResourceTypes(ctx, equivRepoRT)) + require.NoError(t, cur.PutResources(ctx, repo)) + scope := sourcecache.HashScope("grants:r1") + require.NoError(t, cur.PutGrants(sourcecache.WithScope(ctx, scope), g)) + + s := &syncer{store: cur, syncType: connectorstore.SyncTypeFull, failFastInvariants: true} + + // Sanity: a normal sync detects the orphan (replay-integrity class). + err = s.checkSourceCacheScopeConsistency(ctx) + require.Error(t, err) + require.ErrorIs(t, err, ErrReplayIntegrity) + require.Contains(t, err.Error(), "ingest invariant I6") + + // The expansion-only pass must skip the check entirely. + s.onlyExpandGrants = true + require.NoError(t, s.checkSourceCacheScopeConsistency(ctx), + "expand-grants-only syncs must not evaluate I6: fold outputs legitimately hold stamps without a manifest") + + require.NoError(t, cur.Close(ctx)) +} diff --git a/pkg/sync/ingest_invariants_i9_mixed_test.go b/pkg/sync/ingest_invariants_i9_mixed_test.go new file mode 100644 index 000000000..32ca0fb4b --- /dev/null +++ b/pkg/sync/ingest_invariants_i9_mixed_test.go @@ -0,0 +1,96 @@ +package sync //nolint:revive,nolintlint // we can't change the package name for backwards compatibility + +// Mixed-population pin for I9's exemption: exemptions must be judged per +// GRANT, not per principal. One dangling principal carrying BOTH an +// unprocessed external-match carrier and a plain grant must keep exactly +// the carrier and drop exactly the plain grant in default mode, and must +// hard-fail (dropping nothing) in fail-fast mode. This is the fixture +// shape that homogeneous exemption tests structurally cannot check — the +// escape route the per-entitlement-resource I8 exemption granularity bug +// took. + +import ( + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + reader_v2 "github.com/conductorone/baton-sdk/pb/c1/reader/v2" + "github.com/conductorone/baton-sdk/pkg/annotations" + "github.com/conductorone/baton-sdk/pkg/connectorstore" + "github.com/conductorone/baton-sdk/pkg/logging" + et "github.com/conductorone/baton-sdk/pkg/types/entitlement" + "github.com/conductorone/baton-sdk/pkg/types/grant" + rs "github.com/conductorone/baton-sdk/pkg/types/resource" +) + +func TestIngestInvariantI9MixedPrincipalPopulation(t *testing.T) { + ctx, err := logging.Init(t.Context()) + require.NoError(t, err) + tmpDir := t.TempDir() + + repo, err := rs.NewResource("Repo r1", equivRepoRT, "r1") + require.NoError(t, err) + adminEnt := et.NewAssignmentEntitlement(repo, "admin") + viewerEnt := et.NewAssignmentEntitlement(repo, "viewer") + ghost := v2.Resource_builder{ + Id: v2.ResourceId_builder{ResourceType: "user", Resource: "ghost"}.Build(), + }.Build() + + // The SAME dangling principal carries one match carrier and one + // plain grant. + carrier := grant.NewGrant(repo, "admin", ghost.GetId()) + annos := annotations.Annotations(carrier.GetAnnotations()) + annos.Update(v2.ExternalResourceMatch_builder{Key: "email", Value: "ghost@example.com"}.Build()) + carrier.SetAnnotations(annos) + plain := grant.NewGrant(repo, "viewer", ghost.GetId()) + + cur := newRepairTestStore(ctx, t, filepath.Join(tmpDir, "cur.c1z"), tmpDir) + _, err = cur.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + require.NoError(t, cur.PutResourceTypes(ctx, equivRepoRT)) + require.NoError(t, cur.PutResources(ctx, repo)) + require.NoError(t, cur.PutEntitlements(ctx, adminEnt, viewerEnt)) + require.NoError(t, cur.PutGrants(ctx, carrier, plain)) + + s := &syncer{store: cur, syncType: connectorstore.SyncTypeFull} + + requireGrant := func(id string, wantPresent bool, msg string) { + t.Helper() + _, err := cur.GetGrant(ctx, reader_v2.GrantsReaderServiceGetGrantRequest_builder{ + GrantId: id, + }.Build()) + if wantPresent { + require.NoError(t, err, msg) + } else { + require.Error(t, err, msg) + } + } + + // Fail-fast: a mixed principal is NOT the exempt shape — it must be + // named as a violation, and nothing may be dropped. + s.failFastInvariants = true + err = s.checkGrantPrincipalReferences(ctx) + require.Error(t, err) + require.Contains(t, err.Error(), "ingest invariant I9") + requireGrant(carrier.GetId(), true, "fail-fast must not drop the carrier") + requireGrant(plain.GetId(), true, "fail-fast must not drop the plain grant") + + // Default mode: per-grant judgment — the carrier survives, the plain + // grant is dropped. + s.failFastInvariants = false + require.NoError(t, s.checkGrantPrincipalReferences(ctx)) + requireGrant(carrier.GetId(), true, "the match carrier must be kept (config-gap evidence)") + requireGrant(plain.GetId(), false, "the plain grant under the dangling principal must be dropped") + + // After the drop, the principal's surviving population is + // carriers-only — the exempt shape — so BOTH modes now pass without + // further mutation (idempotence of the mixed case). + require.NoError(t, s.checkGrantPrincipalReferences(ctx)) + s.failFastInvariants = true + require.NoError(t, s.checkGrantPrincipalReferences(ctx)) + requireGrant(carrier.GetId(), true, "the carrier must survive repeated passes") + + require.NoError(t, cur.Close(ctx)) +} diff --git a/pkg/sync/source_cache.go b/pkg/sync/source_cache.go index 39f8413a9..54e1b5973 100644 --- a/pkg/sync/source_cache.go +++ b/pkg/sync/source_cache.go @@ -62,6 +62,12 @@ type syncerSourceCache struct { // per-sync struct shares the set; guarded internally for parallel // workers. Best-effort across resume (in-memory), stats-only. replayedScopes *replayedScopeSet + // lookupInstalled records that THIS sync installed a lookup (real or + // no-op) into the connector's shared slot, so Close only clears what + // it owns. A syncer that failed before installing must not clear — + // on the shared subprocess slot that would decrement another live + // sync's install and blind it (see sourcecache.GRPCServer). + lookupInstalled bool } type scopeRef struct { @@ -128,7 +134,12 @@ func (s *syncer) configureSourceCache(ctx context.Context, resp *v2.ConnectorSer setLookup, canSetLookup := s.connector.(sourcecache.SourceCacheSetter) degrade := func(reason string) error { if canSetLookup { + // The no-op install still occupies the shared slot: a + // degraded sync's connector must see misses even while + // another sync's real lookup is live, or its replay-illegal + // state could receive hits. setLookup.SetSourceCache(ctx, sourcecache.NoopLookup{}) + s.sourceCache.lookupInstalled = true } if reason != "" { l.Info("source cache disabled", zap.String("reason", reason)) @@ -197,6 +208,7 @@ func (s *syncer) configureSourceCache(ctx context.Context, resp *v2.ConnectorSer s.sourceCache.lookup = lookup if canSetLookup { setLookup.SetSourceCache(ctx, lookup) + s.sourceCache.lookupInstalled = true } else { // The connector declared the capability but the client offers no // way to deliver lookups. Its own lookup stays no-op, so every @@ -229,6 +241,19 @@ func previousSyncReplayUnusableReason(ctx context.Context, reader connectorstore case run == nil: return "previous-sync c1z has no finished sync run", nil case run.UsableAsReplaySource(): + // The record can lie by omission: a compacted artifact produced + // by an SDK that predates the run record's compacted flag carries + // a full type and no flag, and its fold never cleared the base + // copy's source-cache manifest — so both the metadata gate and + // the manifest belt would pass, and the keep-newer merge would + // replay under validators that do not describe its contents. The + // sync TOKEN's compaction provenance section has been written by + // every compactor version, so it is the one signal a mixed-version + // artifact cannot omit. A token that fails to parse proves + // nothing either way and is ignored (the primary gates stand). + if comp, tokenErr := CompactionStatsFromToken(run.SyncToken); tokenErr == nil && comp != nil { + return "previous sync's token carries compaction provenance; the artifact is a merge no upstream validator describes (produced by a pre-compacted-flag compactor)", nil + } return "", nil case run.Compacted: return "previous sync is a compaction artifact; upstream validators do not describe its merged contents", nil @@ -238,10 +263,17 @@ func previousSyncReplayUnusableReason(ctx context.Context, reader connectorstore } // clearSourceCacheLookup detaches the per-sync lookup so a late RPC from -// the connector cannot read a store the syncer no longer owns. +// the connector cannot read a store the syncer no longer owns. Only the +// sync that actually installed a lookup clears: a syncer that failed +// before configureSourceCache's install must not decrement the shared +// slot's accounting or blind a concurrently live sync. func (s *syncer) clearSourceCacheLookup(ctx context.Context) { + if !s.sourceCache.lookupInstalled { + return + } if setLookup, ok := s.connector.(sourcecache.SourceCacheSetter); ok { setLookup.SetSourceCache(ctx, nil) + s.sourceCache.lookupInstalled = false } } @@ -441,6 +473,16 @@ func (s *syncer) executeScopeReplay(ctx context.Context, kind sourcecache.RowKin zap.Int64("stale_skipped", res.StaleSkipped), ) } + if res.ResumedRowsCleared > 0 { + // A re-run over a crashed attempt's committed rows: the engine + // cleared them before re-copying so the copy is idempotent. + // Expected after a resume; noteworthy otherwise. + ctxzap.Extract(ctx).Info("source cache: replay re-run cleared rows committed by a previous attempt", + zap.String("scope_key", page.scopeKey), + zap.Int64("rows_cleared", res.ResumedRowsCleared), + zap.Int64("rows_copied", res.Rows), + ) + } // Side effects for replayed rows are STORE-DERIVED, not armed // here — the engine maintains the needs_expansion index and the // external-match existence bit for replayed rows exactly as for diff --git a/pkg/sync/source_cache_equivalence_test.go b/pkg/sync/source_cache_equivalence_test.go index 10ec3e118..72fbf1e86 100644 --- a/pkg/sync/source_cache_equivalence_test.go +++ b/pkg/sync/source_cache_equivalence_test.go @@ -91,7 +91,10 @@ import ( stdsync "sync" "testing" + "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" "github.com/stretchr/testify/require" + "go.uber.org/zap" + "go.uber.org/zap/zapcore" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" @@ -1838,9 +1841,68 @@ func TestSourceCache_ReplayEquivalence(t *testing.T) { } } +// invariantWarningRecorder captures ingestion-invariant warnings emitted +// during a scenario. Warnings are the invariants' DEFAULT-mode verdicts, +// and nothing in a test suite fails on a log line — which is exactly how +// a false-positive warning (the I6-on-compaction class) can fire on every +// run of a green suite without anyone noticing. The harness therefore +// asserts warning-SILENCE for its clean scenarios: an invariant warning +// here is either an invariant false positive or an undetected data +// problem in the harness fixtures, and both must be named. +type invariantWarningRecorder struct { + mu stdsync.Mutex + msgs []string +} + +func (r *invariantWarningRecorder) record(msg string) { + r.mu.Lock() + defer r.mu.Unlock() + r.msgs = append(r.msgs, msg) +} + +func (r *invariantWarningRecorder) captured() []string { + r.mu.Lock() + defer r.mu.Unlock() + return append([]string(nil), r.msgs...) +} + +// invariantWarningCore is a zapcore.Core that records Warn+ entries whose +// message names an ingestion invariant. Tee'd with the base core so +// human-readable output is unchanged. +type invariantWarningCore struct { + zapcore.LevelEnabler + rec *invariantWarningRecorder +} + +func (c invariantWarningCore) With([]zapcore.Field) zapcore.Core { return c } + +func (c invariantWarningCore) Check(e zapcore.Entry, ce *zapcore.CheckedEntry) *zapcore.CheckedEntry { + if c.Enabled(e.Level) { + return ce.AddCore(e, c) + } + return ce +} + +func (c invariantWarningCore) Write(e zapcore.Entry, _ []zapcore.Field) error { + if strings.Contains(e.Message, "ingest invariant") { + c.rec.record(e.Level.String() + ": " + e.Message) + } + return nil +} + +func (c invariantWarningCore) Sync() error { return nil } + +func withInvariantWarningOracle(ctx context.Context) (context.Context, *invariantWarningRecorder) { + rec := &invariantWarningRecorder{} + base := ctxzap.Extract(ctx) + logger := zap.New(zapcore.NewTee(base.Core(), invariantWarningCore{LevelEnabler: zapcore.WarnLevel, rec: rec})) + return ctxzap.ToContext(ctx, logger), rec +} + func runReplayEquivalenceScenario(t *testing.T, workers int, seed int64, continuation bool) { ctx, err := logging.Init(t.Context()) require.NoError(t, err) + ctx, invariantWarnings := withInvariantWarningOracle(ctx) tmpDir := t.TempDir() rng := rand.New(rand.NewSource(seed)) //nolint:gosec // deterministic test randomness t.Logf("replay-equivalence scenario: workers=%d seed=%d rounds=%d continuation=%v", workers, seed, equivRounds, continuation) @@ -1941,4 +2003,13 @@ func runReplayEquivalenceScenario(t *testing.T, workers int, seed int64, continu require.Zero(t, coldFinal.pages304+coldFinal.pagesOverlay, "harness bug: the cold connector must never replay") require.Zero(t, coldFinal.askPages, "harness bug: the cold connector must never ask (no offer without a warm lookup)") + + // Warning-silence oracle: clean scenarios must emit ZERO ingestion- + // invariant warnings across every warm/cold/resume leg. A warning is + // an invariant verdict that no test asserts on — the escape route the + // I6-on-compaction false positive took (warned on every run of a + // green suite). Either the invariant is falsely firing or the harness + // fixtures have an undetected data problem; both must fail here. + require.Empty(t, invariantWarnings.captured(), + "ingestion-invariant warnings during a clean scenario") } diff --git a/pkg/sync/source_cache_mixed_version_test.go b/pkg/sync/source_cache_mixed_version_test.go new file mode 100644 index 000000000..518784c45 --- /dev/null +++ b/pkg/sync/source_cache_mixed_version_test.go @@ -0,0 +1,90 @@ +package sync //nolint:revive,nolintlint // we can't change the package name for backwards compatibility + +// Mixed-version compaction artifacts and replay eligibility. +// +// A fold compactor built from an SDK that predates the run record's +// compacted flag produces an artifact that LOOKS replay-eligible to the +// metadata gate: the record carries a full type and no compacted flag +// (the old proto has no such field), and the old fold never cleared the +// base copy's source-cache manifest (the old engine has no source-cache +// code), so the manifest belt passes too. The one signal every compactor +// version has always written is the sync TOKEN's compaction provenance +// section (BuildCompactedToken) — so the syncer's replay gate must read +// it and refuse the artifact. + +import ( + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + "github.com/conductorone/baton-sdk/pkg/dotc1z" + "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" + pebbleengine "github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble" + "github.com/conductorone/baton-sdk/pkg/logging" + gt "github.com/conductorone/baton-sdk/pkg/types/grant" +) + +// TestSourceCache_PreCompactedFlagFoldArtifactDegradesToCold pins the +// token-provenance gate: a previous sync whose run record says +// full/non-compacted but whose sync token carries a compaction section +// (the shape an old-binary fold leaves on a new-format base) must never +// serve replay, even with its manifest fully intact. +func TestSourceCache_PreCompactedFlagFoldArtifactDegradesToCold(t *testing.T) { + ctx, err := logging.Init(t.Context()) + require.NoError(t, err) + tmpDir := t.TempDir() + + group, ent, alice, _ := sourceCacheTestFixtures(t) + g1 := gt.NewGrant(group, "member", alice) + + mc := newSourceCacheMockConnector() + mc.AddResource(ctx, group) + mc.AddResource(ctx, alice) + mc.entDB[group.GetId().GetResource()] = []*v2.Entitlement{ent} + mc.grantDB[group.GetId().GetResource()] = []*v2.Grant{g1} + mc.etagByResource[group.GetId().GetResource()] = `W/"etag-v1"` + + sync1 := filepath.Join(tmpDir, "sync1.c1z") + runSourceCacheSync(ctx, t, mc, sync1, "", tmpDir) + + // Rewrite sync 1's run record into the old-fold shape: keep the full + // type, leave compacted UNSET (the old proto has no field to set), + // and give it the compaction-provenance token every fold — old or + // new — writes. The source-cache manifest stays intact, exactly as + // an old fold (no ClearSourceCacheEntries) would leave it. + marker, err := dotc1z.NewStore(ctx, sync1, + dotc1z.WithEngine(c1zstore.EnginePebble), + dotc1z.WithTmpDir(tmpDir), + ) + require.NoError(t, err) + run, err := marker.SyncMeta().LatestFullSync(ctx) + require.NoError(t, err) + require.NotNil(t, run) + engineHolder, ok := any(marker).(interface{ PebbleEngine() *pebbleengine.Engine }) + require.True(t, ok) + rec, err := engineHolder.PebbleEngine().GetSyncRunRecord(ctx, run.ID) + require.NoError(t, err) + require.False(t, rec.GetCompacted(), "sanity: the record must NOT carry the compacted flag for this test to prove the token gate") + compactedToken, err := BuildCompactedToken(rec.GetSyncToken(), CompactionTokenInput{ + Mode: "fold", + BaseSyncID: run.ID, + PartialSyncIDs: []string{"old-partial-1"}, + }) + require.NoError(t, err) + rec.SetSyncToken(compactedToken) + require.NoError(t, engineHolder.PebbleEngine().PutSyncRunRecord(ctx, rec)) + require.True(t, pebbleengine.MarkStoreDirty(marker), "the token write must persist through Close") + require.NoError(t, marker.Close(ctx)) + + // Warm sync against the doctored file: every lookup must MISS + // despite the intact manifest and the clean-looking run record. + sync2 := filepath.Join(tmpDir, "sync2.c1z") + missesBefore := mc.lookupMisses + runSourceCacheSync(ctx, t, mc, sync2, sync1, tmpDir) + require.Zero(t, mc.lookupHits, + "a previous sync whose token carries compaction provenance must never produce a source-cache lookup hit, even without the compacted flag") + require.Greater(t, mc.lookupMisses, missesBefore, + "sanity: the connector must have looked up and missed (no-op lookup installed)") +} diff --git a/pkg/tasks/local/syncer.go b/pkg/tasks/local/syncer.go index a36d65217..1b0a12b7b 100644 --- a/pkg/tasks/local/syncer.go +++ b/pkg/tasks/local/syncer.go @@ -166,7 +166,15 @@ func (m *localSyncer) Process(ctx context.Context, task *v1.Task, cc types.Conne // replayed rows may be wrong; resuming would keep them) and // re-run cold, without the previous-sync source. One retry: a // cold sync cannot fail this way again. See ErrReplayIntegrity. - ctxzap.Extract(ctx).Error("source-cache replay integrity failure; discarding replay state and re-running the sync cold", + // + // Unlike the service-mode runner (c1api), which retires its + // SDK-owned previous-sync spare on this failure, the local + // previous-sync file is USER-SUPPLIED and is deliberately left + // in place — deleting an operator's file is not ours to do. The + // consequence: if the file itself is bad, every future run that + // passes it will warm-fail and cold-retry again, so say so. + ctxzap.Extract(ctx).Error("source-cache replay integrity failure; discarding replay state and re-running the sync cold. "+ + "The previous-sync c1z is retained; if this recurs across runs, the file is implicated — replace it with a fresh sync's output or stop passing it", zap.Error(err), zap.String("previous_sync_c1z", m.previousSyncC1Z)) if rmErr := os.Remove(m.dbPath); rmErr != nil && !os.IsNotExist(rmErr) { From c42a300213e8cbb8a82484e9eda80e981678e920 Mon Sep 17 00:00:00 2001 From: Matt Kaniaris Date: Thu, 16 Jul 2026 08:25:43 -0600 Subject: [PATCH 12/17] centralize drop handling --- pkg/connectorbuilder/connectorbuilder.go | 60 ++- pkg/connectorbuilder/resource_syncer.go | 33 +- .../source_cache_slot_test.go | 50 ++ .../type_scoped_v1_adapter_test.go | 125 +++++ pkg/connectorrunner/runner.go | 1 + pkg/dotc1z/c1zstore/sync_meta_test.go | 35 ++ pkg/dotc1z/engine/pebble/engine.go | 30 ++ pkg/dotc1z/engine/pebble/ingest_repair.go | 310 ++++++++--- .../engine/pebble/ingest_repair_crash_test.go | 173 ++++++ pkg/dotc1z/engine/pebble/keys.go | 23 +- pkg/dotc1z/engine/pebble/mutation_paths.go | 195 +++++++ .../engine/pebble/mutation_paths_test.go | 111 ++++ pkg/dotc1z/engine/pebble/source_cache.go | 95 +++- .../engine/pebble/sync_stats_sidecar.go | 10 + pkg/dotc1z/pebble_store.go | 8 +- pkg/dotc1z/pebble_store_mutated_test.go | 64 +++ pkg/dotc1z/source_cache.go | 54 +- pkg/sourcecache/compat.go | 92 ++++ pkg/sync/expand/drop_stats.go | 102 ++++ pkg/sync/expand/drop_stats_test.go | 48 ++ pkg/sync/expand/expander.go | 21 +- pkg/sync/expand/topological_merge.go | 11 +- pkg/sync/gen_skew_inputs_test.go | 53 ++ pkg/sync/ingest_invariants.go | 84 ++- .../ingest_invariants_granularity_test.go | 28 +- .../ingest_invariants_referential_test.go | 171 ++++-- pkg/sync/ingest_invariants_scale_test.go | 3 +- pkg/sync/pebble_etag_replay_test.go | 5 + pkg/sync/source_cache.go | 81 ++- pkg/sync/source_cache_continuation.go | 50 +- pkg/sync/source_cache_continuation_test.go | 15 +- .../source_cache_equivalence_dirty_test.go | 509 ++++++++++++++++++ pkg/sync/source_cache_equivalence_test.go | 245 ++++++++- pkg/sync/source_cache_sideeffects_test.go | 235 ++++++++ pkg/sync/source_cache_skew_fixture_test.go | 83 +++ pkg/sync/source_cache_test.go | 12 +- pkg/sync/syncer.go | 72 ++- pkg/sync/testdata/README-old-fold.md | 33 ++ pkg/sync/testdata/old-fold-mixed-version.c1z | Bin 0 -> 25762 bytes pkg/sync/type_scoped.go | 10 + pkg/tasks/c1api/full_sync.go | 155 +++++- pkg/tasks/c1api/full_sync_spare_race_test.go | 124 +++++ pkg/tasks/c1api/manager.go | 16 + 43 files changed, 3407 insertions(+), 228 deletions(-) create mode 100644 pkg/connectorbuilder/type_scoped_v1_adapter_test.go create mode 100644 pkg/dotc1z/c1zstore/sync_meta_test.go create mode 100644 pkg/dotc1z/engine/pebble/ingest_repair_crash_test.go create mode 100644 pkg/dotc1z/engine/pebble/mutation_paths.go create mode 100644 pkg/dotc1z/engine/pebble/mutation_paths_test.go create mode 100644 pkg/dotc1z/pebble_store_mutated_test.go create mode 100644 pkg/sourcecache/compat.go create mode 100644 pkg/sync/expand/drop_stats.go create mode 100644 pkg/sync/expand/drop_stats_test.go create mode 100644 pkg/sync/gen_skew_inputs_test.go create mode 100644 pkg/sync/source_cache_equivalence_dirty_test.go create mode 100644 pkg/sync/source_cache_skew_fixture_test.go create mode 100644 pkg/sync/testdata/README-old-fold.md create mode 100644 pkg/sync/testdata/old-fold-mixed-version.c1z create mode 100644 pkg/tasks/c1api/full_sync_spare_race_test.go diff --git a/pkg/connectorbuilder/connectorbuilder.go b/pkg/connectorbuilder/connectorbuilder.go index 34c67f46d..782a4ce34 100644 --- a/pkg/connectorbuilder/connectorbuilder.go +++ b/pkg/connectorbuilder/connectorbuilder.go @@ -83,8 +83,17 @@ type builder struct { // sourceCacheMu: SetSourceCache (per-sync install/clear from the // runner) can race list RPCs reading it via syncOpAttrs. Access // through sourceCacheLookup / SetSourceCache only. - sourceCacheMu stdsync.RWMutex - sourceCache sourcecache.Lookup + sourceCacheMu stdsync.RWMutex + sourceCache sourcecache.Lookup + // sourceCacheInstalls / sourceCacheContended implement the same + // contention blinding as sourcecache.GRPCServer: the slot carries no + // sync-id routing, so with two live installs a lookup cannot be + // attributed to either sync and serving EITHER cross-wires them (one + // sync validates against the other's previous artifact and replays + // from its own — stale rows sealed as current). A miss is always + // safe, so on overlap every lookup misses until the slot drains. + sourceCacheInstalls int + sourceCacheContended bool metadataProvider MetadataProvider validateProvider ValidateProvider ticketManager TicketManagerLimited @@ -283,27 +292,42 @@ func WithSourceCache(lookup sourcecache.Lookup) Opt { // The slot is guarded: the runner supports concurrent tasks, so a set or // clear can race in-flight list RPCs reading the slot via syncOpAttrs. // The slot is still SINGLE — there is no sync-id routing — so two live -// syncs installing lookups cross-wire each other (one sync's connector -// calls resolve against the other's previous store) and the first to -// finish blinds the survivor when it clears. That topology isn't -// supported; warn loudly when an install lands on top of an existing -// non-noop lookup so it is visible instead of silently wrong. +// syncs installing lookups would cross-wire each other (one sync's +// connector would resolve validators against the other's previous store, +// then replay rows from its OWN store under a validator the other +// artifact vouched for — stale rows sealed as current). A miss is always +// safe (the connector fetches cold), so overlapping installs BLIND the +// slot — every lookup misses until all concurrent owners have cleared — +// mirroring sourcecache.GRPCServer's contention defense, and warn loudly +// so the degraded overlap is visible. func (b *builder) SetSourceCache(ctx context.Context, lookup sourcecache.Lookup) { - installing := lookup != nil - if lookup == nil { - lookup = sourcecache.NoopLookup{} - } b.sourceCacheMu.Lock() - prev := b.sourceCache - b.sourceCache = lookup - b.sourceCacheMu.Unlock() - if !installing || prev == nil { + if lookup == nil { + // Clear: the finished sync's lookup must not serve late RPCs. The + // slot stays occupied (NoopLookup, never back to nil) so the + // continuation's "never set" signal remains accurate. + if b.sourceCacheInstalls > 0 { + b.sourceCacheInstalls-- + } + if b.sourceCacheInstalls == 0 { + // Slot fully drained: the next sync starts clean. + b.sourceCacheContended = false + } + b.sourceCache = sourcecache.NoopLookup{} + b.sourceCacheMu.Unlock() return } - if _, prevIsNoop := prev.(sourcecache.NoopLookup); !prevIsNoop { - ctxzap.Extract(ctx).Warn("source cache: installing a lookup over an existing one; " + - "concurrent syncs against one connector share a single lookup slot and will cross-wire or blind each other") + b.sourceCacheInstalls++ + if b.sourceCacheInstalls > 1 || b.sourceCacheContended { + b.sourceCacheContended = true + b.sourceCache = sourcecache.NoopLookup{} + b.sourceCacheMu.Unlock() + ctxzap.Extract(ctx).Warn("source cache: concurrent syncs share this connector's single lookup slot; " + + "serving misses to every sync (cold fetches) until the overlap drains, to avoid cross-wiring validators between previous-sync artifacts") + return } + b.sourceCache = lookup + b.sourceCacheMu.Unlock() } // sourceCacheLookup returns the active slot value (nil when never set — diff --git a/pkg/connectorbuilder/resource_syncer.go b/pkg/connectorbuilder/resource_syncer.go index 4f0ff6e6b..5c6adf654 100644 --- a/pkg/connectorbuilder/resource_syncer.go +++ b/pkg/connectorbuilder/resource_syncer.go @@ -616,8 +616,39 @@ func (b *builder) ListGrants(ctx context.Context, request *v2.GrantsServiceListG return resp, nil } +// newResourceSyncerV1toV2 adapts a V1 ResourceSyncer to the V2 surface. +// The adapter must FORWARD the optional type-scoped interfaces: routing +// and validation type-assert TypeScopedGrantsSyncer / +// TypeScopedEntitlementsSyncer, validation against the ORIGINAL syncer +// but routing against the STORED (wrapped) one — a plain wrapper would +// pass construction and then fail every type-scoped call at runtime +// with InvalidArgument. The composite shapes below satisfy exactly the +// interfaces the wrapped syncer does, so the tsOk routing checks stay +// meaningful for syncers that never opted in. func newResourceSyncerV1toV2(rb ResourceSyncer) ResourceSyncerV2 { - return &resourceSyncerV1toV2{rb: rb} + base := &resourceSyncerV1toV2{rb: rb} + tsg, hasTSG := rb.(TypeScopedGrantsSyncer) + tse, hasTSE := rb.(TypeScopedEntitlementsSyncer) + switch { + case hasTSG && hasTSE: + return &struct { + *resourceSyncerV1toV2 + TypeScopedGrantsSyncer + TypeScopedEntitlementsSyncer + }{base, tsg, tse} + case hasTSG: + return &struct { + *resourceSyncerV1toV2 + TypeScopedGrantsSyncer + }{base, tsg} + case hasTSE: + return &struct { + *resourceSyncerV1toV2 + TypeScopedEntitlementsSyncer + }{base, tse} + default: + return base + } } type resourceSyncerV1toV2 struct { diff --git a/pkg/connectorbuilder/source_cache_slot_test.go b/pkg/connectorbuilder/source_cache_slot_test.go index 1dbf78775..bede40a41 100644 --- a/pkg/connectorbuilder/source_cache_slot_test.go +++ b/pkg/connectorbuilder/source_cache_slot_test.go @@ -54,3 +54,53 @@ func TestSetSourceCacheConcurrentWithListCalls(t *testing.T) { close(stop) wg.Wait() } + +// slotTestLookup is a non-noop lookup whose hits identify their owner. +type slotTestLookup struct{ validator string } + +func (l slotTestLookup) Lookup(context.Context, sourcecache.RowKind, string) (sourcecache.Entry, bool, error) { + return sourcecache.Entry{CacheValidator: l.validator}, true, nil +} + +// TestSetSourceCacheContentionBlindsSlot pins the in-process slot's +// contention defense (mirroring sourcecache.GRPCServer): two live +// installs cannot be attributed by lookups (no sync-id routing), so the +// slot must serve misses to EVERYONE until every owner clears — serving +// either lookup would cross-wire validators between previous-sync +// artifacts. After the overlap drains, a fresh install serves normally. +func TestSetSourceCacheContentionBlindsSlot(t *testing.T) { + ctx := context.Background() + b := &builder{} + + lookupHits := func() (string, bool) { + lk := b.sourceCacheLookup() + require.NotNil(t, lk) + entry, found, err := lk.Lookup(ctx, sourcecache.RowKindGrants, "scope") + require.NoError(t, err) + return entry.CacheValidator, found + } + + // Single owner: served. + b.SetSourceCache(ctx, slotTestLookup{validator: "sync-a"}) + v, found := lookupHits() + require.True(t, found) + require.Equal(t, "sync-a", v) + + // Second concurrent owner: blinded for everyone. + b.SetSourceCache(ctx, slotTestLookup{validator: "sync-b"}) + _, found = lookupHits() + require.False(t, found, "contended slot must serve misses, not either sync's lookup") + + // One owner clears; the survivor must STAY blinded (the slot cannot + // know which lookup the survivor owns). + b.SetSourceCache(ctx, nil) + _, found = lookupHits() + require.False(t, found, "slot must stay blinded until the overlap fully drains") + + // Fully drained: a fresh install serves again. + b.SetSourceCache(ctx, nil) + b.SetSourceCache(ctx, slotTestLookup{validator: "sync-c"}) + v, found = lookupHits() + require.True(t, found, "a drained slot must serve a fresh single owner") + require.Equal(t, "sync-c", v) +} diff --git a/pkg/connectorbuilder/type_scoped_v1_adapter_test.go b/pkg/connectorbuilder/type_scoped_v1_adapter_test.go new file mode 100644 index 000000000..9e84deee9 --- /dev/null +++ b/pkg/connectorbuilder/type_scoped_v1_adapter_test.go @@ -0,0 +1,125 @@ +package connectorbuilder + +// The V1→V2 resource-syncer adapter must FORWARD the optional +// type-scoped interfaces. Registration validates the ORIGINAL syncer +// (validateTypeScopedRegistration) but routing type-asserts the STORED +// value — before the composite adapter shapes in newResourceSyncerV1toV2, +// a V1 syncer implementing TypeScopedGrantsSyncer / +// TypeScopedEntitlementsSyncer passed construction and then failed every +// type-scoped call at runtime with InvalidArgument. + +import ( + "context" + "testing" + + "github.com/stretchr/testify/require" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" + + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + "github.com/conductorone/baton-sdk/pkg/annotations" + "github.com/conductorone/baton-sdk/pkg/pagination" + "github.com/conductorone/baton-sdk/pkg/types/resource" +) + +// typeScopedV1Syncer is a V1 ResourceSyncer (pagination.Token +// signatures) that also implements both type-scoped interfaces and +// carries both annotations on its resource type. +type typeScopedV1Syncer struct { + grantCalls int + entCalls int +} + +func (s *typeScopedV1Syncer) ResourceType(context.Context) *v2.ResourceType { + return v2.ResourceType_builder{ + Id: "group", + DisplayName: "Group", + Annotations: annotations.New(&v2.TypeScopedGrants{}, &v2.TypeScopedEntitlements{}), + }.Build() +} + +func (s *typeScopedV1Syncer) List(context.Context, *v2.ResourceId, *pagination.Token) ([]*v2.Resource, string, annotations.Annotations, error) { + return nil, "", nil, nil +} + +func (s *typeScopedV1Syncer) Entitlements(context.Context, *v2.Resource, *pagination.Token) ([]*v2.Entitlement, string, annotations.Annotations, error) { + return nil, "", nil, nil +} + +func (s *typeScopedV1Syncer) Grants(context.Context, *v2.Resource, *pagination.Token) ([]*v2.Grant, string, annotations.Annotations, error) { + return nil, "", nil, nil +} + +func (s *typeScopedV1Syncer) GrantsForResourceType(_ context.Context, resourceTypeID string, _ resource.SyncOpAttrs) ([]*v2.Grant, *resource.SyncOpResults, error) { + s.grantCalls++ + r := v2.Resource_builder{ + Id: v2.ResourceId_builder{ResourceType: resourceTypeID, Resource: "g1"}.Build(), + }.Build() + return []*v2.Grant{ + v2.Grant_builder{ + Id: "grant-type-1", + Entitlement: v2.Entitlement_builder{Id: "ent-1", Resource: r}.Build(), + Principal: v2.Resource_builder{Id: v2.ResourceId_builder{ResourceType: "user", Resource: "u1"}.Build()}.Build(), + }.Build(), + }, &resource.SyncOpResults{}, nil +} + +func (s *typeScopedV1Syncer) EntitlementsForResourceType(_ context.Context, resourceTypeID string, _ resource.SyncOpAttrs) ([]*v2.Entitlement, *resource.SyncOpResults, error) { + s.entCalls++ + r := v2.Resource_builder{ + Id: v2.ResourceId_builder{ResourceType: resourceTypeID, Resource: "g1"}.Build(), + }.Build() + return []*v2.Entitlement{ + v2.Entitlement_builder{Id: "ent-type-1", Resource: r}.Build(), + }, &resource.SyncOpResults{}, nil +} + +func typeScopedV1GrantsRequest(annos annotations.Annotations) *v2.GrantsServiceListGrantsRequest { + return v2.GrantsServiceListGrantsRequest_builder{ + Resource: v2.Resource_builder{ + Id: v2.ResourceId_builder{ResourceType: "group", Resource: "group"}.Build(), + }.Build(), + Annotations: annos, + }.Build() +} + +// TestTypeScopedV1SyncerRoutesThroughAdapter pins the adapter +// forwarding: a V1 syncer's type-scoped methods must be reachable +// through the stored (wrapped) syncer for both phases. +func TestTypeScopedV1SyncerRoutesThroughAdapter(t *testing.T) { + ctx := context.Background() + ts := &typeScopedV1Syncer{} + conn, err := NewConnector(ctx, newTestConnector([]ResourceSyncer{ts})) + require.NoError(t, err, "a V1 type-scoped syncer must pass registration") + + gResp, err := conn.ListGrants(ctx, typeScopedV1GrantsRequest(annotations.New(&v2.TypeScopedGrants{}))) + require.NoError(t, err, "type-scoped grants routing must reach the V1 syncer through the adapter") + require.Len(t, gResp.GetList(), 1) + require.Equal(t, "grant-type-1", gResp.GetList()[0].GetId()) + require.Equal(t, 1, ts.grantCalls) + + eResp, err := conn.ListEntitlements(ctx, typeScopedEntitlementsRequest(annotations.New(&v2.TypeScopedEntitlements{}))) + require.NoError(t, err, "type-scoped entitlements routing must reach the V1 syncer through the adapter") + require.Len(t, eResp.GetList(), 1) + require.Equal(t, "ent-type-1", eResp.GetList()[0].GetId()) + require.Equal(t, 1, ts.entCalls) +} + +// TestTypeScopedV1AdapterDoesNotOverclaim: a plain V1 syncer (no +// type-scoped interfaces) wrapped by the adapter must still FAIL the +// type-scoped routing assertions — the composite shapes must not make +// every adapted syncer claim the interfaces. +func TestTypeScopedV1AdapterDoesNotOverclaim(t *testing.T) { + adapted := newResourceSyncerV1toV2(newTestResourceSyncer("group")) + _, isTSG := adapted.(TypeScopedGrantsSyncer) + require.False(t, isTSG) + _, isTSE := adapted.(TypeScopedEntitlementsSyncer) + require.False(t, isTSE) + + ctx := context.Background() + conn, err := NewConnector(ctx, newTestConnector([]ResourceSyncer{newTestResourceSyncer("group")})) + require.NoError(t, err) + _, err = conn.ListGrants(ctx, typeScopedV1GrantsRequest(annotations.New(&v2.TypeScopedGrants{}))) + require.Error(t, err) + require.Equal(t, codes.InvalidArgument, status.Code(err)) +} diff --git a/pkg/connectorrunner/runner.go b/pkg/connectorrunner/runner.go index 4db9cf18a..b4295036b 100644 --- a/pkg/connectorrunner/runner.go +++ b/pkg/connectorrunner/runner.go @@ -1166,6 +1166,7 @@ func NewConnectorRunner(ctx context.Context, c types.ConnectorServer, opts ...Op cfg.storageEngine, runner.taskConcurrency, keepPreviousSyncC1Z, + cfg.previousSyncC1Z, ) if err != nil { return nil, err diff --git a/pkg/dotc1z/c1zstore/sync_meta_test.go b/pkg/dotc1z/c1zstore/sync_meta_test.go new file mode 100644 index 000000000..e734ba4bb --- /dev/null +++ b/pkg/dotc1z/c1zstore/sync_meta_test.go @@ -0,0 +1,35 @@ +package c1zstore + +// UsableAsReplaySource is the metadata gate two product non-goals hang +// off of: SQLite artifacts and compacted artifacts must never serve +// source-cache replay. The predicate itself was previously only pinned +// indirectly (through syncer degrade tests); this is its direct truth +// table. + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/conductorone/baton-sdk/pkg/connectorstore" +) + +func TestUsableAsReplaySource(t *testing.T) { + cases := []struct { + name string + run *SyncRun + want bool + }{ + {"nil run", nil, false}, + {"full sync", &SyncRun{Type: connectorstore.SyncTypeFull}, true}, + {"full but compacted", &SyncRun{Type: connectorstore.SyncTypeFull, Compacted: true}, false}, + {"partial sync", &SyncRun{Type: connectorstore.SyncTypePartial}, false}, + {"resources-only sync", &SyncRun{Type: connectorstore.SyncTypeResourcesOnly}, false}, + {"empty type", &SyncRun{}, false}, + } + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + require.Equal(t, tc.want, tc.run.UsableAsReplaySource()) + }) + } +} diff --git a/pkg/dotc1z/engine/pebble/engine.go b/pkg/dotc1z/engine/pebble/engine.go index a75bd5b03..4e3554b8f 100644 --- a/pkg/dotc1z/engine/pebble/engine.go +++ b/pkg/dotc1z/engine/pebble/engine.go @@ -65,6 +65,16 @@ type Engine struct { closing atomic.Bool // strict write-barrier flag, read on every Writer call closeMu sync.Mutex + // mutated latches when ANY write closure runs (see + // withWriteAllowSealed) — the engine-derived backstop for the + // store-level dirty bit. The pebbleStore wrappers mark dirty + // per-method by convention, and every missed wrapper is a lost + // envelope save (the replay pre-clear was one). Latching here is + // conservative in the safe direction: a spurious save costs seconds, + // a skipped one loses the sync. Set even when the closure errors — + // chunked paths may have committed batches before failing. + mutated atomic.Bool + // computedStats holds caller-computed stats records stashed via // StashComputedSyncStats, keyed by sync_id. PersistSyncStats pops // and persists the stashed record instead of re-scanning the @@ -86,6 +96,13 @@ type Engine struct { // as one sorted SST at EndSync — see BuildDeferredGrantIndexes). deferredIdxPending atomic.Bool + // testDropChunkHook, when non-nil, runs after every chunked + // dangling-reference drop batch commit (ingest_repair.go). Test-only: + // deterministic crash injection at the exact seam between a committed + // chunk and the drop's completion, pinning the atomicity contract of + // stageSourceCacheScopeInvalidationLocked. + testDropChunkHook func() error + // grantDigestsPresent reports whether the digest keyspace holds any // nodes — i.e. whether a grant mutation must invalidate the touched // entitlement's digest + hash-index ranges @@ -633,9 +650,22 @@ func (e *Engine) withWriteAllowSealed(fn func() error) error { } e.writeMu.Lock() defer e.writeMu.Unlock() + // Latch BEFORE the closure runs, not on success: a failing closure + // may have committed chunks (see the streaming drops), and a lost + // envelope save is the unrecoverable direction. + e.mutated.Store(true) return fn() } +// Mutated reports whether any write closure has run since Open — the +// engine-derived dirty signal. Store-level Close saves the envelope when +// EITHER its own per-method dirty bit or this latch is set, so an engine +// mutation path missing a store wrapper can no longer silently skip the +// save. +func (e *Engine) Mutated() bool { + return e.mutated.Load() +} + func (e *Engine) Save(ctx context.Context, dest string) error { return errors.New("pebble engine: Save requires the dotc1z.Save shim (envelope write); use CheckpointTo for direct directory access") } diff --git a/pkg/dotc1z/engine/pebble/ingest_repair.go b/pkg/dotc1z/engine/pebble/ingest_repair.go index dffbb72f6..ef32fafc0 100644 --- a/pkg/dotc1z/engine/pebble/ingest_repair.go +++ b/pkg/dotc1z/engine/pebble/ingest_repair.go @@ -27,17 +27,24 @@ import ( ) // EnsureGrantIndexes runs the deferred grant-index build NOW if one is -// pending. The by_principal index is inline for connector-emitted grants -// but deferred for expansion/synth writes; the dangling-principal scan -// (I9) needs it complete. Calling early is near-free: the same build -// would run at EndSync anyway — this just moves it before the invariant -// seam (rows dropped afterward maintain the indexes inline and stage -// digest invalidation, which EndSync's repair pass heals). +// pending, and retires the pending marker. The by_principal index is +// inline for connector-emitted grants but deferred for expansion/synth +// writes; the dangling-principal scan (I9) needs it complete. This is +// the SAME build EndSync would run, so the marker must clear here or +// EndSync repeats the whole O(grants) scan/build a second time. Rows +// dropped afterward maintain the indexes inline and stage digest +// invalidation, which EndSync's RepairMissingGrantDigests pass heals; +// grant drops also invalidate the stashed deferred grant stats +// (invalidateDeferredGrantStats) so the stats sidecar recounts instead +// of reporting dropped rows. func (e *Engine) EnsureGrantIndexes(ctx context.Context) error { if !e.deferredIdxPending.Load() { return nil } - return e.BuildDeferredGrantIndexes(ctx) + if err := e.BuildDeferredGrantIndexes(ctx); err != nil { + return err + } + return e.clearDeferredIdxPending() } // ForEachDanglingGrantPrincipal visits each distinct principal referenced @@ -194,6 +201,78 @@ func (e *Engine) getGrantRecordByIdentity(id grantIdentity) (*v3.GrantRecord, er // row. const dropBatchRows = 4096 +// stageSourceCacheScopeInvalidationLocked stages a dropped row's +// source-scope manifest invalidation INTO the batch that carries the +// row's delete. A drop deletes rows out from under a manifest entry +// whose validator still vouches for the scope's FULL upstream row set; +// left intact, the next sync's lookup would hit, the connector would +// 304, and the replay would carry the sanitized subset forward — +// permanently, even after the dangling rows' referent reappears upstream +// (a cold sync would restore them; a warm one never would). Marking the +// entry invalidated makes the next lookup miss, so exactly the +// dropped-from scopes re-fetch cold and warm/cold equivalence is +// preserved. +// +// ORDERING IS LOAD-BEARING: the invalidation must ride the SAME batch as +// the scope's FIRST staged delete, never a later write. Batches commit +// atomically and the WAL preserves commit order (a crash loses only a +// suffix), so "a committed delete for a scope implies its committed +// invalidation" holds through any crash or mid-loop error — including a +// crash between chunked commits. Invalidating after the deletes (a +// separate write) would open the one unrecoverable window: rows durably +// gone, validator retained, and the resumed invariant pass finds nothing +// dangling to re-trigger the invalidation — the next sync 304s the +// sanitized subset forever. The inverse leak (invalidation committed, +// deletes lost or unfinished) is safe: the scope re-fetches cold once, +// and the resumed pass re-detects whatever still dangles. +// +// The entry is kept (marked, not deleted) so the scope's SURVIVING +// stamped rows don't read as an I6 orphan (lost manifest write), and the +// stale validator stays visible to the audit tooling. Scopes with no +// manifest entry are skipped: nothing vouches for them, so there is +// nothing to invalidate (and writing one would hide a real I6 orphan). +// Idempotent — re-marking an invalidated entry is a no-op — so resumed +// syncs re-reaching the invariant seam converge. staged dedupes within +// one drop call; the read sees committed state, so a scope invalidated +// by an earlier call (or an earlier committed chunk) is skipped too. +// Caller must hold the write lock (runs inside the drop's withWrite +// closure). +func (e *Engine) stageSourceCacheScopeInvalidationLocked(batch *pebble.Batch, rowKind, scopeKey string, staged map[string]struct{}) error { + if scopeKey == "" { + return nil + } + if _, ok := staged[scopeKey]; ok { + return nil + } + staged[scopeKey] = struct{}{} + key := encodeSourceCacheEntryKey(rowKind, scopeKey) + val, closer, err := e.db.Get(key) + if err != nil { + if errors.Is(err, pebble.ErrNotFound) { + return nil + } + return fmt.Errorf("invalidate source-cache scope %q: %w", scopeKey, err) + } + rec := &v3.SourceCacheEntryRecord{} + unmarshalErr := unmarshalRecord(val, rec) + closer.Close() + if unmarshalErr != nil { + return fmt.Errorf("invalidate source-cache scope %q: unmarshal: %w", scopeKey, unmarshalErr) + } + if rec.GetInvalidated() { + return nil + } + rec.SetInvalidated(true) + newVal, err := marshalRecord(rec) + if err != nil { + return fmt.Errorf("invalidate source-cache scope %q: marshal: %w", scopeKey, err) + } + if err := batch.Set(key, newVal, nil); err != nil { + return fmt.Errorf("invalidate source-cache scope %q: %w", scopeKey, err) + } + return nil +} + // dropWriteOpts returns the write options for streaming drops. func (e *Engine) dropWriteOpts() *pebble.WriteOptions { if e.IsFreshSync() { @@ -202,11 +281,124 @@ func (e *Engine) dropWriteOpts() *pebble.WriteOptions { return writeOpts(e.opts.durability) } +// dropBatcher is the single choke point for dangling-reference row +// deletion. Every dropped row carries cross-cutting obligations — +// secondary-index cleanup, digest invalidation, source-cache manifest +// invalidation, chunked-commit crash ordering — and every one of them +// has produced a bug when a call site was left to remember it. The +// batcher makes the obligations structural: a drop arm stages rows +// through stageGrantDelete / stageEntitlementDelete and calls finish; +// it never touches a raw batch, so it cannot stage a delete without its +// obligations or commit them in an unsafe order. +// +// Crash contract (see stageSourceCacheScopeInvalidationLocked): a +// scope's manifest invalidation rides the same batch as its first +// staged delete, chunks commit in WAL order, and the test hook fires +// between chunk commits so the crash-window tests can land exactly +// there. Callers must hold the engine write lock (the batcher runs +// inside the drop's withWrite closure). +type dropBatcher struct { + e *Engine + ctx context.Context + opts *pebble.WriteOptions + batch *pebble.Batch + rowsInBatch int + stagedScopes map[string]struct{} + // committedRows counts rows whose chunk commit LANDED — consumed by + // obligations that must fire even when a later chunk errors (the + // entitlement bare-id keyspace note). + committedRows int64 +} + +func (e *Engine) newDropBatcher(ctx context.Context) *dropBatcher { + return &dropBatcher{ + e: e, + ctx: ctx, + opts: e.dropWriteOpts(), + batch: e.db.NewBatch(), + stagedScopes: map[string]struct{}{}, + } +} + +// close releases the open batch; safe after finish (callers defer it). +func (b *dropBatcher) close() { _ = b.batch.Close() } + +// stageGrantDelete stages one grant row's COMPLETE drop: manifest +// invalidation for its source scope (first occurrence, same batch), +// every secondary index + digest invalidation derived from the raw +// value, and the primary delete; then rotates the chunk if full. +func (b *dropBatcher) stageGrantDelete(priKey, rawValue []byte, sourceScopeKey string) error { + if err := b.ctx.Err(); err != nil { + return err + } + if err := b.e.stageSourceCacheScopeInvalidationLocked(b.batch, "grants", sourceScopeKey, b.stagedScopes); err != nil { + return err + } + if err := b.e.deleteGrantIndexesRaw(b.batch, "", rawValue); err != nil { + return err + } + if err := b.batch.Delete(priKey, nil); err != nil { + return err + } + return b.rotate() +} + +// stageEntitlementDelete is stageGrantDelete's entitlement twin (the +// scope entry is the only entitlement secondary index). +func (b *dropBatcher) stageEntitlementDelete(id entitlementIdentity, priKey []byte, scopeKey string) error { + if err := b.ctx.Err(); err != nil { + return err + } + if err := b.e.stageSourceCacheScopeInvalidationLocked(b.batch, "entitlements", scopeKey, b.stagedScopes); err != nil { + return err + } + if scopeKey != "" { + if err := b.batch.Delete(encodeEntitlementBySourceScopeIndexKey(scopeKey, id), nil); err != nil { + return err + } + } + if err := b.batch.Delete(priKey, nil); err != nil { + return err + } + return b.rotate() +} + +func (b *dropBatcher) rotate() error { + b.rowsInBatch++ + if b.rowsInBatch < dropBatchRows { + return nil + } + if err := b.batch.Commit(b.opts); err != nil { + return err + } + b.committedRows += int64(b.rowsInBatch) + _ = b.batch.Close() + b.batch = b.e.db.NewBatch() + b.rowsInBatch = 0 + if b.e.testDropChunkHook != nil { + return b.e.testDropChunkHook() + } + return nil +} + +// finish commits the final partial chunk. +func (b *dropBatcher) finish() error { + if err := b.batch.Commit(b.opts); err != nil { + return err + } + b.committedRows += int64(b.rowsInBatch) + b.rowsInBatch = 0 + return nil +} + // DeleteGrantsForPrincipal deletes every grant of one principal EXCEPT // rows carrying ExternalResourceMatch* annotations (unprocessed match // carriers are evidence of a missing external-resource file, not bad // data). Streams one chunked batch pass — never one commit per row. -// Returns (deleted, skippedMatchAnnotated). +// Dropped rows' source scopes get their manifest entries invalidated +// atomically with each scope's first staged delete (see +// stageSourceCacheScopeInvalidationLocked). Returns +// (deleted, skippedMatchAnnotated). func (e *Engine) DeleteGrantsForPrincipal(ctx context.Context, principalRT, principalID string) (int64, int64, error) { indexPrefix := encodeGrantByPrincipalPrefix(principalRT, principalID) @@ -221,10 +413,8 @@ func (e *Engine) DeleteGrantsForPrincipal(ctx context.Context, principalRT, prin } defer iter.Close() - opts := e.dropWriteOpts() - batch := e.db.NewBatch() - defer func() { _ = batch.Close() }() - rowsInBatch := 0 + b := e.newDropBatcher(ctx) + defer b.close() for iter.First(); iter.Valid(); iter.Next() { if err := ctx.Err(); err != nil { @@ -237,11 +427,11 @@ func (e *Engine) DeleteGrantsForPrincipal(ctx context.Context, principalRT, prin var comps [4]string off := 0 for i := range comps { - b, next, ok := codec.DecodeTupleStringAlias(tail, off) + c, next, ok := codec.DecodeTupleStringAlias(tail, off) if !ok { return fmt.Errorf("by_principal index: malformed key tail %x", iter.Key()) } - comps[i] = string(b) + comps[i] = string(c) off = next + 1 } id := grantIdentity{ @@ -272,33 +462,24 @@ func (e *Engine) DeleteGrantsForPrincipal(ctx context.Context, principalRT, prin skipped++ continue } - if err := e.deleteGrantIndexesRaw(batch, "", val); err != nil { - closer.Close() - return err - } + stageErr := b.stageGrantDelete(priKey, val, rec.GetSourceScopeKey()) closer.Close() - if err := batch.Delete(priKey, nil); err != nil { - return err + if stageErr != nil { + return stageErr } deleted++ - rowsInBatch++ - if rowsInBatch >= dropBatchRows { - if err := batch.Commit(opts); err != nil { - return err - } - _ = batch.Close() - batch = e.db.NewBatch() - rowsInBatch = 0 - } } if err := iter.Error(); err != nil { return err } - return batch.Commit(opts) + return b.finish() }) if err != nil { return 0, 0, err } + if deleted > 0 { + e.invalidateDeferredGrantStats() + } return deleted, skipped, nil } @@ -311,7 +492,10 @@ func (e *Engine) DeleteGrantsForPrincipal(ctx context.Context, principalRT, prin // a connector-owned grant under the same missing entitlement is a // dangling reference and drops. Streams the primary prefix with values in // hand (index cleanup derives from the value), chunked batches, no -// per-row commits. Returns (deleted, skippedInsertFact). +// per-row commits. Dropped rows' source scopes get their manifest +// entries invalidated atomically with each scope's first staged delete +// (see stageSourceCacheScopeInvalidationLocked). Returns +// (deleted, skippedInsertFact). func (e *Engine) DeleteGrantsForEntitlement(ctx context.Context, entitlementID, entResourceTypeID, entResourceID string) (int64, int64, error) { entID := entitlementIdentityFromParts(entResourceTypeID, entResourceID, entitlementID) prefix := encodeGrantPrimaryEntitlementIdentityPrefix( @@ -328,10 +512,8 @@ func (e *Engine) DeleteGrantsForEntitlement(ctx context.Context, entitlementID, } defer iter.Close() - opts := e.dropWriteOpts() - batch := e.db.NewBatch() - defer func() { _ = batch.Close() }() - rowsInBatch := 0 + b := e.newDropBatcher(ctx) + defer b.close() for iter.First(); iter.Valid(); iter.Next() { if err := ctx.Err(); err != nil { @@ -345,42 +527,40 @@ func (e *Engine) DeleteGrantsForEntitlement(ctx context.Context, entitlementID, skipped++ continue } - if err := e.deleteGrantIndexesRaw(batch, "", iter.Value()); err != nil { - return err + _, _, _, _, _, _, sourceScopeKey, scanErr := scanGrantIndexFieldsRaw(iter.Value()) + if scanErr != nil { + return fmt.Errorf("dangling entitlement drop: scan scope: %w", scanErr) } key := iter.Key() k := make([]byte, len(key)) copy(k, key) - if err := batch.Delete(k, nil); err != nil { + if err := b.stageGrantDelete(k, iter.Value(), sourceScopeKey); err != nil { return err } deleted++ - rowsInBatch++ - if rowsInBatch >= dropBatchRows { - if err := batch.Commit(opts); err != nil { - return err - } - _ = batch.Close() - batch = e.db.NewBatch() - rowsInBatch = 0 - } } if err := iter.Error(); err != nil { return err } - return batch.Commit(opts) + return b.finish() }) if err != nil { return 0, 0, err } + if deleted > 0 { + e.invalidateDeferredGrantStats() + } return deleted, skipped, nil } // DeleteEntitlementsForResource deletes every entitlement row under one // resource — the drop arm for entitlements referencing a resource with no -// row. Returns the count and up to maxIDs deleted external ids for the -// caller's aggregate report; maxIDs <= 0 collects none (callers pass a -// shrinking example budget, so the guard below doubles as the clamp). +// row. Dropped rows' source scopes get their manifest entries +// invalidated atomically with each scope's first staged delete (see +// stageSourceCacheScopeInvalidationLocked). Returns the count +// and up to maxIDs deleted external ids for the caller's aggregate +// report; maxIDs <= 0 collects none (callers pass a shrinking example +// budget, so the guard below doubles as the clamp). func (e *Engine) DeleteEntitlementsForResource(ctx context.Context, resourceTypeID, resourceID string, maxIDs int) (int64, []string, error) { prefix := encodeEntitlementPrimaryResourcePrefix(resourceTypeID, resourceID) type entRow struct { @@ -448,15 +628,19 @@ func (e *Engine) DeleteEntitlementsForResource(ctx context.Context, resourceType var deleted int64 var ids []string err = e.withWrite(func() error { - batch := e.db.NewBatch() - defer batch.Close() - for _, row := range rows { - if row.scopeKey != "" { - if err := batch.Delete(encodeEntitlementBySourceScopeIndexKey(row.scopeKey, row.id), nil); err != nil { - return err - } + b := e.newDropBatcher(ctx) + defer b.close() + // Keyed on commits that LANDED, not on success: a failure after a + // partial chunk commit has already mutated the entitlement + // keyspace, so the bare-id lookup map must be invalidated even as + // the error unwinds (same contract as the replay pre-clear). + defer func() { + if b.committedRows > 0 { + e.noteEntitlementKeyspaceWrite() } - if err := batch.Delete(row.primaryKy, nil); err != nil { + }() + for _, row := range rows { + if err := b.stageEntitlementDelete(row.id, row.primaryKy, row.scopeKey); err != nil { return err } deleted++ @@ -464,13 +648,7 @@ func (e *Engine) DeleteEntitlementsForResource(ctx context.Context, resourceType ids = append(ids, row.id.externalID()) } } - if err := batch.Commit(e.dropWriteOpts()); err != nil { - return err - } - if deleted > 0 { - e.noteEntitlementKeyspaceWrite() - } - return nil + return b.finish() }) return deleted, ids, err } diff --git a/pkg/dotc1z/engine/pebble/ingest_repair_crash_test.go b/pkg/dotc1z/engine/pebble/ingest_repair_crash_test.go new file mode 100644 index 000000000..6105053f2 --- /dev/null +++ b/pkg/dotc1z/engine/pebble/ingest_repair_crash_test.go @@ -0,0 +1,173 @@ +package pebble + +// Deterministic crash test for the dangling-reference drops' manifest +// invalidation. The drops delete rows in chunked batches; a crash (or +// mid-loop error) between a committed chunk and the drop's completion +// must NEVER leave a scope with durably deleted rows and an intact +// manifest validator — the resumed invariant pass would find nothing +// dangling to re-trigger the invalidation, and every later sync would +// 304-replay the sanitized subset (the exact permanent-divergence bug +// manifest invalidation exists to prevent). The contract +// (stageSourceCacheScopeInvalidationLocked): the invalidation rides the +// SAME batch as the scope's first staged delete, so batch atomicity plus +// WAL prefix ordering make "committed delete ⇒ committed invalidation" +// hold through any crash point. + +import ( + "context" + "errors" + "fmt" + "testing" + + "github.com/stretchr/testify/require" + + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + "github.com/conductorone/baton-sdk/pkg/connectorstore" + "github.com/conductorone/baton-sdk/pkg/sourcecache" +) + +// TestDanglingDropCrashBetweenChunksKeepsInvalidation crashes +// DeleteGrantsForEntitlement after its FIRST committed chunk (via the +// engine's test hook — the in-process analog of a process kill at that +// seam) and asserts the half-dropped scope's manifest entry is already +// invalidated. The re-run (the resumed sync's idempotent invariant pass) +// then converges: remaining rows drop, the entry stays invalidated. +func TestDanglingDropCrashBetweenChunksKeepsInvalidation(t *testing.T) { + ctx := context.Background() + a := newAdapter(t) + _, err := a.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + e := a.PebbleEngine() + + // More than one drop chunk of grants under ONE dangling entitlement + // (scGrant's group:g1:member), all stamped with one scope that holds + // a valid manifest entry. + const total = dropBatchRows + 500 + scopedCtx := sourcecache.WithScope(ctx, scopeA) + const putChunk = 1000 + for start := 0; start < total; start += putChunk { + end := min(start+putChunk, total) + page := make([]*v2.Grant, 0, end-start) + for i := start; i < end; i++ { + page = append(page, scGrant("member", fmt.Sprintf("u%05d", i), false)) + } + require.NoError(t, a.PutGrants(scopedCtx, page...)) + } + require.NoError(t, e.PutSourceCacheEntry(ctx, "grants", scopeA, "etag-v1")) + + scopePrefix := encodeGrantBySourceScopePrefix(scopeA) + before, err := e.countSourceScopeIndexRange(ctx, scopePrefix) + require.NoError(t, err) + require.Equal(t, int64(total), before) + + // Crash after the first committed chunk. + injected := errors.New("injected crash between drop chunks") + e.testDropChunkHook = func() error { return injected } + _, _, err = e.DeleteGrantsForEntitlement(ctx, "group:g1:member", "group", "g1") + require.ErrorIs(t, err, injected) + e.testDropChunkHook = nil + + // The crash landed between a committed chunk and completion: some + // rows are durably gone, some remain. + remaining, err := e.countSourceScopeIndexRange(ctx, scopePrefix) + require.NoError(t, err) + require.Equal(t, int64(total-dropBatchRows), remaining, + "exactly the first chunk must be committed at the crash point") + + // THE CONTRACT: the scope's manifest entry must already be + // invalidated — it rode the first chunk's batch. An intact validator + // here is the permanent-sanitized-replay bug. + rec, err := e.GetSourceCacheEntry(ctx, "grants", scopeA) + require.NoError(t, err) + require.True(t, rec.GetInvalidated(), + "a committed drop chunk without a committed invalidation would 304-replay the sanitized scope forever") + require.Equal(t, "etag-v1", rec.GetCacheValidator(), + "the stale validator is kept (marked, not erased) for the audit surface") + + // Resume: the idempotent re-run converges — remaining rows drop and + // the entry stays invalidated. + deleted, skipped, err := e.DeleteGrantsForEntitlement(ctx, "group:g1:member", "group", "g1") + require.NoError(t, err) + require.Zero(t, skipped) + require.Equal(t, int64(total-dropBatchRows), deleted) + after, err := e.countSourceScopeIndexRange(ctx, scopePrefix) + require.NoError(t, err) + require.Zero(t, after) + rec, err = e.GetSourceCacheEntry(ctx, "grants", scopeA) + require.NoError(t, err) + require.True(t, rec.GetInvalidated()) +} + +// TestManifestSnapshotInvalidationIsTyped pins the snapshot's typed +// invalidation verdict: validators are OPAQUE connector strings, so a +// legitimate validator that happens to end with any in-band sentinel +// must never be misclassified as invalidated (the audit once encoded +// invalidation as a "\x00invalidated" validator suffix — this is its +// regression test), and an invalidated entry must report its verdict in +// the typed field with the original validator intact. +func TestManifestSnapshotInvalidationIsTyped(t *testing.T) { + ctx := context.Background() + a := newAdapter(t) + _, err := a.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + e := a.PebbleEngine() + + // A hostile-but-legal validator: byte-identical to the old in-band + // sentinel encoding. + trapValidator := "W/\"etag\"\x00invalidated" + require.NoError(t, e.PutSourceCacheEntry(ctx, "grants", scopeA, trapValidator)) + + // A genuinely invalidated scope, via the drop path. + require.NoError(t, a.PutGrants(sourcecache.WithScope(ctx, scopeB), + scGrant("member", "alice", false))) + require.NoError(t, e.PutSourceCacheEntry(ctx, "grants", scopeB, "etag-b")) + _, _, err = e.DeleteGrantsForEntitlement(ctx, "group:g1:member", "group", "g1") + require.NoError(t, err) + + snap, err := e.SourceCacheManifestSnapshot(ctx) + require.NoError(t, err) + + trap := snap["grants\x00"+scopeA] + require.False(t, trap.Invalidated, + "a legitimate validator matching the old sentinel encoding must not be misclassified") + require.Equal(t, trapValidator, trap.CacheValidator, + "the opaque validator must round-trip byte-exact") + + dropped := snap["grants\x00"+scopeB] + require.True(t, dropped.Invalidated, "the dropped-from scope must report its typed verdict") + require.Equal(t, "etag-b", dropped.CacheValidator, + "invalidation must not mutate the stored validator") +} + +// TestDanglingDropCleanCompletionInvalidates pins the non-crash contract +// for all three drop arms: a completed drop leaves every touched scope's +// manifest entry invalidated (and only touched scopes — a skipped +// exempt-carrier's scope keeps its validator). +func TestDanglingDropCleanCompletionInvalidates(t *testing.T) { + ctx := context.Background() + a := newAdapter(t) + _, err := a.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + e := a.PebbleEngine() + + // Scope A: dangling grants (dropped). Scope B: untouched grants under + // a different entitlement (validator must survive). + require.NoError(t, a.PutGrants(sourcecache.WithScope(ctx, scopeA), + scGrant("member", "alice", false), scGrant("member", "bob", false))) + require.NoError(t, a.PutGrants(sourcecache.WithScope(ctx, scopeB), + scGrant("owner", "carol", false))) + require.NoError(t, e.PutSourceCacheEntry(ctx, "grants", scopeA, "etag-a")) + require.NoError(t, e.PutSourceCacheEntry(ctx, "grants", scopeB, "etag-b")) + + deleted, skipped, err := e.DeleteGrantsForEntitlement(ctx, "group:g1:member", "group", "g1") + require.NoError(t, err) + require.Zero(t, skipped) + require.Equal(t, int64(2), deleted) + + recA, err := e.GetSourceCacheEntry(ctx, "grants", scopeA) + require.NoError(t, err) + require.True(t, recA.GetInvalidated(), "the dropped-from scope must be invalidated") + recB, err := e.GetSourceCacheEntry(ctx, "grants", scopeB) + require.NoError(t, err) + require.False(t, recB.GetInvalidated(), "an untouched scope's validator must survive") +} diff --git a/pkg/dotc1z/engine/pebble/keys.go b/pkg/dotc1z/engine/pebble/keys.go index 06f611a51..bd6a1b973 100644 --- a/pkg/dotc1z/engine/pebble/keys.go +++ b/pkg/dotc1z/engine/pebble/keys.go @@ -536,12 +536,31 @@ func encodeSourceCacheEntryKey(rowKind, scopeKey string) []byte { return codec.AppendTupleStrings(buf, rowKind, scopeKey) } -// encodeSourceCachePrefix is the by-type prefix for all source-cache -// entries. +// encodeSourceCachePrefix is the by-type prefix for ALL source-cache +// state (manifest entries and the compat record). ClearSourceCacheEntries +// and ResetForNewSync cover this whole prefix. func encodeSourceCachePrefix() []byte { return []byte{versionV3, typeSourceCache} } +// encodeSourceCacheManifestPrefix is the sub-prefix for (row_kind, +// scope_key) MANIFEST entries only — the 0x00 discriminator every +// encodeSourceCacheEntryKey starts with. Manifest iterators (snapshot, +// clear-check) bound here so the compat record (0x01 discriminator) +// never unmarshals as a manifest entry. +func encodeSourceCacheManifestPrefix() []byte { + return []byte{versionV3, typeSourceCache, 0x00} +} + +// encodeSourceCacheCompatKey is the singleton replay-compatibility key +// record for the sync (see SourceCacheCompatRecord): one key under the +// source-cache prefix with a 0x01 discriminator, so the fold's +// ClearSourceCacheEntries and ResetForNewSync wipe it with the manifest +// while manifest iterators never see it. +func encodeSourceCacheCompatKey() []byte { + return []byte{versionV3, typeSourceCache, 0x01, 'c', 'o', 'm', 'p', 'a', 't'} +} + // --- ResourceType --- // encodeResourceTypeKey returns the primary key for a resource_type: diff --git a/pkg/dotc1z/engine/pebble/mutation_paths.go b/pkg/dotc1z/engine/pebble/mutation_paths.go new file mode 100644 index 000000000..9b36f5211 --- /dev/null +++ b/pkg/dotc1z/engine/pebble/mutation_paths.go @@ -0,0 +1,195 @@ +package pebble + +// The mutation-path registry. +// +// THE LAW THIS FILE ENCODES: bug count scales with (paths × obligations) +// when cross-cutting write obligations are enforced by call-site +// convention, and with (paths + obligations) when they are derived at +// choke points. Three generations of the same bug family prove it here: +// the replay ingestion path missed the fresh path's side effects (the +// I1–I4 invariants exist because of it), the deletion paths missed +// index/manifest/dirty obligations (the dropBatcher and the +// invalidation-with-first-delete contract exist because of it), and each +// was found by review after shipping. This registry is the +// sideEffectAnnotationCoverage trick applied to code paths: every file +// in this package that touches a RAW Pebble write primitive must declare +// what it writes, how it satisfies the cross-cutting obligations, and +// (where they exist in-package) the tests that pin its contracts. +// mutation_paths_test.go enforces both directions mechanically — an +// undeclared raw-writing file and a stale declaration both fail — so a +// NEW mutation path cannot ship without having this conversation in the +// PR that adds it. +// +// THE OBLIGATIONS every declaration must account for (explicitly or as +// n/a-with-reason): +// +// - secondary indexes: by_principal / by_source_scope / +// needs_expansion / by_parent entries staged with the row mutation +// (or deliberately deferred to the seal build). +// - digest invalidation: post-seal grant mutations must stage +// per-entitlement digest invalidation (present-means-exact). +// - source-cache manifest: deleting rows out from under a scope's +// validator must invalidate the entry IN THE SAME BATCH as the +// scope's first delete (see stageSourceCacheScopeInvalidationLocked). +// - envelope save: mutations must survive Close — the wrapper dirty +// bit plus the Engine.Mutated latch (withWriteAllowSealed) backstop. +// - entitlement keyspace note: entitlement-keyspace mutations must +// invalidate the bare-id lookup map, keyed on commits that LANDED. +// - crash ordering: multi-batch operations must leave every WAL prefix +// safe (durable markers, idempotent re-runs, obligation-first +// batching). +// +// SYNCER-LEVEL MUTATION PATHS (they all funnel into the files below; +// listed here so the inventory has one home): +// +// 1. fresh response pages (Put*) — oracle: the replay- +// equivalence harness's cold legs + model-derived expected sets. +// 2. source-cache replay copy — oracle: warm ≡ cold +// (source_cache_equivalence_test.go) + halt-point sweep. +// 3. deletion family (tombstones, dangling drops, replay pre-clear) +// — oracle: the dirty +// scenario family + drop crash tests; all drops via dropBatcher. +// 4. raw-SST ingestion (ToPebble conversion, compactor merge) +// — oracle: byte-equivalence +// vs the canonical write path (TestToPebbleBulkImportEquivalence, +// pkg/dotc1z). +// 5. expansion/synth writes — deferred index + fused +// digest build at seal; durable pending markers. +// 6. external-match transform (delete carriers + write transformed) +// — composite path over +// grants.go primitives; deliberately does NOT invalidate manifests +// (transformed grants replace carriers under the SAME scope, so +// warm/cold converge — pinned by the harness's external-churn +// divergence trap). +// 7. I3 repair copy (CopyResourceRowFrom) — single-row ingestion +// from the previous artifact. +// +// PRODUCT NON-GOALS the registry records so they are not re-opened by +// accident: SQLite artifacts are never source-cache replay sources +// (SourceCacheStore is Pebble-only; SQLite rows carry no scope stamps), +// and compacted artifacts are never replay sources (keep-newer merges +// that no input's validators describe; triple-gated by the compacted +// flag, manifest clearing, and token provenance). + +// rawWriteDeclaration documents one file's raw write surface. +type rawWriteDeclaration struct { + // Writes summarizes what the file mutates and on which path(s). + Writes string + // Obligations states how the cross-cutting obligations are satisfied + // (or why they do not apply). + Obligations string + // PinnedBy lists IN-PACKAGE test functions pinning the file's write + // contracts; existence is enforced. Cross-package oracles are named + // in Obligations text instead. + PinnedBy []string +} + +// rawWriteRegistry: one entry per non-test file in this package that +// invokes a raw Pebble write primitive (NewBatch, db.Set, db.Delete, +// DeleteRange, db.Ingest, IngestAndExcise, db.Apply). Enforced by +// TestRawWriteRegistryCoversAllRawWritingFiles. +var rawWriteRegistry = map[string]rawWriteDeclaration{ + "assets.go": { + Writes: "asset rows (put/delete) on the fresh ingestion path", + Obligations: "no secondary indexes, digests, or scope stamps; envelope save via store wrappers + Engine.Mutated backstop", + }, + "bulk_import.go": { + Writes: "raw-SST bulk ingestion (ToPebble conversion from sorted SQLite sources)", + Obligations: "maintains grantIndexFamilies inline (incl. by_source_scope; SQLite sources carry no scope stamps — SQLite replay is a non-goal); " + + "digests deliberately absent (derived at seal); byte-equivalence vs the canonical write path pinned by TestToPebbleBulkImportEquivalence (pkg/dotc1z)", + }, + "deferred_index.go": { + Writes: "seal-time by_principal index rebuild + primary-grant keyspace rebuild via IngestAndExcise", + Obligations: "crash ordering via the durable deferred-index pending marker (interrupted build re-runs at resumed EndSync); " + + "stats stash invalidated by drops (invalidateDeferredGrantStats)", + }, + "digest.go": { + Writes: "grant digest nodes and invalidation ranges", + Obligations: "present-means-exact: mutations stage invalidation for touched entitlements; build/repair own reconstruction", + }, + "entitlements.go": { + Writes: "entitlement primary rows + by_source_scope index (fresh, replay overlay, tombstones)", + Obligations: "scope index staged with the row; bare-id lookup map invalidated via noteEntitlementKeyspaceWrite keyed on landed commits", + }, + "grant_digest.go": { + Writes: "digest keyspace drops (wholesale invalidation)", + Obligations: "drop-first recovery: every crash converts to digests-absent, which present-means-exact treats as safe", + }, + "grant_digest_build.go": { + Writes: "seal-time by_entitlement_principal_hash index + digest construction (fused with the deferred pass)", + Obligations: "crash ordering via the durable digest-build pending marker (crash mid-build drops digest state at next open/repair " + + "instead of trusting partial roots)", + }, + "grant_digest_repair.go": { + Writes: "digest repair for entitlements missing roots (post-seal mutation healing)", + Obligations: "runs under the write lock end-to-end so no grant write interleaves mid-partition; trusts only present roots", + }, + "grants.go": { + Writes: "grant primary rows + inline indexes (needs_expansion, by_source_scope), synth-layer SST ingestion, deferred-index marker arming", + Obligations: "by_principal deferred to seal (durable marker); post-seal mutations stage digest invalidation; " + + "scope index staged with the row (replay equivalence depends on it)", + }, + "id_index_format.go": { + Writes: "engine-meta markers for the id-index layout (deferred-index pending, digest-build pending)", + Obligations: "markers ARE the crash-ordering mechanism for the seal builds; durable (survive restart) by design", + }, + "id_index_migration.go": { + Writes: "on-open id-index layout migration via IngestAndExcise", + Obligations: "runs before the store serves reads; migrated-on-open forces an envelope save so the migration is never repaid", + }, + "index_migrations.go": { + Writes: "on-open secondary-index backfills", + Obligations: "same on-open contract as the id-index migration: complete before serving, dirty-on-migrate", + }, + "if_newer.go": { + Writes: "keep-newer merge primitive (compaction fold path)", + Obligations: "fold outputs are never replay sources (manifest cleared + compacted flag + token provenance — see the non-goals above)", + }, + "ingest_facts.go": { + Writes: "existence-bit fact markers (external-match grants seen, etc.)", + Obligations: "facts are monotone hints re-derived from the store at consuming seams (I2 repair); losing one is repaired, never silent", + }, + "ingest_repair.go": { + Writes: "dangling-reference drops (I7/I8/I9 arms) — ALL row deletion flows through dropBatcher", + Obligations: "the choke point: secondary indexes + digest invalidation derived from raw values; manifest invalidation rides the SAME batch " + + "as each scope's first delete; chunked commits with the crash hook between chunks; entitlement keyspace note keyed on landed commits", + PinnedBy: []string{ + "TestDanglingDropCrashBetweenChunksKeepsInvalidation", + "TestDanglingDropCleanCompletionInvalidates", + "TestManifestSnapshotInvalidationIsTyped", + }, + }, + "keyspace_version.go": { + Writes: "keyspace version stamp (engine-meta)", + Obligations: "written at store creation/migration boundaries only; no row obligations", + }, + "resource_types.go": { + Writes: "resource type rows (fresh ingestion)", + Obligations: "no secondary indexes or scope stamps; envelope save via wrappers + Mutated backstop", + }, + "resources.go": { + Writes: "resource primary rows + by_parent and by_source_scope indexes (fresh, replay, tombstones, I3 repair copy)", + Obligations: "indexes staged with the row; repair copy reuses the row's own index derivation", + }, + "session_store.go": { + Writes: "session key/value rows", + Obligations: "sync-scoped scratch state; no cross-cutting obligations beyond the envelope save", + }, + "source_cache.go": { + Writes: "source-cache manifest entries, replay copies (pre-clear + copy + recount), delta tombstones", + Obligations: "replay copy is the scope's first writer (resume pre-clear restores that precondition; ResumedRowsCleared marks the store dirty); " + + "post-copy recount fails partial copies before a manifest entry can seal them; tombstones delete rows + indexes together", + PinnedBy: []string{ + "TestSourceCacheReplayResumeAfterOverlayCommit", + "TestDeleteGrantsByPrincipalsInScope", + }, + }, + "sync_runs.go": { + Writes: "sync run records (start/end/compacted/token stamps)", + Obligations: "run records are the replay metadata gate (UsableAsReplaySource); compacted stamping is part of the fold's refusal contract", + }, + "sync_stats_sidecar.go": { + Writes: "stats sidecar records", + Obligations: "advisory (readers fall back to O(N) iteration); drop paths invalidate the deferred stash so counts never include dropped rows", + }, +} diff --git a/pkg/dotc1z/engine/pebble/mutation_paths_test.go b/pkg/dotc1z/engine/pebble/mutation_paths_test.go new file mode 100644 index 000000000..c88e0c96f --- /dev/null +++ b/pkg/dotc1z/engine/pebble/mutation_paths_test.go @@ -0,0 +1,111 @@ +package pebble + +// Enforcement for the mutation-path registry (mutation_paths.go): every +// non-test file in this package that touches a raw Pebble write +// primitive must carry a declaration, and every declaration must +// describe a file that still raw-writes. This is how a NEW mutation +// path is forced to state its obligations in the PR that adds it — the +// same mechanism sideEffectAnnotationCoverage uses for side-effecting +// annotations. + +import ( + "os" + "regexp" + "sort" + "strings" + "testing" + + "github.com/stretchr/testify/require" +) + +// rawWriteSignals are the textual markers of raw Pebble write usage. +// Deliberately simple substring checks over source text: false +// positives just require a (cheap, documentation-bearing) declaration, +// while a regex clever enough to dodge comments would be a second +// implementation to keep correct. +var rawWriteSignals = []string{ + ".NewBatch(", + ".db.Set(", + ".db.Delete(", + "DeleteRange(", + ".db.Ingest(", + "IngestAndExcise(", + ".db.Apply(", +} + +func TestRawWriteRegistryCoversAllRawWritingFiles(t *testing.T) { + entries, err := os.ReadDir(".") + require.NoError(t, err) + + detected := map[string]bool{} + for _, entry := range entries { + name := entry.Name() + if entry.IsDir() || !strings.HasSuffix(name, ".go") || strings.HasSuffix(name, "_test.go") { + continue + } + src, err := os.ReadFile(name) + require.NoError(t, err) + for _, signal := range rawWriteSignals { + if strings.Contains(string(src), signal) { + detected[name] = true + break + } + } + } + + for _, name := range sortedNames(detected) { + _, declared := rawWriteRegistry[name] + require.True(t, declared, + "%s uses raw Pebble write primitives but has NO mutation-path declaration.\n"+ + "Add an entry to rawWriteRegistry (mutation_paths.go) stating what it writes and how it satisfies\n"+ + "the cross-cutting obligations (secondary indexes, digest invalidation, source-cache manifest,\n"+ + "envelope save, entitlement keyspace note, crash ordering). This is deliberate friction:\n"+ + "every undeclared mutation path in this package's history has shipped the same bug family.", + name) + } + for name := range rawWriteRegistry { + require.True(t, detected[name], + "rawWriteRegistry declares %s but it no longer uses raw write primitives (or was renamed); remove or update the stale entry", name) + } + + // Declarations must actually declare. + for name, decl := range rawWriteRegistry { + require.NotEmpty(t, decl.Writes, "registry entry %s: Writes must describe the mutation path", name) + require.NotEmpty(t, decl.Obligations, "registry entry %s: Obligations must state how (or why not) the cross-cutting obligations apply", name) + } +} + +func TestRawWriteRegistryPinnedByTestsExist(t *testing.T) { + testFuncs := map[string]bool{} + entries, err := os.ReadDir(".") + require.NoError(t, err) + testFuncRe := regexp.MustCompile(`(?m)^func (Test\w+)\(`) + for _, entry := range entries { + name := entry.Name() + if entry.IsDir() || !strings.HasSuffix(name, "_test.go") { + continue + } + src, err := os.ReadFile(name) + require.NoError(t, err) + for _, m := range testFuncRe.FindAllStringSubmatch(string(src), -1) { + testFuncs[m[1]] = true + } + } + + for name, decl := range rawWriteRegistry { + for _, pinned := range decl.PinnedBy { + require.True(t, testFuncs[pinned], + "registry entry %s names pinning test %q, which does not exist in this package; fix the name or move the pin into Obligations text (cross-package oracles)", + name, pinned) + } + } +} + +func sortedNames(set map[string]bool) []string { + out := make([]string, 0, len(set)) + for k := range set { + out = append(out, k) + } + sort.Strings(out) + return out +} diff --git a/pkg/dotc1z/engine/pebble/source_cache.go b/pkg/dotc1z/engine/pebble/source_cache.go index 0d08d6099..5de9a0591 100644 --- a/pkg/dotc1z/engine/pebble/source_cache.go +++ b/pkg/dotc1z/engine/pebble/source_cache.go @@ -82,22 +82,38 @@ type SourceCacheReplayResult struct { ResumedRowsCleared int64 } +// SourceCacheManifestEntry is one manifest entry as reported by +// SourceCacheManifestSnapshot. Typed on purpose: the validator is an +// OPAQUE connector string, so invalidation must be a separate field — +// any in-band sentinel encoding could collide with a legitimate +// validator and misclassify it. +type SourceCacheManifestEntry struct { + // CacheValidator is the stored upstream validator (kept even on + // invalidated entries, for the audit surface). + CacheValidator string + // Invalidated reports the dangling-reference drops marked this scope + // (see stageSourceCacheScopeInvalidationLocked): lookups miss it and + // the scope re-fetches cold next sync. + Invalidated bool +} + // SourceCacheManifestSnapshot returns every manifest entry as -// "row_kind\x00scope_key" -> etag. Inspection surface: the -// replay-equivalence harness compares a warm sync's manifest against a -// cold sync's (they must be identical — same scopes, same validators), -// catching phantom or missing entries that are invisible through the v2 -// read API but poison FUTURE syncs' replays. -func (e *Engine) SourceCacheManifestSnapshot(ctx context.Context) (map[string]string, error) { +// "row_kind\x00scope_key" -> SourceCacheManifestEntry. Inspection +// surface: the replay-equivalence harness compares a warm sync's +// manifest against a cold sync's (they must be identical — same scopes, +// same validators, same invalidation verdicts), catching phantom or +// missing entries that are invisible through the v2 read API but poison +// FUTURE syncs' replays. +func (e *Engine) SourceCacheManifestSnapshot(ctx context.Context) (map[string]SourceCacheManifestEntry, error) { iter, err := e.db.NewIter(&pebble.IterOptions{ - LowerBound: encodeSourceCachePrefix(), - UpperBound: upperBoundOf(encodeSourceCachePrefix()), + LowerBound: encodeSourceCacheManifestPrefix(), + UpperBound: upperBoundOf(encodeSourceCacheManifestPrefix()), }) if err != nil { return nil, err } defer iter.Close() - out := map[string]string{} + out := map[string]SourceCacheManifestEntry{} for iter.First(); iter.Valid(); iter.Next() { if err := ctx.Err(); err != nil { return nil, err @@ -106,7 +122,10 @@ func (e *Engine) SourceCacheManifestSnapshot(ctx context.Context) (map[string]st if err := unmarshalRecord(iter.Value(), rec); err != nil { return nil, fmt.Errorf("SourceCacheManifestSnapshot: unmarshal: %w", err) } - out[rec.GetRowKind()+"\x00"+rec.GetScopeKey()] = rec.GetCacheValidator() + out[rec.GetRowKind()+"\x00"+rec.GetScopeKey()] = SourceCacheManifestEntry{ + CacheValidator: rec.GetCacheValidator(), + Invalidated: rec.GetInvalidated(), + } } return out, iter.Error() } @@ -269,14 +288,16 @@ func (e *Engine) SourceCacheOrphanScopes(ctx context.Context) (map[string][]stri return out, nil } -// ClearSourceCacheEntries removes every source-cache manifest entry. -// Compaction calls this on fold outputs: a keep-newer upsert merge is not -// a faithful snapshot of ANY input sync (base rows the newer sync deleted -// survive), so no input's validator may be allowed to 304-validate the -// merged row set. With the manifest empty, every lookup against the -// artifact misses and the next sync fetches fresh — cold-correct. The -// scope stamps and scope-index entries on rows are left in place: replay -// only reads them after a manifest hit, which can no longer happen. +// ClearSourceCacheEntries removes every source-cache manifest entry AND +// the replay-compatibility record (the range covers the whole +// typeSourceCache prefix). Compaction calls this on fold outputs: a +// keep-newer upsert merge is not a faithful snapshot of ANY input sync +// (base rows the newer sync deleted survive), so no input's validator +// may be allowed to 304-validate the merged row set. With the manifest +// empty, every lookup against the artifact misses and the next sync +// fetches fresh — cold-correct. The scope stamps and scope-index entries +// on rows are left in place: replay only reads them after a manifest +// hit, which can no longer happen. func (e *Engine) ClearSourceCacheEntries(ctx context.Context) error { return e.withWrite(func() error { if err := e.requireCurrentSync(); err != nil { @@ -311,6 +332,44 @@ func (e *Engine) PutSourceCacheEntry(ctx context.Context, rowKind, scopeKey, cac }) } +// PutSourceCacheCompat writes the sync's replay-compatibility key record +// (singleton; see SourceCacheCompatRecord). Written when the source +// cache's write side enables — BEFORE any manifest entry — so an +// artifact can never hold validators without the key that scopes their +// validity. Idempotent overwrite on resume. +func (e *Engine) PutSourceCacheCompat(ctx context.Context, rec *v3.SourceCacheCompatRecord) error { + return e.withWrite(func() error { + if err := e.requireCurrentSync(); err != nil { + return err + } + rec.SetId("compat") + val, err := marshalRecord(rec) + if err != nil { + return err + } + opts := writeOpts(e.opts.durability) + if e.IsFreshSync() { + opts = pebble.NoSync + } + return e.db.Set(encodeSourceCacheCompatKey(), val, opts) + }) +} + +// GetSourceCacheCompat returns the sync's replay-compatibility key +// record, or pebble.ErrNotFound (pre-key artifacts, folds). +func (e *Engine) GetSourceCacheCompat(ctx context.Context) (*v3.SourceCacheCompatRecord, error) { + val, closer, err := e.db.Get(encodeSourceCacheCompatKey()) + if err != nil { + return nil, err + } + defer closer.Close() + rec := &v3.SourceCacheCompatRecord{} + if err := unmarshalRecord(val, rec); err != nil { + return nil, fmt.Errorf("GetSourceCacheCompat: unmarshal: %w", err) + } + return rec, nil +} + // GetSourceCacheEntry returns the manifest entry for (rowKind, scopeKey), // or pebble.ErrNotFound. func (e *Engine) GetSourceCacheEntry(ctx context.Context, rowKind, scopeKey string) (*v3.SourceCacheEntryRecord, error) { diff --git a/pkg/dotc1z/engine/pebble/sync_stats_sidecar.go b/pkg/dotc1z/engine/pebble/sync_stats_sidecar.go index 040cfedda..41d2ab6f8 100644 --- a/pkg/dotc1z/engine/pebble/sync_stats_sidecar.go +++ b/pkg/dotc1z/engine/pebble/sync_stats_sidecar.go @@ -240,6 +240,16 @@ func (e *Engine) stashDeferredGrantStats(ds *deferredGrantStats) { e.deferredGrantStatsMu.Unlock() } +// invalidateDeferredGrantStats drops any stashed grant counts. Called by +// the dangling-reference grant drops (ingest_repair.go): rows deleted +// AFTER the deferred build ran would make the stash overcount, so the +// stats sidecar must fall back to its own post-drop scan. +func (e *Engine) invalidateDeferredGrantStats() { + e.deferredGrantStatsMu.Lock() + e.deferredGrantStats = nil + e.deferredGrantStatsMu.Unlock() +} + // takeDeferredGrantStats pops the stashed grant stats if they belong to // syncID, or returns nil. Consume-once: a later RecalculateStats falls back // to the full scan. diff --git a/pkg/dotc1z/pebble_store.go b/pkg/dotc1z/pebble_store.go index 9714d4281..5d3afbc34 100644 --- a/pkg/dotc1z/pebble_store.go +++ b/pkg/dotc1z/pebble_store.go @@ -320,7 +320,7 @@ func (s *pebbleStore) CloseEngineOnly() error { s.closeMu.Unlock() return nil } - if !s.readOnly && s.dirty { + if !s.readOnly && (s.dirty || s.engine.Mutated()) { s.closeMu.Unlock() return errors.New("pebble CloseEngineOnly: refusing to discard dirty writable store") } @@ -675,7 +675,11 @@ func (s *pebbleStore) Close(ctx context.Context) (retErr error) { return nil } - if !s.readOnly && s.dirty { + // The engine's mutated latch is the backstop for the per-method + // wrapper dirty bit: any engine write path that bypassed a wrapper + // still forces the envelope save. Conservative in the safe + // direction — see Engine.Mutated. + if !s.readOnly && (s.dirty || s.engine.Mutated()) { if err := s.save(ctx); err != nil { // Tear NOTHING down: the unpacked DB under tmpDir is the only // copy of the synced data, and save failures are frequently diff --git a/pkg/dotc1z/pebble_store_mutated_test.go b/pkg/dotc1z/pebble_store_mutated_test.go new file mode 100644 index 000000000..f16049f2a --- /dev/null +++ b/pkg/dotc1z/pebble_store_mutated_test.go @@ -0,0 +1,64 @@ +package dotc1z + +// Engine-derived dirty backstop: the pebbleStore marks its envelope-save +// dirty bit per wrapper method BY CONVENTION, and every engine mutation +// path that misses a wrapper silently skips the save on Close — the +// clear-only replay pre-clear was one such path, and the class was only +// fixed instance-by-instance. Engine.Mutated latches when any write +// closure runs, and Close saves on EITHER signal, so the class is closed +// structurally: bypassing a wrapper can cost a spurious save, never a +// lost one. + +import ( + "context" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + + "github.com/conductorone/baton-sdk/pkg/connectorstore" + "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" + pebbleengine "github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble" + "github.com/conductorone/baton-sdk/pkg/sourcecache" +) + +func TestPebbleCloseSavesOnEngineMutationBypassingWrappers(t *testing.T) { + ctx := context.Background() + tmpDir := t.TempDir() + path := filepath.Join(tmpDir, "mutated.c1z") + + // A finished sync, saved and closed: the reopened handle starts with + // the wrapper dirty bit clear. + store, err := NewStore(ctx, path, WithEngine(c1zstore.EnginePebble), WithTmpDir(tmpDir)) + require.NoError(t, err) + syncID, err := store.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + require.NoError(t, store.EndSync(ctx)) + require.NoError(t, store.Close(ctx)) + + // Reopen writable and mutate through the ENGINE directly — the + // wrapper-bypass shape (any future engine method without a + // dirty-marking wrapper takes exactly this path). + store, err = NewStore(ctx, path, WithEngine(c1zstore.EnginePebble), WithTmpDir(tmpDir)) + require.NoError(t, err) + require.NoError(t, store.SetCurrentSync(ctx, syncID)) + holder, ok := any(store).(interface{ PebbleEngine() *pebbleengine.Engine }) + require.True(t, ok) + require.NoError(t, holder.PebbleEngine().PutSourceCacheEntry(ctx, "grants", "scope-x", "etag-x")) + require.NoError(t, store.Close(ctx)) + + // The mutation must have survived Close: without the engine-derived + // backstop the wrapper dirty bit was never set, the envelope save was + // skipped, and the write existed only in the discarded temp dir. + reopened, err := NewStore(ctx, path, WithEngine(c1zstore.EnginePebble), WithReadOnly(true), WithTmpDir(tmpDir)) + require.NoError(t, err) + defer func() { _ = reopened.Close(ctx) }() + require.NoError(t, reopened.SetCurrentSync(ctx, syncID)) + sc, ok := reopened.(SourceCacheStore) + require.True(t, ok) + entry, found, err := sc.LookupSourceCacheEntry(ctx, sourcecache.RowKindGrants, "scope-x") + require.NoError(t, err) + require.True(t, found, + "an engine mutation that bypassed the store wrappers must still force the envelope save on Close") + require.Equal(t, "etag-x", entry.CacheValidator) +} diff --git a/pkg/dotc1z/source_cache.go b/pkg/dotc1z/source_cache.go index 6134397e1..042b4a5cd 100644 --- a/pkg/dotc1z/source_cache.go +++ b/pkg/dotc1z/source_cache.go @@ -7,6 +7,7 @@ import ( cdbpebble "github.com/cockroachdb/pebble/v2" + v3 "github.com/conductorone/baton-sdk/pb/c1/storage/v3" "github.com/conductorone/baton-sdk/pkg/bid" "github.com/conductorone/baton-sdk/pkg/connectorstore" "github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble" @@ -62,6 +63,14 @@ type SourceCacheStore interface { // invariant I6): the orphaned stamps would poison a future sync's // replay. SourceCacheOrphanScopes(ctx context.Context) (map[string][]string, error) + + // PutSourceCacheCompat records the current sync's replay- + // compatibility key (written when the write side enables, BEFORE any + // manifest entry). GetSourceCacheCompat returns the recorded key; + // found is false on pre-key artifacts and compaction folds — the + // syncer treats absent or unreadable keys as a mismatch (cold). + PutSourceCacheCompat(ctx context.Context, key sourcecache.CompatKey) error + GetSourceCacheCompat(ctx context.Context) (sourcecache.CompatKey, bool, error) } var _ SourceCacheStore = (*pebbleStore)(nil) @@ -72,8 +81,10 @@ var _ SourceCacheStore = (*pebbleStore)(nil) // reaching for the raw engine. type SourceCacheInspector interface { // SourceCacheManifestSnapshot returns every manifest entry as - // "row_kind\x00scope_key" -> cache validator. - SourceCacheManifestSnapshot(ctx context.Context) (map[string]string, error) + // "row_kind\x00scope_key" -> typed entry (validator + invalidation + // verdict as separate fields — validators are opaque connector + // strings, so invalidation must never be encoded in-band). + SourceCacheManifestSnapshot(ctx context.Context) (map[string]pebble.SourceCacheManifestEntry, error) // SourceScopeIndexSnapshot returns, per row kind, scope_key -> the // number of by_source_scope index entries. SourceScopeIndexSnapshot(ctx context.Context) (map[string]map[string]int, error) @@ -85,7 +96,7 @@ type SourceCacheInspector interface { var _ SourceCacheInspector = (*pebbleStore)(nil) -func (s *pebbleStore) SourceCacheManifestSnapshot(ctx context.Context) (map[string]string, error) { +func (s *pebbleStore) SourceCacheManifestSnapshot(ctx context.Context) (map[string]pebble.SourceCacheManifestEntry, error) { return s.engine.SourceCacheManifestSnapshot(ctx) } @@ -271,6 +282,13 @@ func (s *pebbleStore) LookupSourceCacheEntry(ctx context.Context, kind sourcecac } return sourcecache.Entry{}, false, err } + if rec.GetInvalidated() { + // The dangling-reference drops removed rows from this scope after + // its manifest entry was written: the validator no longer + // describes the artifact's contents, so the scope must miss and + // re-fetch cold rather than 304-replay the sanitized subset. + return sourcecache.Entry{}, false, nil + } return sourcecache.Entry{ CacheValidator: rec.GetCacheValidator(), DiscoveredAt: rec.GetDiscoveredAt().AsTime(), @@ -284,6 +302,31 @@ func (s *pebbleStore) PutSourceCacheEntry(ctx context.Context, kind sourcecache. return s.markDirty(s.engine.PutSourceCacheEntry(ctx, string(kind), scopeKey, cacheValidator)) } +func (s *pebbleStore) PutSourceCacheCompat(ctx context.Context, key sourcecache.CompatKey) error { + rec := &v3.SourceCacheCompatRecord{} + rec.SetConnectorCacheGeneration(key.ConnectorCacheGeneration) + rec.SetConnectorConfigFingerprint(key.ConnectorConfigFingerprint) + rec.SetSdkMaterializationGeneration(key.SDKMaterializationGeneration) + rec.SetSyncSelectionFingerprint(key.SyncSelectionFingerprint) + return s.markDirty(s.engine.PutSourceCacheCompat(ctx, rec)) +} + +func (s *pebbleStore) GetSourceCacheCompat(ctx context.Context) (sourcecache.CompatKey, bool, error) { + rec, err := s.engine.GetSourceCacheCompat(ctx) + if err != nil { + if errors.Is(err, cdbpebble.ErrNotFound) { + return sourcecache.CompatKey{}, false, nil + } + return sourcecache.CompatKey{}, false, err + } + return sourcecache.CompatKey{ + ConnectorCacheGeneration: rec.GetConnectorCacheGeneration(), + ConnectorConfigFingerprint: rec.GetConnectorConfigFingerprint(), + SDKMaterializationGeneration: rec.GetSdkMaterializationGeneration(), + SyncSelectionFingerprint: rec.GetSyncSelectionFingerprint(), + }, true, nil +} + func (s *pebbleStore) ReplaySourceCache(ctx context.Context, prev connectorstore.Reader, kind sourcecache.RowKind, scopeKey string) (SourceCacheReplayResult, error) { prevEngine, ok := sourceCacheEngine(prev) if !ok { @@ -304,7 +347,10 @@ func (s *pebbleStore) ReplaySourceCache(ctx context.Context, prev connectorstore if err != nil { return SourceCacheReplayResult{}, err } - if res.Rows > 0 { + if res.Rows > 0 || res.ResumedRowsCleared > 0 { + // ResumedRowsCleared counts destination rows the resume pre-clear + // deleted before an (possibly empty) copy — a mutation that must + // survive Close even when nothing was copied back. s.MarkDirty() } return res, nil diff --git a/pkg/sourcecache/compat.go b/pkg/sourcecache/compat.go new file mode 100644 index 000000000..0619381cb --- /dev/null +++ b/pkg/sourcecache/compat.go @@ -0,0 +1,92 @@ +package sourcecache + +import ( + "crypto/sha256" + "encoding/hex" + "sort" + "strconv" + "strings" +) + +// Replay-compatibility key. +// +// Replay is OPTIONAL — a pure optimization — and is permitted only when +// the prior artifact and the current run match BYTE-EXACTLY on every +// component below. Any absent, unreadable, or mismatched component +// degrades the sync to cold (no-op lookup): a wrong cold sync does not +// exist, a wrong warm sync is silent bad data. Compatibility is never +// inferred from versions (semver says nothing about replay semantics); +// each component is an explicit declaration by the party that owns it. + +// MaterializationPolicyGeneration is the SDK's replay-compatibility +// component: bump it when the SDK changes how response rows or their +// side effects MATERIALIZE into the store in a replay-visible way — row +// translation, scope stamping, index derivation semantics, replay-carried +// side effects (child scheduling), or the post-collection invariant +// policy. Bumping it costs every deployment exactly one cold sync after +// upgrade; NOT bumping it when materialization changed replays rows the +// new code would have written differently. +const MaterializationPolicyGeneration = "1" + +// CompatKey is the replay-compatibility key recorded into every +// source-cache-enabled artifact and required to match exactly before +// that artifact may serve replay. +type CompatKey struct { + // ConnectorCacheGeneration is the connector's declared cache + // generation (SourceCacheCapability.cache_generation): scope + // computation, validator semantics, row/id construction. + ConnectorCacheGeneration string + // ConnectorConfigFingerprint is the connector's declared digest of + // config/permission inputs that change what upstream data it can see + // (SourceCacheCapability.config_fingerprint). + ConnectorConfigFingerprint string + // SDKMaterializationGeneration is MaterializationPolicyGeneration at + // record time. + SDKMaterializationGeneration string + // SyncSelectionFingerprint digests the SDK-side sync-shaping inputs + // (enabled resource types, skip flags): a selection change means the + // prior artifact's scopes cover a different row universe. + SyncSelectionFingerprint string +} + +// MismatchReason compares the current run's key against the previous +// artifact's recorded key and returns "" on an exact match, or a +// human-readable reason naming the FIRST mismatched component. Values +// are deliberately not echoed (fingerprints may derive from secrets); +// the component name is enough to act on. +func (k CompatKey) MismatchReason(prev CompatKey) string { + switch { + case k.ConnectorCacheGeneration != prev.ConnectorCacheGeneration: + return "connector cache-generation changed since the previous artifact was recorded" + case k.ConnectorConfigFingerprint != prev.ConnectorConfigFingerprint: + return "connector configuration/permission fingerprint changed since the previous artifact was recorded" + case k.SDKMaterializationGeneration != prev.SDKMaterializationGeneration: + return "SDK materialization-policy generation changed since the previous artifact was recorded" + case k.SyncSelectionFingerprint != prev.SyncSelectionFingerprint: + return "sync-selection fingerprint (enabled types, filters) changed since the previous artifact was recorded" + default: + return "" + } +} + +// SelectionFingerprint digests the sync-shaping inputs into the +// SyncSelectionFingerprint component. Canonical: resource type ids are +// sorted and length-prefixed (no delimiter injection), flags appended +// explicitly. An empty selection (all types, nothing skipped) has a +// well-defined fingerprint too — "no selection" must still match only +// "no selection". +func SelectionFingerprint(resourceTypeIDs []string, skipEntitlementsAndGrants, skipGrants bool) string { + sorted := append([]string{}, resourceTypeIDs...) + sort.Strings(sorted) + parts := make([]string, 0, len(sorted)+3) + parts = append(parts, "v1\x00types:") + for _, id := range sorted { + parts = append(parts, strconv.Itoa(len(id))+":"+id+"\x00") + } + parts = append(parts, + "skipEG:"+strconv.FormatBool(skipEntitlementsAndGrants), + "\x00skipG:"+strconv.FormatBool(skipGrants), + ) + sum := sha256.Sum256([]byte(strings.Join(parts, ""))) + return hex.EncodeToString(sum[:]) +} diff --git a/pkg/sync/expand/drop_stats.go b/pkg/sync/expand/drop_stats.go new file mode 100644 index 000000000..6261ebdca --- /dev/null +++ b/pkg/sync/expand/drop_stats.go @@ -0,0 +1,102 @@ +package expand + +import ( + "context" + "sync" + + "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" + "go.uber.org/zap" +) + +// DroppedEdgeStats aggregates expansion edges dropped because a +// referenced entitlement has no row — connector magic-id bugs and +// disabled-by-default resource types. Production measured ~2.9M +// per-edge warnings a week from these sites, dominated by a few +// thousand distinct ids, so per-edge logging is demoted to Debug and +// ONE aggregated warning per sync (emitted by the syncer when +// expansion completes) carries the totals and a capped sample of +// distinct ids — the same reporting shape as the ingestion invariants' +// dangling-reference warnings. +// +// One instance lives on the syncer for the whole sync (the Expander is +// reconstructed every step); guarded for parallel use. In-memory only: +// a resume after a crash undercounts, which is acceptable for +// observability. +type DroppedEdgeStats struct { + mu sync.Mutex + sourceMissing int64 + destinationMissing int64 + seen map[string]struct{} + examples []string +} + +const maxDroppedEdgeExamples = 25 + +func (s *DroppedEdgeStats) record(entitlementID string, dest bool, edges int64) { + if s == nil || edges <= 0 { + return + } + s.mu.Lock() + defer s.mu.Unlock() + if dest { + s.destinationMissing += edges + } else { + s.sourceMissing += edges + } + if s.seen == nil { + s.seen = map[string]struct{}{} + } + if _, ok := s.seen[entitlementID]; ok { + return + } + s.seen[entitlementID] = struct{}{} + if len(s.examples) < maxDroppedEdgeExamples { + s.examples = append(s.examples, entitlementID) + } +} + +// RecordSourceMissing counts ONE edge dropped because its SOURCE +// entitlement has no row. +func (s *DroppedEdgeStats) RecordSourceMissing(entitlementID string) { + s.record(entitlementID, false, 1) +} + +// RecordSourceMissingEdges counts a BATCH of edges sharing one missing +// source entitlement (the legacy expander drops a whole action's edges +// when the shared source is gone — the total must count every edge, not +// the batch). +func (s *DroppedEdgeStats) RecordSourceMissingEdges(entitlementID string, edges int) { + s.record(entitlementID, false, int64(edges)) +} + +// RecordDestinationMissing counts ONE edge dropped because its +// DESTINATION entitlement has no row. +func (s *DroppedEdgeStats) RecordDestinationMissing(entitlementID string) { + s.record(entitlementID, true, 1) +} + +// RecordDestinationMissingEdges counts every incoming edge skipped when +// one destination entitlement has no row (the topological path skips the +// destination's whole reduction — all of its incoming edges — at once). +func (s *DroppedEdgeStats) RecordDestinationMissingEdges(entitlementID string, edges int) { + s.record(entitlementID, true, int64(edges)) +} + +// LogSummary emits the one aggregated warning for the sync, if any +// edges were dropped. Safe on nil. +func (s *DroppedEdgeStats) LogSummary(ctx context.Context) { + if s == nil { + return + } + s.mu.Lock() + defer s.mu.Unlock() + if s.sourceMissing == 0 && s.destinationMissing == 0 { + return + } + ctxzap.Extract(ctx).Warn("grant expansion: DROPPED edges referencing entitlements with no entitlement row (connector bug or resource type not enabled — check magic-id construction)", + zap.Int64("edges_missing_source_entitlement", s.sourceMissing), + zap.Int64("edges_missing_destination_entitlement", s.destinationMissing), + zap.Int("distinct_missing_entitlements", len(s.seen)), + zap.Strings("entitlement_id_examples", s.examples), + ) +} diff --git a/pkg/sync/expand/drop_stats_test.go b/pkg/sync/expand/drop_stats_test.go new file mode 100644 index 000000000..e7cde6252 --- /dev/null +++ b/pkg/sync/expand/drop_stats_test.go @@ -0,0 +1,48 @@ +package expand + +// DroppedEdgeStats counting semantics: totals count EDGES, not drop +// events. The legacy expander drops a whole action's edge batch when the +// shared SOURCE entitlement is missing, and the topological path skips a +// destination's entire reduction (every incoming edge) when the +// DESTINATION is missing — both previously incremented the aggregate +// once per event, undercounting the very numbers the per-sync warning +// exists to report. + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestDroppedEdgeStatsCountsEdgesNotEvents(t *testing.T) { + s := &DroppedEdgeStats{} + + // A missing source shared by a 5-destination batch drops 5 edges. + s.RecordSourceMissingEdges("ent-src", 5) + // A missing destination with 3 incoming edges skips 3 edges. + s.RecordDestinationMissingEdges("ent-dst", 3) + // Single-edge sites still count one. + s.RecordSourceMissing("ent-src-2") + s.RecordDestinationMissing("ent-dst-2") + // Zero/negative counts are no-ops (defensive: empty batches). + s.RecordSourceMissingEdges("ent-none", 0) + s.RecordDestinationMissingEdges("ent-none", -1) + + s.mu.Lock() + defer s.mu.Unlock() + require.Equal(t, int64(6), s.sourceMissing, "source total must count every edge in a dropped batch") + require.Equal(t, int64(4), s.destinationMissing, "destination total must count every skipped incoming edge") + require.Len(t, s.seen, 4, "distinct-id set counts entitlements, not edges; no-op records add nothing") + require.NotContains(t, s.seen, "ent-none") +} + +func TestDroppedEdgeStatsExamplesDedupeAcrossRepeats(t *testing.T) { + s := &DroppedEdgeStats{} + for i := 0; i < 3; i++ { + s.RecordSourceMissingEdges("ent-repeat", 2) + } + s.mu.Lock() + defer s.mu.Unlock() + require.Equal(t, int64(6), s.sourceMissing, "repeat drops of one id still count their edges") + require.Equal(t, []string{"ent-repeat"}, s.examples, "examples list one entry per distinct id") +} diff --git a/pkg/sync/expand/expander.go b/pkg/sync/expand/expander.go index c54e25b41..034a1d142 100644 --- a/pkg/sync/expand/expander.go +++ b/pkg/sync/expand/expander.go @@ -137,6 +137,15 @@ type entitlementGrantPrincipalKeyLister interface { type Expander struct { store ExpanderStore graph *EntitlementGraph + // dropStats, when set, aggregates dropped-edge reporting for the + // whole sync (the syncer owns it across step-wise Expander + // reconstruction). Nil is safe: recording no-ops. + dropStats *DroppedEdgeStats +} + +// SetDropStats installs the sync-scoped dropped-edge aggregator. +func (e *Expander) SetDropStats(s *DroppedEdgeStats) { + e.dropStats = s } // NewExpander creates a new Expander with the given store and graph. @@ -419,7 +428,12 @@ func (e *Expander) runAction(ctx context.Context, action *EntitlementGraphAction // drop them all and complete the action. Mirrors the inline handling // of missing destinations below. (MarkEdgeExpanded no-ops on deleted // edges, so returning "" here completes the action cleanly.) - l.Warn("runAction: source entitlement not found, dropping batch edges", + // Per-edge logging is Debug: production measured millions of + // these a week; the aggregate warning (DroppedEdgeStats) is + // the per-sync report. The batch shares one source, but every + // destination edge drops — count them all. + e.dropStats.RecordSourceMissingEdges(action.SourceEntitlementID, len(dests)) + l.Debug("runAction: source entitlement not found, dropping batch edges", zap.String("source_entitlement_id", action.SourceEntitlementID)) for _, d := range dests { _ = e.graph.DeleteEdge(ctx, action.SourceEntitlementID, d.EntitlementID) @@ -453,8 +467,9 @@ func (e *Expander) runAction(ctx context.Context, action *EntitlementGraphAction if err != nil { if status.Code(err) == codes.NotFound { // A single missing descendant drops only its own edge; the rest - // of the batch still expands. - l.Warn("runAction: descendant entitlement not found, dropping edge", + // of the batch still expands. Debug + aggregate, as above. + e.dropStats.RecordDestinationMissing(d.EntitlementID) + l.Debug("runAction: descendant entitlement not found, dropping edge", zap.String("descendant_entitlement_id", d.EntitlementID)) _ = e.graph.DeleteEdge(ctx, action.SourceEntitlementID, d.EntitlementID) continue diff --git a/pkg/sync/expand/topological_merge.go b/pkg/sync/expand/topological_merge.go index 594df7c2f..c3b358078 100644 --- a/pkg/sync/expand/topological_merge.go +++ b/pkg/sync/expand/topological_merge.go @@ -399,9 +399,14 @@ func (e *Expander) driveTopologicalLayer( destEntitlement := entitlements[destID] if destEntitlement == nil { // Same drop-don't-fail policy as the legacy expander's - // missing-descendant handling, but say so: silently zeroing - // a subtree of expansion is undiagnosable in production. - ctxzap.Extract(ctx).Warn("topological expansion: destination entitlement not in store; skipping its reduction", + // missing-descendant handling. Recorded on the sync-wide + // aggregate (one warning per sync with distinct-id + // examples); per-edge logging stays at Debug so a large + // dangling family can't flood the logs. Skipping this + // destination skips its ENTIRE reduction — every incoming + // edge — so the total counts each one. + e.dropStats.RecordDestinationMissingEdges(destID, len(incoming)) + ctxzap.Extract(ctx).Debug("topological expansion: destination entitlement not in store; skipping its reduction", zap.String("entitlement_id", destID)) continue } diff --git a/pkg/sync/gen_skew_inputs_test.go b/pkg/sync/gen_skew_inputs_test.go new file mode 100644 index 000000000..85fb0f52f --- /dev/null +++ b/pkg/sync/gen_skew_inputs_test.go @@ -0,0 +1,53 @@ +package sync //nolint:revive,nolintlint // we can't change the package name for backwards compatibility + +// Generator for the version-skew fixture's INPUTS (see +// testdata/README-old-fold.md): two new-format, source-cache-stamped +// sync files that an old-binary compactor is then run over. Env-gated +// and skipped in normal runs; retained so the fixture is regenerable: +// +// BATON_GEN_SKEW_INPUTS=/tmp/skew-gen go test -run TestGenerateSkewInputs ./pkg/sync/ + +import ( + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + "github.com/conductorone/baton-sdk/pkg/logging" + gt "github.com/conductorone/baton-sdk/pkg/types/grant" +) + +func TestGenerateSkewInputs(t *testing.T) { + outDir := os.Getenv("BATON_GEN_SKEW_INPUTS") + if outDir == "" { + t.Skip("set BATON_GEN_SKEW_INPUTS to an output dir") + } + ctx, err := logging.Init(t.Context()) + require.NoError(t, err) + require.NoError(t, os.MkdirAll(outDir, 0o755)) //nolint:gosec // operator-supplied output dir in an env-gated generator + tmpDir := t.TempDir() + + group, ent, alice, bob := sourceCacheTestFixtures(t) + mc := newSourceCacheMockConnector() + mc.AddResource(ctx, group) + mc.AddResource(ctx, alice) + mc.AddResource(ctx, bob) + mc.entDB[group.GetId().GetResource()] = []*v2.Entitlement{ent} + mc.grantDB[group.GetId().GetResource()] = []*v2.Grant{gt.NewGrant(group, "member", alice)} + mc.etagByResource[group.GetId().GetResource()] = `W/"skew-v1"` + + base := filepath.Join(outDir, "base.c1z") + runSourceCacheSync(ctx, t, mc, base, "", tmpDir) + + // Second sync with a changed grant set (new-format, manifest-stamped). + mc.grantDB[group.GetId().GetResource()] = []*v2.Grant{ + gt.NewGrant(group, "member", alice), + gt.NewGrant(group, "member", bob), + } + mc.etagByResource[group.GetId().GetResource()] = `W/"skew-v2"` + newer := filepath.Join(outDir, "newer.c1z") + runSourceCacheSync(ctx, t, mc, newer, "", tmpDir) + t.Logf("wrote %s and %s", base, newer) +} diff --git a/pkg/sync/ingest_invariants.go b/pkg/sync/ingest_invariants.go index b308b3d0b..f66715c3e 100644 --- a/pkg/sync/ingest_invariants.go +++ b/pkg/sync/ingest_invariants.go @@ -519,6 +519,29 @@ func (p *danglingTypeProbe) typeExists(ctx context.Context, resourceTypeID strin return exists } +// unexpectedDanglingError renders the ENABLED-TYPE dangling verdict of +// the replay policy: a missing referent whose resource type IS synced is +// never an expected configuration gap — it is connector data/behavior +// (magic-id construction, deleted-referent emission) or replay-carried +// staleness, and the artifact must not seal with the rows silently +// dropped. A WARM sync is replay-implicated: the error carries +// ErrReplayIntegrity so the runners discard the output and retry cold. +// A COLD sync with the same evidence is terminal — the cold retry +// already happened (or could never have helped), so the failure is +// attributed plainly to the connector. +// +// Disabled-type gaps (referent's type never synced) keep the drop arm: +// they are expected configuration shapes, sealed referentially +// consistent with one aggregated warning, and their scopes' manifest +// entries are invalidated so a later type-enable re-fetches them cold. +func (s *syncer) unexpectedDanglingError(format string, args ...any) error { + err := fmt.Errorf(format, args...) + if s.sourceCache.prev != nil { + return fmt.Errorf("%w (warm sync: discard the output and retry cold): %w", err, ErrReplayIntegrity) + } + return err +} + // danglingAttribution renders the config-gap vs bug split for the // aggregated warnings. func danglingAttribution(unsyncedTypeCount, missingRowCount int) string { @@ -564,6 +587,17 @@ func (s *syncer) checkEntitlementResourceReferences(ctx context.Context) error { "(resource type synced: %t — false means a config gap, true a magic-id construction bug)", rt, rid, probe.typeExists(ctx, rt)) } + if !s.onlyExpandGrants && probe.typeExists(ctx, rt) { + // Enabled-type dangling: the referent's type IS synced, so the + // missing row is unexpected — never sealed away (replay + // policy; see unexpectedDanglingError). The compaction + // expand pass is exempt: keep-newer merges legitimately + // manufacture dangling refs no input contained, and the drop + // pass is what converges the artifact. + return s.unexpectedDanglingError( + "ingest invariant I7 violated: entitlements reference resource %s/%s but no resource row exists under a SYNCED type", + rt, rid) + } n, ids, err := facts.DeleteEntitlementsForResource(ctx, rt, rid, maxDanglingIDExamples-len(entIDExamples)) if err != nil { return fmt.Errorf("ingest invariant I7: dropping entitlements for %s/%s: %w", rt, rid, err) @@ -645,6 +679,22 @@ func (s *syncer) checkGrantEntitlementReferences(ctx context.Context) error { "or a grants scope replayed against a refreshed entitlement set)", entitlementID, rt, rid, probe.typeExists(ctx, rt)) } + if !s.onlyExpandGrants && probe.typeExists(ctx, rt) { + // Enabled-type dangling (replay policy; see I7's twin arm). + // The per-grant IRG exemption still applies: an entitlement + // whose grants ALL carry InsertResourceGrants is the + // machinery-owned shape in every mode. + allCarry, err := facts.GrantsForEntitlementAllCarryInsertFact(ctx, entitlementID, rt, rid) + if err != nil { + return fmt.Errorf("ingest invariant I8: probing grant annotations for entitlement %q: %w", entitlementID, err) + } + if allCarry { + return nil + } + return s.unexpectedDanglingError( + "ingest invariant I8 violated: grants without InsertResourceGrants reference entitlement %q (resource %s/%s) but no entitlement row exists under a SYNCED type", + entitlementID, rt, rid) + } n, skipped, err := facts.DeleteGrantsForEntitlement(ctx, entitlementID, rt, rid) if err != nil { return fmt.Errorf("ingest invariant I8: dropping grants for entitlement %q: %w", entitlementID, err) @@ -729,6 +779,16 @@ func (s *syncer) checkGrantPrincipalReferences(ctx context.Context) error { "emitted grants for a principal it never synced)", rt, rid, probe.typeExists(ctx, rt)) } + if !s.onlyExpandGrants && probe.typeExists(ctx, rt) { + // Enabled-type dangling (replay policy; see I7's twin arm). + // Mixed populations land here too: exempt carriers were + // already excluded above (matchAnnotatedOnly), so a plain + // grant on a synced-type principal with no row is never + // silently dropped. + return s.unexpectedDanglingError( + "ingest invariant I9 violated: grants reference principal %s/%s but no resource row exists under a SYNCED type", + rt, rid) + } deleted, skipped, err := facts.DeleteGrantsForPrincipal(ctx, rt, rid) if err != nil { return fmt.Errorf("ingest invariant I9: dropping grants for principal %s/%s: %w", rt, rid, err) @@ -797,19 +857,21 @@ func (s *syncer) checkSourceCacheScopeConsistency(ctx context.Context) error { if len(orphans) == 0 { return nil } - // Classified as replay-integrity: orphaned scope state in the sealed - // output is exactly the inconsistency that poisons a future replay, - // and a cold re-run either produces a clean artifact (replay-path - // fault, self-healed) or reproduces the orphan and fails terminally - // (a put-path bug, correctly fatal). + // A stale replay index in the sealed output ALWAYS fails the sync + // (replay policy): I6 has no repair arm, and warn-and-seal would + // publish an artifact whose next replay silently drops or resurrects + // rows. On a WARM sync the error carries ErrReplayIntegrity — the + // runners discard the output and re-run cold (replay-path fault, + // self-healed). A cold re-run that reproduces the orphan fails + // terminally here WITHOUT the sentinel (a put-path bug, correctly + // fatal and correctly attributed). err = fmt.Errorf( - "ingest invariant I6 violated: scope index entries exist with no manifest entry (lost manifest write or stamp leak): %v: %w", - orphans, ErrReplayIntegrity) - if s.failFastInvariants { - return err + "ingest invariant I6 violated: scope index entries exist with no manifest entry (lost manifest write or stamp leak): %v", + orphans) + if s.sourceCache.prev != nil { + return fmt.Errorf("%w: %w", err, ErrReplayIntegrity) } - ctxzap.Extract(ctx).Warn("ingest invariant I6 violated (default mode)", zap.Error(err)) - return nil + return err } // runIngestionInvariants evaluates the post-collection (check-flavored) diff --git a/pkg/sync/ingest_invariants_granularity_test.go b/pkg/sync/ingest_invariants_granularity_test.go index 72f65a98b..d3216a01d 100644 --- a/pkg/sync/ingest_invariants_granularity_test.go +++ b/pkg/sync/ingest_invariants_granularity_test.go @@ -20,6 +20,7 @@ import ( v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" reader_v2 "github.com/conductorone/baton-sdk/pb/c1/reader/v2" "github.com/conductorone/baton-sdk/pkg/connectorstore" + "github.com/conductorone/baton-sdk/pkg/dotc1z" "github.com/conductorone/baton-sdk/pkg/logging" "github.com/conductorone/baton-sdk/pkg/sourcecache" "github.com/conductorone/baton-sdk/pkg/types/grant" @@ -45,10 +46,12 @@ func TestIngestInvariantI8ExemptionIsPerGrant(t *testing.T) { // connector-owned dangling reference. plainGrant := grant.NewGrant(repo, "viewer", ghost.GetId()) + // NO resource-type row for eq_repo: a disabled-type configuration + // gap, which is the DROP arm of the replay policy (enabled-type + // danglings fail the sync instead — see the referential tests). cur := newRepairTestStore(ctx, t, filepath.Join(tmpDir, "cur.c1z"), tmpDir) _, err = cur.StartNewSync(ctx, connectorstore.SyncTypeFull, "") require.NoError(t, err) - require.NoError(t, cur.PutResourceTypes(ctx, equivRepoRT)) require.NoError(t, cur.PutResources(ctx, repo)) require.NoError(t, cur.PutGrants(ctx, irgGrant, plainGrant)) @@ -100,10 +103,10 @@ func TestIngestInvariantI8MixedGrantsUnderOneEntitlement(t *testing.T) { }.Build() plainGrant := grant.NewGrant(repo, "admin", ghost.GetId()) // same entitlement, no annotation + // Disabled-type gap (no eq_repo type row): the drop arm. cur := newRepairTestStore(ctx, t, filepath.Join(tmpDir, "cur.c1z"), tmpDir) _, err = cur.StartNewSync(ctx, connectorstore.SyncTypeFull, "") require.NoError(t, err) - require.NoError(t, cur.PutResourceTypes(ctx, equivRepoRT)) require.NoError(t, cur.PutResources(ctx, repo)) require.NoError(t, cur.PutGrants(ctx, irgGrant, plainGrant)) @@ -155,12 +158,29 @@ func TestIngestInvariantI6SkippedForExpandOnlySyncs(t *testing.T) { s := &syncer{store: cur, syncType: connectorstore.SyncTypeFull, failFastInvariants: true} - // Sanity: a normal sync detects the orphan (replay-integrity class). + // Sanity: a normal sync detects the orphan and fails in EVERY mode + // (replay policy: a stale replay index is never sealed away). This + // syncer is COLD (no previous sync), so the failure is attributed + // plainly — the replay-integrity sentinel rides only warm syncs, + // where a discarded output and a cold retry can remediate. err = s.checkSourceCacheScopeConsistency(ctx) require.Error(t, err) - require.ErrorIs(t, err, ErrReplayIntegrity) + require.NotErrorIs(t, err, ErrReplayIntegrity) require.Contains(t, err.Error(), "ingest invariant I6") + // Warm shape: the same evidence carries the sentinel so the runners + // discard the output and retry cold. + s.sourceCache.prev = struct{ dotc1z.SourceCacheStore }{} + err = s.checkSourceCacheScopeConsistency(ctx) + require.ErrorIs(t, err, ErrReplayIntegrity) + s.sourceCache.prev = nil + + // Default mode is no longer warn-and-seal: the same cold failure. + s.failFastInvariants = false + err = s.checkSourceCacheScopeConsistency(ctx) + require.Error(t, err, "I6 must fail in default mode too — warn-and-seal would publish a poisoned replay source") + s.failFastInvariants = true + // The expansion-only pass must skip the check entirely. s.onlyExpandGrants = true require.NoError(t, s.checkSourceCacheScopeConsistency(ctx), diff --git a/pkg/sync/ingest_invariants_referential_test.go b/pkg/sync/ingest_invariants_referential_test.go index c4fe51f04..9373cae08 100644 --- a/pkg/sync/ingest_invariants_referential_test.go +++ b/pkg/sync/ingest_invariants_referential_test.go @@ -1,13 +1,17 @@ package sync //nolint:revive,nolintlint // we can't change the package name for backwards compatibility -// I7/I8/I9 referential-invariant tests: entitlements referencing -// vanished resources (I7), grants referencing vanished entitlements -// (I8), and grants referencing vanished principals (I9) — connector -// magic-id bugs and the holes type-granularity scopes open. All are -// connector-attributable: default mode DROPS the rows (platform uplift -// discards them anyway) with one aggregated warning, fail-fast mode fails, -// and none carry ErrReplayIntegrity (a cold retry is clean on a -// dishonest validator, so the sentinel would loop forever). +// I7/I8/I9 referential-invariant tests, under the replay policy's +// attribution split: +// +// - DISABLED-TYPE gaps (the referent's resource type was never +// synced): expected configuration shapes. Default mode DROPS the +// dependent rows with one aggregated warning and seals a +// referentially consistent artifact; fail-fast mode fails. The drop +// fixtures below deliberately omit the referent's type row. +// - ENABLED-TYPE danglings (the type IS synced, the row is not): +// never sealed away. Warm syncs fail with ErrReplayIntegrity +// (discard + cold retry); cold syncs fail plainly (connector +// data/behavior). See the *_EnabledTypeFails tests. import ( "path/filepath" @@ -36,11 +40,11 @@ func TestIngestInvariantI7DropsDanglingEntitlements(t *testing.T) { require.NoError(t, err) ent := et.NewAssignmentEntitlement(repo, "admin") - // The violating shape: an entitlement row whose resource has no row. + // The violating shape: an entitlement row whose resource has no row — + // and no TYPE row either (disabled-type gap: the drop arm). cur := newRepairTestStore(ctx, t, filepath.Join(tmpDir, "cur.c1z"), tmpDir) _, err = cur.StartNewSync(ctx, connectorstore.SyncTypeFull, "") require.NoError(t, err) - require.NoError(t, cur.PutResourceTypes(ctx, equivRepoRT)) require.NoError(t, cur.PutEntitlements(ctx, ent)) s := &syncer{store: cur, syncType: connectorstore.SyncTypeFull} @@ -85,11 +89,12 @@ func TestIngestInvariantI8DropsDanglingGrants(t *testing.T) { }.Build() g := grant.NewGrant(repo, "admin", principal.GetId()) - // The violating shape: a grant row whose entitlement has no row. + // The violating shape: a grant row whose entitlement has no row — + // and no TYPE row for the entitlement's resource type (disabled-type + // gap: the drop arm). cur := newRepairTestStore(ctx, t, filepath.Join(tmpDir, "cur.c1z"), tmpDir) _, err = cur.StartNewSync(ctx, connectorstore.SyncTypeFull, "") require.NoError(t, err) - require.NoError(t, cur.PutResourceTypes(ctx, equivRepoRT)) require.NoError(t, cur.PutResources(ctx, repo)) require.NoError(t, cur.PutGrants(ctx, g)) @@ -266,9 +271,11 @@ func TestIngestInvariantI9ExemptsExternalMatchCarriers(t *testing.T) { // TestDishonestValidatorStrandsGrantEntitlements is the end-to-end I8 // scenario: a connector whose grants validator says "unchanged" (so the // scope replays) while its entitlement listing was refreshed and dropped -// the entitlement. Fail-fast mode names it without the replay sentinel; -// default mode drops the stranded grants and the sync converges to what -// a cold sync would have produced. +// the entitlement. The stranded grants dangle under an ENABLED type, so +// the replay policy applies: the warm output is DISCARDED +// (ErrReplayIntegrity — the runners' cold-retry ladder) and the cold +// re-run, whose fresh fetch has no stranded grants, seals clean. The +// artifact is never silently sanitized. func TestDishonestValidatorStrandsGrantEntitlements(t *testing.T) { ctx, err := logging.Init(t.Context()) require.NoError(t, err) @@ -287,48 +294,132 @@ func TestDishonestValidatorStrandsGrantEntitlements(t *testing.T) { // The dishonesty: the entitlement listing drops the member // entitlement while the grants validator still reports "unchanged". + // The membership grants also vanish upstream (the fresh fetch has + // none) — only the REPLAY carries the stranded grants forward. mc.entDB[group.GetId().GetResource()] = nil + mc.grantDB[group.GetId().GetResource()] = nil - // Fail-fast warm sync: fails, named, not replay-classified. + // Warm sync (default mode): the stranded grants dangle under the + // SYNCED group type — replay is implicated, the output must be + // discarded and retried cold, never sealed sanitized. store2, err := dotc1z.NewStore(ctx, filepath.Join(tmpDir, "s2.c1z"), dotc1z.WithEngine(c1zstore.EnginePebble), dotc1z.WithTmpDir(tmpDir), ) require.NoError(t, err) - warmStrict, err := NewSyncer(ctx, mc, + warm, err := NewSyncer(ctx, mc, WithConnectorStore(store2), WithTmpDir(tmpDir), WithPreviousSyncC1ZPath(sync1), - WithFailFastInvariants(), ) require.NoError(t, err) - err = warmStrict.Sync(ctx) + err = warm.Sync(ctx) require.Error(t, err) require.Contains(t, err.Error(), "ingest invariant I8") - require.NotErrorIs(t, err, ErrReplayIntegrity, - "a dishonest validator must not trigger the cold-retry ladder: the cold run is clean, "+ - "so the retry would mask the connector bug on every sync forever") - require.NoError(t, warmStrict.Close(ctx)) + require.ErrorIs(t, err, ErrReplayIntegrity, + "stranded grants under an enabled type on a warm sync must route to the cold-retry ladder") + require.NoError(t, warm.Close(ctx)) - // Default-mode warm sync: succeeds, and the stranded grants are DROPPED — - // the artifact converges to cold-sync semantics. + // The cold retry (what the runners do with ErrReplayIntegrity): + // clean — the fresh fetch never had the stranded grants. sync3 := filepath.Join(tmpDir, "s3.c1z") - store3, err := dotc1z.NewStore(ctx, sync3, - dotc1z.WithEngine(c1zstore.EnginePebble), - dotc1z.WithTmpDir(tmpDir), - ) - require.NoError(t, err) - warmLenient, err := NewSyncer(ctx, mc, - WithConnectorStore(store3), - WithTmpDir(tmpDir), - WithPreviousSyncC1ZPath(sync1), - ) - require.NoError(t, err) - require.NoError(t, warmLenient.Sync(ctx)) - require.NoError(t, warmLenient.Close(ctx)) - + runSourceCacheSync(ctx, t, mc, sync3, "", tmpDir) grants3 := listGrantsInFile(ctx, t, sync3) require.NotContains(t, grants3, stranded.GetId(), - "default mode must drop grants stranded by the dishonest validator") + "the cold retry's artifact must not contain the stranded grant") _ = memberEnt } + +// TestIngestInvariantEnabledTypeDanglingsFail pins the replay policy's +// strict arm for all three referential invariants: a missing referent +// whose resource TYPE is synced is never sealed away. Cold syncs fail +// plainly (connector data/behavior — the retry could not help); warm +// syncs fail with ErrReplayIntegrity (discard + cold retry); the +// compaction expand pass is exempt (keep-newer merges legitimately +// manufacture dangling refs, and its drop pass converges the artifact). +func TestIngestInvariantEnabledTypeDanglingsFail(t *testing.T) { + ctx, err := logging.Init(t.Context()) + require.NoError(t, err) + tmpDir := t.TempDir() + + repo, err := rs.NewResource("Repo r1", equivRepoRT, "r1") + require.NoError(t, err) + ent := et.NewAssignmentEntitlement(repo, "admin") + userRT := v2.ResourceType_builder{Id: "user", DisplayName: "User"}.Build() + ghost := v2.Resource_builder{ + Id: v2.ResourceId_builder{ResourceType: "user", Resource: "ghost"}.Build(), + }.Build() + + // All three shapes, every referent under a SYNCED type: an + // entitlement whose resource row is missing (I7), a grant whose + // entitlement row is missing (I8), and a grant whose principal row + // is missing (I9). + cur := newRepairTestStore(ctx, t, filepath.Join(tmpDir, "cur.c1z"), tmpDir) + _, err = cur.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + require.NoError(t, cur.PutResourceTypes(ctx, equivRepoRT, userRT)) + + // I7 shape: entitlement row, no resource row, type row present. + require.NoError(t, cur.PutEntitlements(ctx, ent)) + + s := &syncer{store: cur, syncType: connectorstore.SyncTypeFull} + + // COLD: plain failure, no sentinel, nothing dropped. + err = s.checkEntitlementResourceReferences(ctx) + require.Error(t, err) + require.Contains(t, err.Error(), "ingest invariant I7") + require.Contains(t, err.Error(), "SYNCED type") + require.NotErrorIs(t, err, ErrReplayIntegrity) + _, err = cur.GetEntitlement(ctx, reader_v2.EntitlementsReaderServiceGetEntitlementRequest_builder{ + EntitlementId: ent.GetId(), + }.Build()) + require.NoError(t, err, "an enabled-type dangling must never be dropped") + + // WARM: same evidence carries the replay-integrity sentinel. + s.sourceCache.prev = struct{ dotc1z.SourceCacheStore }{} + err = s.checkEntitlementResourceReferences(ctx) + require.ErrorIs(t, err, ErrReplayIntegrity) + s.sourceCache.prev = nil + + // COMPACTION EXPAND PASS: exempt — the drop arm converges the + // merged artifact (keep-newer merges manufacture these by design). + s.onlyExpandGrants = true + require.NoError(t, s.checkEntitlementResourceReferences(ctx), + "the compaction expand pass must drop, not fail: merges manufacture dangling refs no input contained") + s.onlyExpandGrants = false + + // I8 + I9 shapes: resource row present now; a grant under a missing + // entitlement (enabled repo type) and a grant to a missing principal + // (enabled user type). + require.NoError(t, cur.PutResources(ctx, repo)) + gEnt := grant.NewGrant(repo, "missing-ent", ghost.GetId()) + require.NoError(t, cur.PutGrants(ctx, gEnt)) + + err = s.checkGrantEntitlementReferences(ctx) + require.Error(t, err) + require.Contains(t, err.Error(), "ingest invariant I8") + require.Contains(t, err.Error(), "SYNCED type") + require.NotErrorIs(t, err, ErrReplayIntegrity) + s.sourceCache.prev = struct{ dotc1z.SourceCacheStore }{} + require.ErrorIs(t, s.checkGrantEntitlementReferences(ctx), ErrReplayIntegrity) + s.sourceCache.prev = nil + + // Heal I8's shape (put its entitlement) so I9 is isolated: the + // grant's principal "user/ghost" has a type row but no resource row. + missingEnt := et.NewAssignmentEntitlement(repo, "missing-ent") + require.NoError(t, cur.PutEntitlements(ctx, missingEnt)) + err = s.checkGrantPrincipalReferences(ctx) + require.Error(t, err) + require.Contains(t, err.Error(), "ingest invariant I9") + require.Contains(t, err.Error(), "SYNCED type") + require.NotErrorIs(t, err, ErrReplayIntegrity) + s.sourceCache.prev = struct{ dotc1z.SourceCacheStore }{} + require.ErrorIs(t, s.checkGrantPrincipalReferences(ctx), ErrReplayIntegrity) + s.sourceCache.prev = nil + _, err = cur.GetGrant(ctx, reader_v2.GrantsReaderServiceGetGrantRequest_builder{ + GrantId: gEnt.GetId(), + }.Build()) + require.NoError(t, err, "enabled-type danglings must never be dropped in any failing mode") + + require.NoError(t, cur.Close(ctx)) +} diff --git a/pkg/sync/ingest_invariants_scale_test.go b/pkg/sync/ingest_invariants_scale_test.go index 856db9036..f2271dc9a 100644 --- a/pkg/sync/ingest_invariants_scale_test.go +++ b/pkg/sync/ingest_invariants_scale_test.go @@ -41,10 +41,11 @@ func TestIngestInvariantI8DropScale(t *testing.T) { repo, err := rs.NewResource("Repo r1", equivRepoRT, "r1") require.NoError(t, err) + // No eq_repo TYPE row: a disabled-type gap, which is the drop arm of + // the replay policy (an enabled-type dangling would fail instead). cur := newRepairTestStore(ctx, t, filepath.Join(tmpDir, "cur.c1z"), tmpDir) _, err = cur.StartNewSync(ctx, connectorstore.SyncTypeFull, "") require.NoError(t, err) - require.NoError(t, cur.PutResourceTypes(ctx, equivRepoRT)) require.NoError(t, cur.PutResources(ctx, repo)) const grantCount = 100_000 diff --git a/pkg/sync/pebble_etag_replay_test.go b/pkg/sync/pebble_etag_replay_test.go index 045392bbd..cb832323f 100644 --- a/pkg/sync/pebble_etag_replay_test.go +++ b/pkg/sync/pebble_etag_replay_test.go @@ -354,6 +354,11 @@ func TestOptionalPreviousSyncC1ZPath_SoftFails(t *testing.T) { t.Run(name, func(t *testing.T) { mc := newEtagObservingMockConnector("etag-v1") mc.WithData(group, ent, grant) + // The grant's principal must exist: a missing row under the + // synced user type is an enabled-type dangling and fails the + // sync under the replay policy (this fixture was previously + // referentially dishonest and the drop arm hid it). + mc.AddResource(ctx, user) store, err := dotc1z.NewStore(ctx, filepath.Join(t.TempDir(), "out.c1z"), dotc1z.WithEngine(c1zstore.EnginePebble), dotc1z.WithTmpDir(tempDir), diff --git a/pkg/sync/source_cache.go b/pkg/sync/source_cache.go index 54e1b5973..65e39d2b2 100644 --- a/pkg/sync/source_cache.go +++ b/pkg/sync/source_cache.go @@ -68,6 +68,11 @@ type syncerSourceCache struct { // on the shared subprocess slot that would decrement another live // sync's install and blind it (see sourcecache.GRPCServer). lookupInstalled bool + // compatKey is this sync's replay-compatibility key, computed by + // configureSourceCache and persisted into the artifact right after + // the sync starts (see Sync). The read side requires the previous + // artifact's recorded key to match it byte-exactly. + compatKey sourcecache.CompatKey } type scopeRef struct { @@ -141,6 +146,22 @@ func (s *syncer) configureSourceCache(ctx context.Context, resp *v2.ConnectorSer setLookup.SetSourceCache(ctx, sourcecache.NoopLookup{}) s.sourceCache.lookupInstalled = true } + // An operator who EXPLICITLY configured --previous-sync-c1z (not + // the runner's optional spare) is expecting replay; degrading to + // a cold sync at Info level left them silently replay-less — the + // default storage engine (SQLite) doesn't support source cache, + // so the flag alone quietly did nothing without + // --storage-engine pebble. Warn with the reason instead. + if s.previousSyncC1ZPath != "" && !s.previousSyncC1ZPathOptional { + if reason == "" { + reason = "connector does not declare the source-cache capability" + } + l.Warn("source cache disabled; --previous-sync-c1z will be ignored and the sync runs cold", + zap.String("reason", reason), + zap.String("previous_sync_c1z", s.previousSyncC1ZPath), + ) + return nil + } if reason != "" { l.Info("source cache disabled", zap.String("reason", reason)) } @@ -173,10 +194,29 @@ func (s *syncer) configureSourceCache(ctx context.Context, resp *v2.ConnectorSer return degrade("storage engine does not support source cache") } + // The replay-compatibility key: the exact conditions this sync's + // manifest is recorded under, and the conditions a previous artifact + // must have been recorded under to serve replay. Components are + // explicit declarations (never inferred from versions): the + // connector's cache generation and config/permission fingerprint + // (Validate annotations), the SDK's materialization-policy + // generation, and the sync-selection fingerprint. + compatKey := sourcecache.CompatKey{ + ConnectorCacheGeneration: capability.GetCacheGeneration(), + ConnectorConfigFingerprint: capability.GetConfigFingerprint(), + SDKMaterializationGeneration: sourcecache.MaterializationPolicyGeneration, + SyncSelectionFingerprint: sourcecache.SelectionFingerprint(s.syncResourceTypes, s.skipEntitlementsAndGrants, s.skipGrants), + } + // Write side enabled: rows produced under a scope get stamped and // manifest entries get written, so this sync is usable as the NEXT // sync's replay source even when this one has nothing to replay from. - s.sourceCache = syncerSourceCache{enabled: true, current: current, replayedScopes: &replayedScopeSet{}} + // The compat key is stashed here and persisted by Sync right after + // the sync starts (configureSourceCache runs before StartOrResumeSync, + // so the store has no current sync yet) — still BEFORE any page could + // write a manifest entry, so the artifact can never hold validators + // without the key that scopes their validity. + s.sourceCache = syncerSourceCache{enabled: true, current: current, replayedScopes: &replayedScopeSet{}, compatKey: compatKey} // Read side: a usable previous sync makes lookups hit and replay legal. var readReason string @@ -196,6 +236,12 @@ func (s *syncer) configureSourceCache(ctx context.Context, resp *v2.ConnectorSer // (suspenders) — "what kind of sync is this" answers "can it be // used for replay" without inspecting keyspaces. readReason = reason + } else if compatReason := replayCompatMismatchReason(ctx, compatKey, prev); compatReason != "" { + // The replay-compatibility gate: exact key match or cold. An + // absent key (pre-key artifact, fold) and an unreadable key both + // count as mismatches — replay is optional, and "cannot prove + // compatible" means incompatible. + readReason = compatReason } else { s.sourceCache.prev = prev s.sourceCache.prevReader = s.previousSyncReader @@ -222,6 +268,24 @@ func (s *syncer) configureSourceCache(ctx context.Context, resp *v2.ConnectorSer return nil } +// replayCompatMismatchReason evaluates the replay-compatibility gate: +// the previous artifact's recorded key must match the current run's key +// byte-exactly on every component, or the sync runs cold. Absent +// (pre-key artifacts, compaction folds) and unreadable keys are +// mismatches, not errors: replay is a pure optimization, and a key we +// cannot verify is a key that does not match. +func replayCompatMismatchReason(ctx context.Context, current sourcecache.CompatKey, prev dotc1z.SourceCacheStore) string { + prevKey, found, err := prev.GetSourceCacheCompat(ctx) + if err != nil { + ctxzap.Extract(ctx).Warn("error reading previous sync's replay-compatibility key; running cold", zap.Error(err)) + return "previous sync's replay-compatibility key is unreadable" + } + if !found { + return "previous sync carries no replay-compatibility key (recorded by a pre-key SDK, or a compaction fold)" + } + return current.MismatchReason(prevKey) +} + // previousSyncReplayUnusableReason inspects the previous-sync reader's // newest finished run record against the replay-source predicate // (c1zstore.SyncRun.UsableAsReplaySource) and returns a human-readable @@ -465,13 +529,14 @@ func (s *syncer) executeScopeReplay(ctx context.Context, kind sourcecache.RowKin } if res.StaleSkipped > 0 { // Partial staleness: some rows copied, some index entries were - // dead. Not data loss by itself (the live rows carried their - // true stamp), but a signal a writer skipped index cleanup. - ctxzap.Extract(ctx).Warn("source cache: replay skipped stale scope-index entries", - zap.String("scope_key", page.scopeKey), - zap.Int64("rows", res.Rows), - zap.Int64("stale_skipped", res.StaleSkipped), - ) + // dead — a writer skipped index cleanup somewhere upstream. The + // replay policy treats a stale replay index as untrustworthy + // wholesale: discard the warm output and re-run cold rather than + // guess which side of the divergence is right. + return fmt.Errorf( + "source cache: replay for scope %q skipped %d stale scope-index entries (rows copied: %d); "+ + "the previous file's replay index cannot be trusted: %w", + page.scopeKey, res.StaleSkipped, res.Rows, ErrReplayIntegrity) } if res.ResumedRowsCleared > 0 { // A re-run over a crashed attempt's committed rows: the engine diff --git a/pkg/sync/source_cache_continuation.go b/pkg/sync/source_cache_continuation.go index 0dc8b5967..2cd64b62d 100644 --- a/pkg/sync/source_cache_continuation.go +++ b/pkg/sync/source_cache_continuation.go @@ -50,9 +50,24 @@ const ( // later request field's validator. maxEnqueuedPageTokenBytes = 1 << 20 - // sourceCacheAnswerBudget caps the FOUND-etag payload attached to one - // re-invoke, keeping the request under single-shot transport payload - // limits (Lambda invokes cap at 6MB; the dual-encoded frame at 5MiB). + // maxEnqueuedPageTokenTotalBytes bounds the AGGREGATE token bytes of + // one EnqueuePageTokens response. The per-item caps alone still admit + // 1024 × 1MiB = 1GiB of tokens in a single legal response — all of it + // persisted into every subsequent checkpoint token until the actions + // drain. Real planner tokens are tiny (chunk ids, cursor strings); + // 16MiB of aggregate budget is orders of magnitude above any sane + // fan-out while keeping worst-case checkpoints bounded. + maxEnqueuedPageTokenTotalBytes = 16 << 20 + + // sourceCacheAnswerBudget caps the TOTAL answers payload attached to + // one re-invoke — validator bytes AND per-answer identity bytes + // (row_kind, scope_key, proto framing) — keeping the request under + // single-shot transport payload limits (Lambda invokes cap at 6MB; + // the dual-encoded frame at 5MiB). Identity bytes must be charged + // too: at the proto caps (16384 accumulated answers × 256-byte scope + // keys) identities alone reach ~5MiB with zero validator bytes, so a + // validator-only budget could not actually bound the request. + // // A found answer whose etag does not fit the remaining budget is // DEGRADED to a not-found answer: the connector fetches that scope // fresh (cold-correct, just slower). Leaving it absent-and-re-askable @@ -60,8 +75,18 @@ const ( // re-executes from scratch each phase, so every re-invoke carries the // union), so the budget only shrinks and an etag that did not fit once // can never fit later; re-asking would burn straight into the bounce - // cap and hard-fail a sync that cold-degrades correctly. + // cap and hard-fail a sync that cold-degrades correctly. Identity + // bytes are NOT degradable (a not-found answer carries the same + // identity, and an absent answer livelocks), so identity bytes alone + // exceeding the budget is a hard failure — the connector asked for an + // answer set no re-invoke could ever carry. sourceCacheAnswerBudget = 2 << 20 + + // sourceCacheAnswerOverheadBytes approximates one answer's non-string + // proto framing (field tags, length prefixes, the found bool, the Any + // wrapping amortized across the message). Deliberately generous: the + // budget exists to stay under transport limits, not to be exact. + sourceCacheAnswerOverheadBytes = 16 ) // Continuation counters live in SourceCacheStats on the sync state (see @@ -167,9 +192,12 @@ func (s *syncer) withSourceCacheContinuation(ctx context.Context, op string, iss return fmt.Errorf("%s: invalid source-cache lookup ask: %w", op, err) } + answerIdentityCost := func(q sourcecache.Query) int { + return len(q.RowKind) + len(q.ScopeKey) + sourceCacheAnswerOverheadBytes + } budget := sourceCacheAnswerBudget for _, a := range ordered { - budget -= len(a.CacheValidator) + budget -= answerIdentityCost(a.Query) + len(a.CacheValidator) } newAsked, newFound, newNotFound, newTruncated := 0, 0, 0, 0 for _, q := range queries { @@ -177,6 +205,18 @@ func (s *syncer) withSourceCacheContinuation(ctx context.Context, op string, iss continue } newAsked++ + // Identity bytes are spent for EVERY answered query — found, + // not-found, and degraded alike (the answer must exist or the + // connector livelocks re-asking it). If identities alone blow + // the budget, no degradation can produce a carryable request. + identityCost := answerIdentityCost(q) + if identityCost > budget { + s.state.AddSourceCacheStats(SourceCacheStats{LookupCapFailures: 1}) + return fmt.Errorf("%s: source-cache lookup answers exceed the %d-byte request budget on identity bytes alone "+ + "(%d answers accumulated); the connector's asks name more scope bytes than any re-invoke can carry", + op, sourceCacheAnswerBudget, len(ordered)) + } + budget -= identityCost entry, ok, err := s.sourceCache.lookup.Lookup(ctx, q.RowKind, q.ScopeKey) if err != nil { return fmt.Errorf("%s: resolving source-cache lookup ask: %w", op, err) diff --git a/pkg/sync/source_cache_continuation_test.go b/pkg/sync/source_cache_continuation_test.go index 89018a885..91d146771 100644 --- a/pkg/sync/source_cache_continuation_test.go +++ b/pkg/sync/source_cache_continuation_test.go @@ -687,7 +687,20 @@ func TestSourceCacheContinuation_AnswerBudgetDegradesToCold(t *testing.T) { const queryCount = 40 etag := strings.Repeat("e", 65536) // the wire cap for one etag - fitting := sourceCacheAnswerBudget / len(etag) + // Mirror the budget arithmetic: every answered query spends its + // identity bytes (row_kind + scope_key + framing overhead) whether + // found or degraded; a found answer additionally needs its etag to + // fit the remaining budget. + identityCost := len(sourcecache.RowKindGrants) + len("scope-000") + sourceCacheAnswerOverheadBytes + fitting := 0 + for i, budget := 0, sourceCacheAnswerBudget; i < queryCount; i++ { + budget -= identityCost + if len(etag) <= budget { + budget -= len(etag) + fitting++ + } + } + require.Positive(t, fitting, "test must fit some answers") require.Less(t, fitting, queryCount, "test must overflow the budget") s := &syncer{ diff --git a/pkg/sync/source_cache_equivalence_dirty_test.go b/pkg/sync/source_cache_equivalence_dirty_test.go new file mode 100644 index 000000000..222c2e910 --- /dev/null +++ b/pkg/sync/source_cache_equivalence_dirty_test.go @@ -0,0 +1,509 @@ +package sync //nolint:revive,nolintlint // we can't change the package name for backwards compatibility + +// Dirty-connector replay-equivalence scenarios. +// +// THE PROPERTY: default-mode warm ≡ default-mode cold, for connectors +// whose data carries EXPECTED configuration gaps — dangling references +// whose referent's resource TYPE is not synced (disabled). Under the +// replay policy those are the DROP arm of the referential invariants +// (I7/I8/I9): dependent rows drop with one aggregated warning, the +// artifact seals referentially consistent, and every dropped-from +// scope's manifest entry is INVALIDATED so the scope re-fetches cold +// next round. Enabled-type danglings are NOT equivalence material — they +// fail the sync (warm: ErrReplayIntegrity + cold retry; cold: plain) and +// are pinned by TestIngestInvariantEnabledTypeDanglingsFail and +// TestDishonestValidatorStrandsGrantEntitlements. +// +// The clean harness (source_cache_equivalence_test.go) runs fail-fast, +// so the drop arms never execute inside its property; this family runs +// default mode on both legs. The manifest-invalidation-on-drop contract +// is what the dirty→clean transition pins: without it, a later honest +// 304 replays the sanitized subset while a cold sync fetches the full +// set — warm ≠ cold exactly when a dropped row's referent appears. +// +// DIRTY SHAPES (all disabled-type; forced, every scenario): +// - I9: groups hold members of the SHADOW type (ids "shadow-*"), +// which is disabled at round 0 — member grants drop; and of the +// ghost2 type (ids "ghost2-*"), which NEVER enables — steady dirty. +// - I8: gd8's grants page emits a grant on the shadow HOST's "admin" +// entitlement, which exists only once the shadow type is enabled. +// - I7 (type-scoped): team chunk 0's entitlements page emits an +// entitlement for the shadow host2 resource (and chunk 0's grants +// page a grant under it — the I8 cascade). +// - Exempt carriers: the ext group's unprocessed ExternalResourceMatch +// carriers (no external c1z configured in this family) are KEPT in +// both artifacts every round. +// +// TRANSITIONS (round schedule; see mutateDirty): +// r0 initial dirty — shadow type disabled: I7, I8, and I9 drops fire. +// r1 dirty→clean (THE BUG CELL): the shadow type is ENABLED — its +// type row, resources, and host entitlements appear — while every +// dropped-from GRANTS/ENTS scope's validator stays put. Without +// manifest invalidation the warm side 304s the sanitized subsets +// and diverges from cold here. +// r2 no-op + crash/resume: the warm sync is crashed mid-grants on a +// steady-dirty scope and resumed (drops + invalidation must +// behave identically through resume). +// r3 clean→dirty via OVERLAY: gdelta's delta round adds a ghost2 +// member — dangling data arriving on the overlay path. +// r4 no-op — steady dirty chain-decay probe: ghost2 scopes re-fetch +// cold (invalidated), re-drop, re-warn; all clean scopes 304; the +// output must STILL equal cold. +// +// ORACLES: the clean harness's full semantic diff + manifest and +// scope-index comparison (manifest entries carry the invalidation +// verdict); exact model-derived post-drop row sets asserted against +// BOTH artifacts; and the warning-silence oracle INVERTED — every round +// asserts the expected "ingest invariant ... DROPPED" warnings are +// present on both legs (the anti-vacuity proof that drops executed) and +// that nothing unexpected warned. A dirty scenario that never dropped +// anything fails itself. + +import ( + "fmt" + "math/rand" + "path/filepath" + "strings" + "testing" + + "github.com/stretchr/testify/require" + + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + "github.com/conductorone/baton-sdk/pkg/logging" + "github.com/conductorone/baton-sdk/pkg/types" + gt "github.com/conductorone/baton-sdk/pkg/types/grant" +) + +// newDirtyEquivModel is the clean topology plus the dirty shapes. The +// clean groups/orgs/teams are kept so the dirty rounds still have a +// large clean surface that must keep 304ing correctly around the drops. +func newDirtyEquivModel() *equivModel { + m := newEquivModel() + // I9 dirty→clean: a shadow-typed member, missing until the shadow + // type is enabled in round 1. + m.groups["gd9"] = &equivGroup{id: "gd9", members: map[string]bool{"u0": true, "shadow-mem": true}} + // I9 steady dirty: a ghost2-typed member whose type never enables. + m.groups["gd"] = &equivGroup{id: "gd", members: map[string]bool{"u0": true, "ghost2-mem": true}} + // I9 clean→dirty via overlay: gains a ghost2 member in round 3. + m.groups["gdelta"] = &equivGroup{id: "gdelta", members: map[string]bool{"u3": true}} + // I8: the shadow-host admin grant is emitted every round; its + // entitlement row exists only from round 1. + m.groups["gd8"] = &equivGroup{id: "gd8", members: map[string]bool{"u1": true}, ghostAdminGrant: true} + // I7 (+ I8 cascade) in the type-scoped chunk scopes. + m.ghostTeamEnt = true + // The shadow type's row universe once enabled. + m.shadowUsers = map[string]bool{"shadow-mem": true} + return m +} + +// mutateDirty is the dirty family's forced round schedule (no seeded +// churn: every transition is load-bearing and must occur exactly where +// the oracles expect it). +func (m *equivModel) mutateDirty(round int) { + m.clearDeltas() + switch round { + case 1: + // Dirty→clean: ENABLE the shadow resource type. Its type row, + // resources, and host entitlements appear via their own pages — + // every previously dropped-from scope's validator stays put. + m.shadowEnabled = true + case 3: + // Clean→dirty via a delta overlay round (ghost2 never enables). + m.addMember("gdelta", "ghost2-d") + default: + // Rounds 2 and 4: no-op (r2 additionally crashes/resumes the + // warm leg; steady-dirty scopes re-fetch and re-drop each round). + } +} + +// expectedDirtyResourceKeys is expectedResourceKeys minus external +// principals (no external c1z in this family) plus the shadow universe +// once enabled. +func expectedDirtyResourceKeys(m *equivModel) []string { + set := map[string]struct{}{} + for uid := range m.users { + set[userResourceType.GetId()+"/"+uid] = struct{}{} + } + for gid := range m.groups { + set[groupResourceType.GetId()+"/"+gid] = struct{}{} + } + for orgID, org := range m.orgs { + set[equivOrgRT.GetId()+"/"+orgID] = struct{}{} + for projID := range org.projects { + set[equivProjectRT.GetId()+"/"+projID] = struct{}{} + } + for repoID := range org.repoGrants { + set[equivRepoRT.GetId()+"/"+repoID] = struct{}{} + } + } + for tid := range m.teams { + set[equivTeamRT.GetId()+"/"+tid] = struct{}{} + } + if m.shadowEnabled { + for id := range m.shadowUsers { + set[equivShadowRT.GetId()+"/"+id] = struct{}{} + } + set[equivShadowRT.GetId()+"/"+equivShadowHostID] = struct{}{} + set[equivShadowRT.GetId()+"/"+equivShadowHost2ID] = struct{}{} + } + return sortedKeys(set) +} + +// expectedDirtyEntitlementIDs is the POST-DROP entitlement set: the +// shadow hosts' entitlements exist only while the type is enabled (I7 +// drops the host2 entitlement otherwise; the host's admin entitlement +// is simply not listed). +func expectedDirtyEntitlementIDs(m *equivModel) []string { + set := map[string]struct{}{} + for gid, g := range m.groups { + set[equivMemberEnt(equivGroupResource(gid), g.exclusionDefault).GetId()] = struct{}{} + } + for tid := range m.teams { + set[equivMemberEnt(equivTeamResource(tid), false).GetId()] = struct{}{} + } + if m.shadowEnabled { + set[equivShadowAdminEnt().GetId()] = struct{}{} + if m.ghostTeamEnt { + set[equivShadowHost2Ent().GetId()] = struct{}{} + } + } + return sortedKeys(set) +} + +// expectedDirtyGrantIDs is the POST-DROP grant set: +// - member grants for user-typed members always; shadow members only +// while the type is enabled (I9 drops them otherwise); ghost2 +// members never (their type never enables); +// - gd8's shadow-admin grant and the chunk-0 host2 grant only while +// the shadow type is enabled (I8 / I7-cascade drops otherwise); +// - the ext group's match carriers are KEPT (exempt) every round. +func expectedDirtyGrantIDs(m *equivModel) []string { + set := map[string]struct{}{} + memberExpected := func(uid string) bool { + switch { + case strings.HasPrefix(uid, "shadow-"): + return m.shadowEnabled && m.shadowUsers[uid] + case strings.HasPrefix(uid, "ghost2-"): + return false + default: + return m.users[uid] + } + } + for gid, g := range m.groups { + if gid == m.extGroupID { + continue + } + for uid := range g.members { + if memberExpected(uid) { + set[equivMemberGrant(gid, uid).GetId()] = struct{}{} + } + } + if g.ghostAdminGrant && m.shadowEnabled { + set[equivShadowAdminGrant("u1").GetId()] = struct{}{} + } + } + // The expandable group-to-group membership and its derived grants. + parentRes := equivGroupResource(m.expandableParent) + parentEnt := equivMemberEnt(parentRes, false) + set[gt.NewGrantID(equivGroupResource(m.expandableChild), parentEnt)] = struct{}{} + for uid := range m.groups[m.expandableChild].members { + if m.users[uid] { + set[gt.NewGrantID( + v2.ResourceId_builder{ResourceType: userResourceType.GetId(), Resource: uid}.Build(), + parentEnt)] = struct{}{} + } + } + // Unprocessed match carriers survive whole (exempt in I9). + for _, extID := range m.extMatchIDs { + set[equivMemberGrant(m.extGroupID, "placeholder-"+extID).GetId()] = struct{}{} + } + set[equivMemberGrant(m.extGroupID, "placeholder-all").GetId()] = struct{}{} + for _, org := range m.orgs { + for repoID, uid := range org.repoGrants { + set[gt.NewGrant(equivRepoResource(repoID), "admin", + v2.ResourceId_builder{ResourceType: userResourceType.GetId(), Resource: uid}.Build()).GetId()] = struct{}{} + } + } + for tid, members := range m.teams { + for uid := range members { + set[equivTeamGrant(tid, uid).GetId()] = struct{}{} + } + } + if m.ghostTeamEnt && m.shadowEnabled { + set[equivShadowHost2Grant("u0").GetId()] = struct{}{} + } + return sortedKeys(set) +} + +// assertDirtyArtifacts asserts one artifact's row sets EXACTLY equal the +// model-derived post-drop expectation — dropped rows absent, exempt +// carriers and IRG shapes present. Applied to BOTH the cold and warm +// snapshots (the differential diff alone can't distinguish "both sides +// dropped" from "neither side dropped"). +func assertDirtyArtifacts(t *testing.T, round int, m *equivModel, label string, snap *c1zSnapshot) { + t.Helper() + require.Equal(t, expectedDirtyResourceKeys(m), sortedKeys(snap.resources), + "round %d: %s resource set must equal the post-drop expectation", round, label) + require.Equal(t, expectedDirtyEntitlementIDs(m), sortedKeys(snap.entitlements), + "round %d: %s entitlement set must equal the post-drop expectation", round, label) + require.Equal(t, expectedDirtyGrantIDs(m), sortedKeys(snap.grants), + "round %d: %s grant set must equal the post-drop expectation", round, label) + // Exempt carriers, named explicitly (also implied by the exact sets). + require.Contains(t, snap.grants, equivMemberGrant(m.extGroupID, "placeholder-ext-0").GetId(), + "round %d: %s must keep the exempt match carriers", round, label) +} + +// Invariant warning fingerprints (must match the aggregated warn +// messages in ingest_invariants.go). +const ( + dirtyWarnI7Drop = "ingest invariant I7: DROPPED entitlements" + dirtyWarnI8Drop = "ingest invariant I8: DROPPED grants" + dirtyWarnI9Drop = "ingest invariant I9: DROPPED grants" + dirtyWarnI9Kept = "ingest invariant I9: kept unprocessed external-match carrier grants" + dirtyWarnI3Never = "ingest invariant I3: grants reference a resource that was never synced" +) + +// dirtyExpectedWarnings returns the warning fingerprints REQUIRED on +// every leg of a round (warm and cold alike — drops are a pure function +// of the converged store). Anything captured that matches no required +// fingerprint fails the leg. +func dirtyExpectedWarnings(round int) []string { + if round == 0 { + // Shadow type disabled: the I7 host2 entitlement, the I8 + // shadow-admin + host2-cascade grants (I3 warns about their + // never-synced entitlement resources before I8 drops them), and + // the I9 shadow/ghost2 members, plus the kept-carriers report. + return []string{dirtyWarnI7Drop, dirtyWarnI8Drop, dirtyWarnI9Drop, dirtyWarnI9Kept, dirtyWarnI3Never} + } + // Rounds 1+: the shadow shapes are clean; only the ghost2 steady- + // dirty members keep dropping (I9) and the carriers keep being kept. + return []string{dirtyWarnI9Drop, dirtyWarnI9Kept} +} + +func assertDirtyWarnings(t *testing.T, round int, label string, msgs []string) { + t.Helper() + expected := dirtyExpectedWarnings(round) + for _, want := range expected { + found := false + for _, msg := range msgs { + if strings.Contains(msg, want) { + found = true + break + } + } + require.True(t, found, + "round %d (%s): expected invariant warning %q never fired — the drop arm did not execute (vacuous dirty round). captured: %v", + round, label, want, msgs) + } + for _, msg := range msgs { + allowed := false + for _, want := range expected { + if strings.Contains(msg, want) { + allowed = true + break + } + } + require.True(t, allowed, + "round %d (%s): unexpected ingestion-invariant warning %q", round, label, msg) + } +} + +const dirtyEquivRounds = 4 + +// dirtyCrashRound crashes the warm sync mid-grants and resumes it, in a +// round where the steady-dirty drops fire — drops and manifest +// invalidation must behave identically through resume. +const dirtyCrashRound = 2 + +// TestSourceCache_DirtyHaltAtInvariantSeam crashes a DIRTY sync at the +// "ingest-invariants-complete" seam — after the disabled-type drops (and +// their manifest invalidations) committed, before the checkpoint and +// EndSync — then resumes the same file. The resumed sync re-runs the +// whole invariant pass over already-dropped state, so this pins the +// post-drop crash contract end to end at the SYNCER level (the engine's +// chunk-boundary crash test pins the intra-drop window): drops are +// idempotent on re-run, the invalidations committed by the first attempt +// survive and are not misread (I6 must not report the invalidated scopes +// as orphans), and the resumed output equals a cold sync — including +// through round 1's dirty→clean replay, where a lost invalidation would +// 304 the sanitized subsets. +func TestSourceCache_DirtyHaltAtInvariantSeam(t *testing.T) { + ctx, err := logging.Init(t.Context()) + require.NoError(t, err) + tmpDir := t.TempDir() + + model := newDirtyEquivModel() + warmConn := newEquivConnector(model, false) + coldConn := newEquivConnector(model, true) + var warmClient types.ConnectorClient = warmConn + var coldClient types.ConnectorClient = coldConn + + halt := &haltOnce{stage: "ingest-invariants-complete"} + hookOpt := func(s *syncer) { s.testSourceCacheHaltHook = halt.hook } + + // Round 0: the warm leg crashes at the seam, then resumes the SAME + // file to completion. + warm0 := filepath.Join(tmpDir, "warm-r0.c1z") + haltErr := runEquivSyncMode(ctx, t, warmClient, warm0, "", tmpDir, false, hookOpt) + require.Error(t, haltErr, "the armed halt must crash the dirty sync at the invariant seam") + require.Contains(t, haltErr.Error(), "injected halt at ingest-invariants-complete") + require.NoError(t, runEquivSyncMode(ctx, t, warmClient, warm0, "", tmpDir, false, hookOpt), + "resume after the invariant-seam halt must complete (drops must be idempotent over already-dropped state)") + + cold0 := filepath.Join(tmpDir, "cold-r0.c1z") + require.NoError(t, runEquivSyncMode(ctx, t, coldClient, cold0, "", tmpDir, false)) + + warmSnap := snapshotC1z(ctx, t, warm0) + coldSnap := snapshotC1z(ctx, t, cold0) + require.Equal(t, 1, warmSnap.syncRuns, "the halted sync must RESUME (one sync run), not restart") + assertDirtyArtifacts(t, 0, model, "warm", warmSnap) + assertDirtyArtifacts(t, 0, model, "cold", coldSnap) + requireSnapshotsEqual(t, 0, coldSnap, warmSnap) + invalidated := 0 + for _, entry := range warmSnap.manifest { + if entry.Invalidated { + invalidated++ + } + } + require.Positive(t, invalidated, + "round 0's drops must leave invalidated manifest entries in the resumed artifact") + + // Round 1 (dirty→clean): warm replays from the resumed artifact. A + // lost invalidation would 304 the sanitized scopes here and diverge + // from cold. + model.mutateDirty(1) + warm1 := filepath.Join(tmpDir, "warm-r1.c1z") + cold1 := filepath.Join(tmpDir, "cold-r1.c1z") + require.NoError(t, runEquivSyncMode(ctx, t, warmClient, warm1, warm0, tmpDir, false)) + require.NoError(t, runEquivSyncMode(ctx, t, coldClient, cold1, "", tmpDir, false)) + warmSnap1 := snapshotC1z(ctx, t, warm1) + coldSnap1 := snapshotC1z(ctx, t, cold1) + assertDirtyArtifacts(t, 1, model, "warm", warmSnap1) + assertDirtyArtifacts(t, 1, model, "cold", coldSnap1) + requireSnapshotsEqual(t, 1, coldSnap1, warmSnap1) +} + +func TestSourceCache_ReplayEquivalenceDirty(t *testing.T) { + scenarios := []struct { + workers int + seed int64 + continuation bool + }{ + {workers: 0, seed: 11}, + {workers: 4, seed: 12}, + {workers: 0, seed: 13, continuation: true}, + } + for _, sc := range scenarios { + name := fmt.Sprintf("workers=%d/seed=%d", sc.workers, sc.seed) + if sc.continuation { + name += "/continuation" + } + t.Run(name, func(t *testing.T) { + runDirtyReplayEquivalenceScenario(t, sc.workers, sc.seed, sc.continuation) + }) + } +} + +func runDirtyReplayEquivalenceScenario(t *testing.T, workers int, seed int64, continuation bool) { + ctx, err := logging.Init(t.Context()) + require.NoError(t, err) + ctx, invariantWarnings := withInvariantWarningOracle(ctx) + tmpDir := t.TempDir() + // The schedule is fully forced; the seed only labels the scenario + // (kept for parity with the clean family and future churn). + _ = rand.New(rand.NewSource(seed)) //nolint:gosec // deterministic test randomness + t.Logf("dirty replay-equivalence scenario: workers=%d seed=%d rounds=%d continuation=%v", workers, seed, dirtyEquivRounds, continuation) + + model := newDirtyEquivModel() + warmConn := newEquivConnector(model, false) + coldConn := newEquivConnector(model, true) + warmConn.continuation = continuation + coldConn.continuation = continuation + var warmClient types.ConnectorClient = warmConn + var coldClient types.ConnectorClient = coldConn + if continuation { + warmClient = equivContinuationClient{warmConn} + coldClient = equivContinuationClient{coldConn} + } + + // NO external c1z: unprocessed match carriers must survive to the + // invariant seam (the exempt shape) — with a file configured the + // match op deletes every carrier it processes. + syncOpts := func() []SyncOpt { + var opts []SyncOpt + if workers > 0 { + opts = append(opts, WithWorkerCount(workers)) + } + return opts + } + + // runLeg runs one default-mode sync and asserts the leg's invariant + // warnings: the round's expected drops MUST have fired (anti- + // vacuity), and nothing else may warn. + runLeg := func(round int, label string, cc types.ConnectorClient, path, prevPath string) { + t.Helper() + before := len(invariantWarnings.captured()) + require.NoError(t, runEquivSyncMode(ctx, t, cc, path, prevPath, tmpDir, false, syncOpts()...)) + assertDirtyWarnings(t, round, label, invariantWarnings.captured()[before:]) + } + + prevWarm := "" + for round := 0; round <= dirtyEquivRounds; round++ { + if round > 0 { + model.mutateDirty(round) + } + warmPath := filepath.Join(tmpDir, fmt.Sprintf("warm-r%d.c1z", round)) + coldPath := filepath.Join(tmpDir, fmt.Sprintf("cold-r%d.c1z", round)) + + before := warmConn.snapshotCounters() + if round == dirtyCrashRound { + // Crash the warm sync mid-grants on a steady-dirty scope + // (gd re-fetches every round: its manifest entry was + // invalidated by the previous round's drop), then resume the + // SAME file. + warmConn.injectGrantsFailure("gd") + crashErr := runEquivSyncMode(ctx, t, warmClient, warmPath, prevWarm, tmpDir, false, syncOpts()...) + require.Error(t, crashErr, "round %d: the injected failure must crash the warm sync", round) + require.Contains(t, crashErr.Error(), "injected crash") + require.Equal(t, 1, warmConn.snapshotCounters().injectedFailures-before.injectedFailures, + "round %d: exactly one injected failure expected", round) + } + runLeg(round, "warm", warmClient, warmPath, prevWarm) + after := warmConn.snapshotCounters() + if round > 0 { + require.Positive(t, (after.pages304+after.pagesOverlay)-(before.pages304+before.pagesOverlay), + "round %d: VACUOUS RUN — the warm sync replayed nothing, so equivalence proves nothing", round) + } + runLeg(round, "cold", coldClient, coldPath, "") + + coldSnap := snapshotC1z(ctx, t, coldPath) + warmSnap := snapshotC1z(ctx, t, warmPath) + if round == dirtyCrashRound { + require.Equal(t, 1, warmSnap.syncRuns, + "round %d: the crashed warm sync must RESUME (one sync run), not restart", round) + } + assertDirtyArtifacts(t, round, model, "cold", coldSnap) + assertDirtyArtifacts(t, round, model, "warm", warmSnap) + requireSnapshotsEqual(t, round, coldSnap, warmSnap) + + prevWarm = warmPath + } + + // Scenario-level coverage: the dirty rounds must still have + // exercised the replay machinery they claim to run drops through. + final := warmConn.snapshotCounters() + require.Positive(t, final.pages304, "coverage: no 304 replay round ever happened") + require.Positive(t, final.pagesOverlay, "coverage: no delta-overlay round ever happened (gdelta round 3)") + require.Positive(t, final.pagesFresh, "coverage: no cold-miss page ever happened") + require.Positive(t, final.chunkPages304, "coverage: no spawned chunk cursor ever replayed") + require.Positive(t, final.chunkPagesFresh, "coverage: no spawned chunk cursor ever fetched fresh") + require.Positive(t, final.injectedFailures, "coverage: the crash/resume round never crashed") + if continuation { + require.Positive(t, final.askPages, "coverage: continuation topology never asked") + } else { + require.Zero(t, final.askPages, "harness bug: direct topology must never ask") + } + coldFinal := coldConn.snapshotCounters() + require.Zero(t, coldFinal.pages304+coldFinal.pagesOverlay, + "harness bug: the cold connector must never replay") +} diff --git a/pkg/sync/source_cache_equivalence_test.go b/pkg/sync/source_cache_equivalence_test.go index 72fbf1e86..1681352ba 100644 --- a/pkg/sync/source_cache_equivalence_test.go +++ b/pkg/sync/source_cache_equivalence_test.go @@ -54,6 +54,16 @@ package sync //nolint:revive,nolintlint // we can't change the package name for // replayed state (replay idempotency, checkpoint interplay) are part of // the compared output, every scenario. // +// THE DIRTY FAMILY (source_cache_equivalence_dirty_test.go) is this +// harness's DEFAULT-MODE sibling: connectors that emit referentially +// broken data (I7/I8/I9 dangling shapes, mixed exempt-carrier +// populations), run WITHOUT fail-fast so the drop arms execute inside +// the equivalence property, with the warning-silence oracle inverted +// (expected drop warnings must be PRESENT). Every scenario in THIS file +// stays fail-fast and warning-silent; the dirty knobs on the model +// (ghostAdminGrant, ghostTeamEnt, extraTeamResources, entsVersion) are +// zero here and must stay zero. +// // EXPLICITLY NOT COVERED HERE (do not mistake green for proof of these): // - error-outcome parity (covered by the dedicated exclusion-group // parity test in source_cache_sideeffects_test.go); @@ -156,6 +166,22 @@ type equivGroup struct { // default annotation to the member entitlement, so replayed // entitlement rows carry exclusion state through validation. exclusionDefault bool + + // Dirty-connector knobs (see the dirty scenario family in + // source_cache_equivalence_dirty_test.go). All zero for the clean + // model, keeping clean-scenario behavior byte-identical. + // + // ghostAdminGrant makes the group's grants page emit a grant on the + // SHADOW host resource's "admin" entitlement (see + // equivShadowAdminGrant). While the shadow type is disabled the + // grant is an I8-class dangling reference inside a stamped scope + // (an expected configuration gap: the referent's TYPE is not + // synced); enabling the shadow type is the "referent appears" + // transition while the grants scope's validator stays put. + ghostAdminGrant bool + // entsVersion is the group's ENTITLEMENTS scope validator generation + // (the clean model never rotates it). + entsVersion int } type equivOrg struct { @@ -210,6 +236,28 @@ type equivModel struct { // are fresh-or-304 scopes, no deltas. teams map[string]map[string]bool teamChunkVersions []int + + // Dirty-connector knobs (zero for the clean model; see the dirty + // scenario family in source_cache_equivalence_dirty_test.go). + // + // ghostTeamEnt makes chunk 0's type-scoped ENTITLEMENTS page emit an + // entitlement for the SHADOW resource "shadow-host2" — and chunk 0's + // grants page a grant under it. While the shadow type is disabled + // the entitlement is an I7-class dangling reference (and the grant + // its I8 cascade) — expected configuration gaps, since the referent + // TYPE is not synced — inside stamped type-scoped scopes whose + // validators never rotate. + ghostTeamEnt bool + // shadowEnabled "enables the resource type" eq_shadow: the type is + // listed in resource types and its resources (shadowUsers + the two + // fixed hosts) are listed fresh each sync. The dirty family's + // dirty→clean transition flips this while every dangling scope's + // validator stays put. + shadowEnabled bool + // shadowUsers are the shadow-typed principals dirty groups reference + // as members (ids prefixed "shadow-"). Rows exist only while + // shadowEnabled. + shadowUsers map[string]bool } // teamChunkOf returns the chunk index a team belongs to (two teams per @@ -510,6 +558,22 @@ func (c *equivConnector) SetSourceCache(_ context.Context, lookup sourcecache.Lo c.lookup = lookup } +// ListResourceTypes includes the dirty family's shadow type only while +// the model has "enabled" it — the store-level type row is what the +// invariants' enabled/disabled attribution probes, so the type must be +// genuinely absent before the transition. Clean scenarios never enable +// it and see exactly rtDB. +func (c *equivConnector) ListResourceTypes(context.Context, *v2.ResourceTypesServiceListResourceTypesRequest, ...grpc.CallOption) (*v2.ResourceTypesServiceListResourceTypesResponse, error) { + c.mu.Lock() + enabled := c.model.shadowEnabled + c.mu.Unlock() + list := c.rtDB + if enabled { + list = append(append([]*v2.ResourceType{}, list...), equivShadowRT) + } + return v2.ResourceTypesServiceListResourceTypesResponse_builder{List: list}.Build(), nil +} + func (c *equivConnector) Validate(context.Context, *v2.ConnectorServiceValidateRequest, ...grpc.CallOption) (*v2.ConnectorServiceValidateResponse, error) { return v2.ConnectorServiceValidateResponse_builder{ Annotations: annotations.New(v2.SourceCacheCapability_builder{ @@ -704,8 +768,87 @@ func equivMemberEnt(g *v2.Resource, exclusionDefault bool) *v2.Entitlement { return ent } +// equivMemberGrant builds a group-membership grant. Principal type +// routes by id prefix for the dirty family: "shadow-" members are the +// toggleable disabled type (eq_shadow), "ghost2-" members a type that +// NEVER gets enabled (steady-dirty shapes). Clean-model member ids use +// neither prefix and stay user-typed. func equivMemberGrant(gid, uid string) *v2.Grant { + principalType := userResourceType.GetId() + switch { + case strings.HasPrefix(uid, "shadow-"): + principalType = equivShadowRT.GetId() + case strings.HasPrefix(uid, "ghost2-"): + principalType = equivGhost2TypeID + } return gt.NewGrant(equivGroupResource(gid), "member", + v2.ResourceId_builder{ResourceType: principalType, Resource: uid}.Build()) +} + +// equivGhost2TypeID is the dirty family's never-enabled principal type: +// no resource type row ever exists for it, so references stay expected +// configuration gaps (the drop arm) in every round. +const equivGhost2TypeID = "eq_ghost2" + +// --- dirty-family shadow builders ------------------------------------- +// +// equivShadowRT is the dirty family's toggleable resource type: DISABLED +// (never listed) until the model's shadowEnabled flips. References into +// it are expected configuration gaps — the replay policy's DROP arm — +// and enabling it is the dirty→clean "referent appears" transition. +// Never used by clean scenarios. +var equivShadowRT = v2.ResourceType_builder{Id: "eq_shadow", DisplayName: "Shadow"}.Build() + +const ( + // equivShadowHostID hosts the "admin" entitlement ghostAdminGrant + // references (the I8 dirty shape). + equivShadowHostID = "shadow-host" + // equivShadowHost2ID hosts the entitlement the ghost team-chunk + // emission references (the I7 dirty shape + I8 cascade). + equivShadowHost2ID = "shadow-host2" +) + +func equivShadowResource(id string) *v2.Resource { + r, err := rs.NewResource("Shadow "+id, equivShadowRT, id, rs.WithAnnotation(&v2.SkipEntitlementsAndGrants{})) + if err != nil { + panic(err) + } + return r +} + +// equivShadowHostResource is a shadow host WITHOUT the skip annotation: +// hosts serve entitlement pages once the type is enabled. +func equivShadowHostResource(id string) *v2.Resource { + r, err := rs.NewResource("Shadow "+id, equivShadowRT, id) + if err != nil { + panic(err) + } + return r +} + +// equivShadowAdminEnt / equivShadowAdminGrant: the I8 dirty shape — the +// grant is emitted (inside a group's stamped grants scope) whether or +// not the shadow type is enabled; the entitlement row exists only once +// it is. +func equivShadowAdminEnt() *v2.Entitlement { + ent := et.NewAssignmentEntitlement(equivShadowHostResource(equivShadowHostID), "admin", et.WithGrantableTo(userResourceType)) + ent.SetSlug("admin") + return ent +} + +func equivShadowAdminGrant(uid string) *v2.Grant { + return gt.NewGrant(equivShadowHostResource(equivShadowHostID), "admin", + v2.ResourceId_builder{ResourceType: userResourceType.GetId(), Resource: uid}.Build()) +} + +// equivShadowHost2Ent / equivShadowHost2Grant: the I7 dirty shape and +// its I8 cascade, emitted inside the type-scoped team chunk-0 scopes. +func equivShadowHost2Ent() *v2.Entitlement { + return equivMemberEnt(equivShadowHostResource(equivShadowHost2ID), false) +} + +func equivShadowHost2Grant(uid string) *v2.Grant { + return gt.NewGrant(equivShadowHostResource(equivShadowHost2ID), "member", v2.ResourceId_builder{ResourceType: userResourceType.GetId(), Resource: uid}.Build()) } @@ -786,6 +929,31 @@ func (c *equivConnector) listResourcesInner(ctx context.Context, lookup sourceca out = append(out, equivTeamResource(id)) } return v2.ResourcesServiceListResourcesResponse_builder{List: out}.Build(), nil + case equivShadowRT.GetId(): + // The dirty family's "enable the resource type" switch: listed + // UNSCOPED and fresh (like teams) only while enabled; before + // that, references into the type are expected configuration + // gaps that the invariants drop. + if !m.shadowEnabled { + c.mu.Unlock() + return v2.ResourcesServiceListResourcesResponse_builder{}.Build(), nil + } + ids := sortedKeys(m.shadowUsers) + ids = append(ids, equivShadowHostID, equivShadowHost2ID) + sort.Strings(ids) + c.mu.Unlock() + out := make([]*v2.Resource, 0, len(ids)) + for _, id := range ids { + if id == equivShadowHostID { + // The admin-ent host serves an entitlements page (no + // skip annotation); everything else is skip-annotated + // (host2's entitlement arrives via the team chunk). + out = append(out, equivShadowHostResource(id)) + continue + } + out = append(out, equivShadowResource(id)) + } + return v2.ResourcesServiceListResourcesResponse_builder{List: out}.Build(), nil case userResourceType.GetId(): version := m.usersVersion userIDs := sortedKeys(m.users) @@ -902,33 +1070,50 @@ func (c *equivConnector) listEntitlementsInner( } return c.teamEntitlementsPage(ctx, lookup, in.GetPageToken()) } + if rid.GetResourceType() == equivShadowRT.GetId() { + // Shadow HOST entitlements: unscoped, fresh — served only while + // the type is enabled (the dirty family's dirty→clean referent). + c.mu.Lock() + enabled := c.model.shadowEnabled + c.mu.Unlock() + if enabled && rid.GetResource() == equivShadowHostID { + return v2.EntitlementsServiceListEntitlementsResponse_builder{ + List: []*v2.Entitlement{equivShadowAdminEnt()}, + }.Build(), nil + } + return v2.EntitlementsServiceListEntitlementsResponse_builder{}.Build(), nil + } if rid.GetResourceType() != groupResourceType.GetId() { return v2.EntitlementsServiceListEntitlementsResponse_builder{}.Build(), nil } c.mu.Lock() g := c.model.groups[rid.GetResource()] + var entsVersion int + if g != nil { + entsVersion = g.entsVersion + } c.mu.Unlock() if g == nil { return v2.EntitlementsServiceListEntitlementsResponse_builder{}.Build(), nil } scope := "ents:" + g.id - // Entitlement scopes never rotate after creation: warm rounds are - // pure 304s, which is exactly what keeps exclusion-group state on - // the replayed path. - verdict, err := c.revalidate(ctx, lookup, sourcecache.RowKindEntitlements, sourcecache.HashScope(scope), 0) + // Entitlement scopes never rotate after creation (entsVersion stays + // 0): warm rounds are pure 304s, which is exactly what keeps + // exclusion-group state on the replayed path. + verdict, err := c.revalidate(ctx, lookup, sourcecache.RowKindEntitlements, sourcecache.HashScope(scope), entsVersion) if err != nil { return nil, err } if verdict == equivCurrent { c.count(func(ct *equivCounters) { ct.pages304++ }) return v2.EntitlementsServiceListEntitlementsResponse_builder{ - Annotations: equivReplayAnnos(scope, 0), + Annotations: equivReplayAnnos(scope, entsVersion), }.Build(), nil } c.count(func(ct *equivCounters) { ct.pagesFresh++ }) return v2.EntitlementsServiceListEntitlementsResponse_builder{ List: []*v2.Entitlement{equivMemberEnt(equivGroupResource(g.id), g.exclusionDefault)}, - Annotations: equivScopeAnnos(scope, 0), + Annotations: equivScopeAnnos(scope, entsVersion), }.Build(), nil } @@ -1000,6 +1185,7 @@ func (c *equivConnector) teamEntitlementsPage(ctx context.Context, lookup source } c.mu.Lock() teamIDs := c.model.teamsInChunk(chunk) + ghost := c.model.ghostTeamEnt && chunk == 0 c.mu.Unlock() scope := fmt.Sprintf("ents:team-chunk:%d", chunk) @@ -1018,6 +1204,12 @@ func (c *equivConnector) teamEntitlementsPage(ctx context.Context, lookup source for _, tid := range teamIDs { rows = append(rows, equivMemberEnt(equivTeamResource(tid), false)) } + if ghost { + // The I7-class dangling shape: an entitlement for the shadow + // host2 resource, which is only listed once the shadow type is + // enabled. The scope's validator never rotates. + rows = append(rows, equivShadowHost2Ent()) + } return v2.EntitlementsServiceListEntitlementsResponse_builder{ List: rows, Annotations: equivScopeAnnos(scope, 0), @@ -1050,6 +1242,7 @@ func (c *equivConnector) teamGrantsPage(ctx context.Context, lookup sourcecache. m := c.model version := m.teamChunkVersions[chunk] teamIDs := m.teamsInChunk(chunk) + ghost := m.ghostTeamEnt && chunk == 0 membersByTeam := make(map[string][]string, len(teamIDs)) for _, tid := range teamIDs { membersByTeam[tid] = sortedKeys(m.teams[tid]) @@ -1074,6 +1267,11 @@ func (c *equivConnector) teamGrantsPage(ctx context.Context, lookup sourcecache. rows = append(rows, equivTeamGrant(tid, uid)) } } + if ghost { + // The shadow host2 entitlement's grant: dropped as I7's I8 + // cascade while the shadow type is disabled, kept once enabled. + rows = append(rows, equivShadowHost2Grant("u0")) + } return v2.GrantsServiceListGrantsResponse_builder{ List: rows, Annotations: equivScopeAnnos(scope, version), @@ -1100,6 +1298,7 @@ func (c *equivConnector) groupGrantsPage(ctx context.Context, lookup sourcecache isMultiPage := gid == m.multiPageGroup expandableChild := m.expandableChild version := g.version + ghostAdmin := g.ghostAdminGrant members := sortedKeys(g.members) deltaAdds := append([]string{}, g.deltaAdds...) deltaRemoves := append([]string{}, g.deltaRemoves...) @@ -1130,6 +1329,12 @@ func (c *equivConnector) groupGrantsPage(ctx context.Context, lookup sourcecache for _, uid := range members { out = append(out, equivMemberGrant(gid, uid)) } + if ghostAdmin { + // Emitted every fresh fetch; an I8-class dangling (expected + // disabled-type gap) until the shadow type is enabled and + // its host's admin entitlement row exists. + out = append(out, equivShadowAdminGrant("u1")) + } if isExpandableParent { childRes := equivGroupResource(expandableChild) childEnt := equivMemberEnt(childRes, false) @@ -1325,10 +1530,11 @@ type c1zSnapshot struct { // grantsByEntResource: entitlement-resource "rt/id" -> grant ids via // the by_entitlement_resource index read path. grantsByEntResource map[string][]string - // manifest ("row_kind\x00scope_key" -> etag) and scopeIndexCounts - // (row kind -> scope key -> index entries): the source-cache state - // invisible to v2 reads but load-bearing for the NEXT sync's replay. - manifest map[string]string + // manifest ("row_kind\x00scope_key" -> typed entry: validator + + // invalidation verdict) and scopeIndexCounts (row kind -> scope key + // -> index entries): the source-cache state invisible to v2 reads + // but load-bearing for the NEXT sync's replay. + manifest map[string]pebbleengine.SourceCacheManifestEntry scopeIndexCounts map[string]map[string]int // syncRuns is the number of sync runs in the file — the // crash/resume round asserts exactly one (resume, not restart). @@ -1786,16 +1992,27 @@ const equivCrashRound = 2 // unfinished sync), returning Sync's error. Path-based store creation — // not WithConnectorStore — so a crashed sync's file can be reopened and // resumed exactly like production resume. +// +// Clean scenarios run fail-fast: check-only ingestion invariants +// hard-fail, so a violation names itself before the differential +// comparison would catch its downstream divergence. The dirty family +// (source_cache_equivalence_dirty_test.go) uses runEquivSyncMode with +// failFast=false — its PROPERTY is default-warm ≡ default-cold, which +// requires the drop arms to actually execute. func runEquivSync(ctx context.Context, t *testing.T, cc types.ConnectorClient, path, prevPath, tmpDir string, extraOpts ...SyncOpt) error { + t.Helper() + return runEquivSyncMode(ctx, t, cc, path, prevPath, tmpDir, true, extraOpts...) +} + +func runEquivSyncMode(ctx context.Context, t *testing.T, cc types.ConnectorClient, path, prevPath, tmpDir string, failFast bool, extraOpts ...SyncOpt) error { t.Helper() opts := []SyncOpt{ WithC1ZPath(path), WithStorageEngine(c1zstore.EnginePebble), WithTmpDir(tmpDir), - // Check-only ingestion invariants hard-fail in the harness: a - // violation must name itself here before the differential - // comparison would catch its downstream divergence. - WithFailFastInvariants(), + } + if failFast { + opts = append(opts, WithFailFastInvariants()) } if prevPath != "" { opts = append(opts, WithPreviousSyncC1ZPath(prevPath)) diff --git a/pkg/sync/source_cache_sideeffects_test.go b/pkg/sync/source_cache_sideeffects_test.go index 598f86c18..5e9f365fd 100644 --- a/pkg/sync/source_cache_sideeffects_test.go +++ b/pkg/sync/source_cache_sideeffects_test.go @@ -672,6 +672,241 @@ func TestSourceCache_PartialPreviousSyncDegradesToCold(t *testing.T) { "sanity: the connector must have looked up and missed (no-op lookup installed)") } +// --------------------------------------------------------------------- +// Replay-compatibility key: exact match or cold. +// --------------------------------------------------------------------- + +// TestSourceCache_CompatKeyGate pins the replay-compatibility key: replay +// is permitted ONLY when the previous artifact's recorded key matches the +// current run's key byte-exactly on every component. Each subtest changes +// exactly one component between two otherwise-identical syncs and +// requires zero lookup hits; the control run (identical key) must hit. +// Compatibility is never inferred from versions — the gate compares +// explicit declarations. +func TestSourceCache_CompatKeyGate(t *testing.T) { + newFixtureConnector := func(ctx context.Context, t *testing.T) *sourceCacheMockConnector { + t.Helper() + group, ent, alice, _ := sourceCacheTestFixtures(t) + mc := newSourceCacheMockConnector() + mc.AddResource(ctx, group) + mc.AddResource(ctx, alice) + mc.entDB[group.GetId().GetResource()] = []*v2.Entitlement{ent} + mc.grantDB[group.GetId().GetResource()] = []*v2.Grant{gt.NewGrant(group, "member", alice)} + mc.etagByResource[group.GetId().GetResource()] = `W/"etag-v1"` + return mc + } + + t.Run("control: identical key replays", func(t *testing.T) { + ctx, err := logging.Init(t.Context()) + require.NoError(t, err) + tmpDir := t.TempDir() + mc := newFixtureConnector(ctx, t) + mc.cacheGeneration = "gen-1" + mc.configFingerprint = "cfg-abc" + + sync1 := filepath.Join(tmpDir, "sync1.c1z") + runSourceCacheSync(ctx, t, mc, sync1, "", tmpDir) + runSourceCacheSync(ctx, t, mc, filepath.Join(tmpDir, "sync2.c1z"), sync1, tmpDir) + require.Positive(t, mc.lookupHits, "an exactly-matching compatibility key must permit replay") + }) + + t.Run("connector cache-generation change degrades to cold", func(t *testing.T) { + ctx, err := logging.Init(t.Context()) + require.NoError(t, err) + tmpDir := t.TempDir() + mc := newFixtureConnector(ctx, t) + mc.cacheGeneration = "gen-1" + + sync1 := filepath.Join(tmpDir, "sync1.c1z") + runSourceCacheSync(ctx, t, mc, sync1, "", tmpDir) + mc.cacheGeneration = "gen-2" // connector changed its cache semantics + missesBefore := mc.lookupMisses + runSourceCacheSync(ctx, t, mc, filepath.Join(tmpDir, "sync2.c1z"), sync1, tmpDir) + require.Zero(t, mc.lookupHits, "a cache-generation bump must refuse the prior artifact") + require.Greater(t, mc.lookupMisses, missesBefore, "sanity: lookups must have missed, not been skipped") + }) + + t.Run("connector config fingerprint change degrades to cold", func(t *testing.T) { + ctx, err := logging.Init(t.Context()) + require.NoError(t, err) + tmpDir := t.TempDir() + mc := newFixtureConnector(ctx, t) + mc.configFingerprint = "cfg-abc" + + sync1 := filepath.Join(tmpDir, "sync1.c1z") + runSourceCacheSync(ctx, t, mc, sync1, "", tmpDir) + mc.configFingerprint = "cfg-def" // permissions/config changed what upstream is visible + missesBefore := mc.lookupMisses + runSourceCacheSync(ctx, t, mc, filepath.Join(tmpDir, "sync2.c1z"), sync1, tmpDir) + require.Zero(t, mc.lookupHits, "a config-fingerprint change must refuse the prior artifact") + require.Greater(t, mc.lookupMisses, missesBefore) + }) + + t.Run("sync-selection change degrades to cold", func(t *testing.T) { + ctx, err := logging.Init(t.Context()) + require.NoError(t, err) + tmpDir := t.TempDir() + mc := newFixtureConnector(ctx, t) + + sync1 := filepath.Join(tmpDir, "sync1.c1z") + runSourceCacheSync(ctx, t, mc, sync1, "", tmpDir) + // Same connector, same config — but the SDK-side selection now + // names the types explicitly, changing the selection fingerprint. + missesBefore := mc.lookupMisses + runSourceCacheSync(ctx, t, mc, filepath.Join(tmpDir, "sync2.c1z"), sync1, tmpDir, + WithSyncResourceTypes([]string{groupResourceType.GetId(), userResourceType.GetId()})) + require.Zero(t, mc.lookupHits, "a sync-selection change must refuse the prior artifact") + require.Greater(t, mc.lookupMisses, missesBefore) + }) + + t.Run("pre-key artifact degrades to cold", func(t *testing.T) { + ctx, err := logging.Init(t.Context()) + require.NoError(t, err) + tmpDir := t.TempDir() + + // Hand-build a FULL previous sync with a valid-looking manifest + // over stamped rows but NO compatibility record — the shape every + // artifact written by a pre-key SDK has. + group, ent, alice, _ := sourceCacheTestFixtures(t) + g1 := gt.NewGrant(group, "member", alice) + grantsScope := grantScopeForResource(group.GetId().GetResource()) + sync1 := filepath.Join(tmpDir, "prekey-prev.c1z") + prevStore, err := dotc1z.NewStore(ctx, sync1, + dotc1z.WithEngine(c1zstore.EnginePebble), + dotc1z.WithTmpDir(tmpDir), + ) + require.NoError(t, err) + _, isNew, err := prevStore.StartOrResumeSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + require.True(t, isNew) + require.NoError(t, prevStore.PutResourceTypes(ctx, groupResourceType, userResourceType)) + require.NoError(t, prevStore.PutResources(ctx, group, alice)) + require.NoError(t, prevStore.PutEntitlements(ctx, ent)) + require.NoError(t, prevStore.PutGrants(sourcecache.WithScope(ctx, grantsScope), g1)) + scPrev, ok := any(prevStore).(dotc1z.SourceCacheStore) + require.True(t, ok) + require.NoError(t, scPrev.PutSourceCacheEntry(ctx, sourcecache.RowKindGrants, grantsScope, `W/"etag-v1"`)) + _, found, err := scPrev.GetSourceCacheCompat(ctx) + require.NoError(t, err) + require.False(t, found, "fixture: the hand-built artifact must carry NO compat record, or this test proves nothing") + require.NoError(t, prevStore.EndSync(ctx)) + require.NoError(t, prevStore.Close(ctx)) + + mc := newFixtureConnector(ctx, t) + missesBefore := mc.lookupMisses + runSourceCacheSync(ctx, t, mc, filepath.Join(tmpDir, "sync2.c1z"), sync1, tmpDir) + require.Zero(t, mc.lookupHits, "an artifact without a compatibility key must never serve replay") + require.Greater(t, mc.lookupMisses, missesBefore) + }) +} + +// --------------------------------------------------------------------- +// SQLite artifacts must not replay (engine non-goal). +// --------------------------------------------------------------------- + +// TestSourceCache_SQLitePreviousSyncDegradesToCold pins the READ-side +// engine gate: SQLite replay is an explicit non-goal, so a SQLite +// previous-sync c1z — even a complete, finished full sync — must degrade +// to a cold sync (every lookup misses; the file opens fine but is not a +// dotc1z.SourceCacheStore). The current sync's OUTPUT store is Pebble, +// so the write side stays enabled: this isolates exactly the +// "previous-sync store engine does not support source cache" arm of +// configureSourceCache. +func TestSourceCache_SQLitePreviousSyncDegradesToCold(t *testing.T) { + ctx, err := logging.Init(t.Context()) + require.NoError(t, err) + tmpDir := t.TempDir() + + group, ent, alice, _ := sourceCacheTestFixtures(t) + g1 := gt.NewGrant(group, "member", alice) + + // A complete SQLITE previous sync (default engine). + sync1 := filepath.Join(tmpDir, "sqlite-prev.c1z") + prevStore, err := dotc1z.NewStore(ctx, sync1, + dotc1z.WithEngine(c1zstore.EngineSQLite), + dotc1z.WithTmpDir(tmpDir), + ) + require.NoError(t, err) + _, isNew, err := prevStore.StartOrResumeSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + require.True(t, isNew) + require.NoError(t, prevStore.PutResourceTypes(ctx, groupResourceType, userResourceType)) + require.NoError(t, prevStore.PutResources(ctx, group, alice)) + require.NoError(t, prevStore.PutEntitlements(ctx, ent)) + require.NoError(t, prevStore.PutGrants(ctx, g1)) + _, isSourceCacheStore := any(prevStore).(dotc1z.SourceCacheStore) + require.False(t, isSourceCacheStore, + "fixture: the SQLite store must NOT implement SourceCacheStore, or this test proves nothing") + require.NoError(t, prevStore.EndSync(ctx)) + require.NoError(t, prevStore.Close(ctx)) + + mc := newSourceCacheMockConnector() + mc.AddResource(ctx, group) + mc.AddResource(ctx, alice) + mc.entDB[group.GetId().GetResource()] = []*v2.Entitlement{ent} + mc.grantDB[group.GetId().GetResource()] = []*v2.Grant{g1} + mc.etagByResource[group.GetId().GetResource()] = `W/"etag-v1"` + + sync2 := filepath.Join(tmpDir, "sync2.c1z") + missesBefore := mc.lookupMisses + runSourceCacheSync(ctx, t, mc, sync2, sync1, tmpDir) + require.Zero(t, mc.lookupHits, + "a SQLite previous sync must never produce a source-cache lookup hit") + require.Greater(t, mc.lookupMisses, missesBefore, + "sanity: the connector must have looked up and missed (no-op lookup installed)") +} + +// TestSourceCache_SQLiteOutputStoreDegradesToCold pins the WRITE-side +// engine gate: when the CURRENT sync's output store is SQLite, source +// cache disables entirely — the connector's scope annotations are +// ignored (not an error), replay would be a hard error, and the sync +// completes cold. This is the exact shape of the default-engine CLI +// footgun (--previous-sync-c1z without --storage-engine pebble), which +// must degrade loudly-but-safely rather than fail or silently corrupt. +func TestSourceCache_SQLiteOutputStoreDegradesToCold(t *testing.T) { + ctx, err := logging.Init(t.Context()) + require.NoError(t, err) + tmpDir := t.TempDir() + + group, ent, alice, _ := sourceCacheTestFixtures(t) + g1 := gt.NewGrant(group, "member", alice) + + // A valid PEBBLE previous sync: eligible on its own, but the write + // side's engine gate must disable the whole feature first. + mc := newSourceCacheMockConnector() + mc.AddResource(ctx, group) + mc.AddResource(ctx, alice) + mc.entDB[group.GetId().GetResource()] = []*v2.Entitlement{ent} + mc.grantDB[group.GetId().GetResource()] = []*v2.Grant{g1} + mc.etagByResource[group.GetId().GetResource()] = `W/"etag-v1"` + sync1 := filepath.Join(tmpDir, "pebble-prev.c1z") + runSourceCacheSync(ctx, t, mc, sync1, "", tmpDir) + + // Warm attempt into a SQLITE output store: the connector still emits + // SourceCacheRecord annotations on its fresh pages (it declared the + // capability and fetched fresh after every miss) — they must be + // ignored without error. + sqliteOut, err := dotc1z.NewStore(ctx, filepath.Join(tmpDir, "sqlite-out.c1z"), + dotc1z.WithEngine(c1zstore.EngineSQLite), + dotc1z.WithTmpDir(tmpDir), + ) + require.NoError(t, err) + missesBefore := mc.lookupMisses + syncer, err := NewSyncer(ctx, mc, + WithConnectorStore(sqliteOut), + WithTmpDir(tmpDir), + WithPreviousSyncC1ZPath(sync1), + ) + require.NoError(t, err) + require.NoError(t, syncer.Sync(ctx), + "a SQLite output store must degrade source cache to cold, never fail the sync") + require.NoError(t, syncer.Close(ctx)) + require.Zero(t, mc.lookupHits, + "with source cache disabled the connector must never see a lookup hit") + require.Greater(t, mc.lookupMisses, missesBefore, + "sanity: the connector must have looked up and missed (no-op lookup installed)") +} + // --------------------------------------------------------------------- // Compacted previous syncs must not replay. // --------------------------------------------------------------------- diff --git a/pkg/sync/source_cache_skew_fixture_test.go b/pkg/sync/source_cache_skew_fixture_test.go new file mode 100644 index 000000000..3fc32c9e6 --- /dev/null +++ b/pkg/sync/source_cache_skew_fixture_test.go @@ -0,0 +1,83 @@ +package sync //nolint:revive,nolintlint // we can't change the package name for backwards compatibility + +// Frozen-artifact version-skew test. Unlike the synthesized +// mixed-version test (source_cache_mixed_version_test.go), this one +// reads REAL bytes produced by the merge-base SDK's fold compactor — +// see testdata/README-old-fold.md — so it also verifies the parts of +// old-binary behavior we would otherwise have to guess (measured: the +// old fold carries the base's source-cache manifest through INTACT, +// validators and all). + +import ( + "io" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + "github.com/conductorone/baton-sdk/pkg/dotc1z" + "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" + "github.com/conductorone/baton-sdk/pkg/logging" + gt "github.com/conductorone/baton-sdk/pkg/types/grant" +) + +func TestSourceCache_FrozenOldFoldArtifactRefusedForReplay(t *testing.T) { + ctx, err := logging.Init(t.Context()) + require.NoError(t, err) + tmpDir := t.TempDir() + + // Copy the frozen artifact out of testdata (syncs open the previous + // file next to their own tmp state). + prev := filepath.Join(tmpDir, "old-fold.c1z") + src, err := os.Open(filepath.Join("testdata", "old-fold-mixed-version.c1z")) + require.NoError(t, err) + dst, err := os.Create(prev) + require.NoError(t, err) + _, err = io.Copy(dst, src) + require.NoError(t, err) + require.NoError(t, src.Close()) + require.NoError(t, dst.Close()) + + // Sanity: this fixture is the dangerous shape — every record-level + // gate passes and the manifest is intact, so ONLY the token + // provenance can refuse it. If any of these fail, the fixture no + // longer proves what it claims (see testdata/README-old-fold.md). + store, err := dotc1z.NewStore(ctx, prev, + dotc1z.WithEngine(c1zstore.EnginePebble), + dotc1z.WithReadOnly(true), + dotc1z.WithTmpDir(tmpDir), + ) + require.NoError(t, err) + run, err := store.SyncMeta().LatestFullSync(ctx) + require.NoError(t, err) + require.NotNil(t, run) + require.False(t, run.Compacted, "fixture: old binary cannot set the compacted flag") + require.True(t, run.UsableAsReplaySource(), "fixture: the record-level gate must PASS, or this test proves nothing") + comp, err := CompactionStatsFromToken(run.SyncToken) + require.NoError(t, err) + require.NotNil(t, comp, "fixture: the token must carry compaction provenance — the one honest signal") + inspector, ok := store.(dotc1z.SourceCacheInspector) + require.True(t, ok) + manifest, err := inspector.SourceCacheManifestSnapshot(ctx) + require.NoError(t, err) + require.NotEmpty(t, manifest, "fixture: the old fold carries the base manifest intact — the manifest belt must PASS too") + require.NoError(t, store.Close(ctx)) + + // The trap: a connector whose current validator MATCHES the carried + // manifest entry. A broken gate would 304 and replay merged rows. + group, ent, alice, _ := sourceCacheTestFixtures(t) + mc := newSourceCacheMockConnector() + mc.AddResource(ctx, group) + mc.AddResource(ctx, alice) + mc.entDB[group.GetId().GetResource()] = []*v2.Entitlement{ent} + mc.grantDB[group.GetId().GetResource()] = []*v2.Grant{gt.NewGrant(group, "member", alice)} + mc.etagByResource[group.GetId().GetResource()] = `W/"skew-v1"` // matches the carried validator + + out := filepath.Join(tmpDir, "out.c1z") + runSourceCacheSync(ctx, t, mc, out, prev, tmpDir) + require.Zero(t, mc.lookupHits, + "an old-binary fold artifact must never serve replay: its rows are a merge no upstream validator describes") + require.Positive(t, mc.lookupMisses, "sanity: lookups must have been offered and missed") +} diff --git a/pkg/sync/source_cache_test.go b/pkg/sync/source_cache_test.go index 4df760679..6139b8bbe 100644 --- a/pkg/sync/source_cache_test.go +++ b/pkg/sync/source_cache_test.go @@ -52,6 +52,12 @@ type sourceCacheMockConnector struct { lookupHits int lookupMisses int + + // Replay-compatibility key components declared on Validate + // (SourceCacheCapability.cache_generation / config_fingerprint). + // Empty by default — a constant, matching generation. + cacheGeneration string + configFingerprint string } func newSourceCacheMockConnector() *sourceCacheMockConnector { @@ -76,9 +82,13 @@ func (mc *sourceCacheMockConnector) SetSourceCache(_ context.Context, lookup sou } func (mc *sourceCacheMockConnector) Validate(context.Context, *v2.ConnectorServiceValidateRequest, ...grpc.CallOption) (*v2.ConnectorServiceValidateResponse, error) { + mc.mu.Lock() + defer mc.mu.Unlock() return v2.ConnectorServiceValidateResponse_builder{ Annotations: annotations.New(v2.SourceCacheCapability_builder{ - Mode: v2.SourceCacheCapability_MODE_READ_WRITE, + Mode: v2.SourceCacheCapability_MODE_READ_WRITE, + CacheGeneration: mc.cacheGeneration, + ConfigFingerprint: mc.configFingerprint, }.Build()), }.Build(), nil } diff --git a/pkg/sync/syncer.go b/pkg/sync/syncer.go index b455aa7d9..4eb84ef4f 100644 --- a/pkg/sync/syncer.go +++ b/pkg/sync/syncer.go @@ -152,6 +152,10 @@ type syncer struct { childSchedule childScheduleSet resourcesPhaseRanHere bool failFastInvariants bool + // expandDropStats aggregates expansion edges dropped over missing + // entitlements across the whole sync (see expand.DroppedEdgeStats); + // summarized once when expansion completes. + expandDropStats *expand.DroppedEdgeStats // testSourceCacheHaltHook, when non-nil, fires at named seams of the // scope lifecycle (replay-copied, rows-committed, tombstones-applied, // manifest-written); returning an error fails the sync at exactly @@ -784,6 +788,16 @@ func (s *syncer) Sync(ctx context.Context) error { l.Debug("resuming previous sync", zap.String("sync_id", syncID)) } + if s.sourceCache.enabled { + // Persist the replay-compatibility key before any page can write + // a manifest entry (idempotent overwrite on resume): an artifact + // must never hold validators without the key that scopes their + // validity. See configureSourceCache. + if err := s.sourceCache.current.PutSourceCacheCompat(ctx, s.sourceCache.compatKey); err != nil { + return s.returnSyncError(l, span, fmt.Errorf("error recording source-cache replay-compatibility key: %w", err)) + } + } + currentStep, err := s.store.CurrentSyncStep(ctx) if err != nil { return err @@ -852,6 +866,16 @@ func (s *syncer) Sync(ctx context.Context) error { if err := s.runIngestionInvariants(ctx); err != nil { return s.returnSyncError(l, span, err) } + if s.testSourceCacheHaltHook != nil { + // The seam AFTER the invariants' drops/repairs committed and + // BEFORE the checkpoint/EndSync below: a resumed sync re-runs the + // whole invariant pass over already-dropped state, so drops must + // be idempotent and their manifest invalidations must already be + // durable (see stageSourceCacheScopeInvalidationLocked). + if err := s.testSourceCacheHaltHook("ingest-invariants-complete", ""); err != nil { + return s.returnSyncError(l, span, err) + } + } // Force a checkpoint to clear completed actions & entitlement graph in sync_token. s.state.ClearEntitlementGraph(ctx) @@ -1693,8 +1717,16 @@ func (s *syncer) syncEntitlementsForResource(ctx context.Context, action *Action if typeScoped { // Cursors don't map 1:1 to resources, so the per-resource // "N of M resources covered" accounting is meaningless here. + // Row progress counts response rows PLUS replayed rows: a warm + // 304 page carries zero response rows while landing the scope's + // whole replayed row set, and counting only the former reported + // near-zero progress for healthy warm syncs. + rows := len(resp.GetList()) + if scPage != nil { + rows += int(scPage.replayRows) + } s.counts.SetEntitlementsCountOnly(resourceID.GetResourceType()) - s.counts.AddEntitlementsProgress(resourceID.GetResourceType(), len(resp.GetList())) + s.counts.AddEntitlementsProgress(resourceID.GetResourceType(), rows) s.counts.LogEntitlementsProgress(ctx, resourceID.GetResourceType()) } else if resp.GetNextPageToken() == "" && !action.Spawned { // A resource counts as covered exactly once: when its ORIGIN @@ -2054,6 +2086,13 @@ func (s *syncer) loadEntitlementGraph(ctx context.Context, action *Action, graph // Only skip not-found entitlements; propagate other errors // to avoid silently dropping edges and yielding incorrect expansions. if status.Code(err) == codes.NotFound { + // Counted on the same sync-wide aggregate as the + // expander's drops (this seam was previously + // Debug-only — invisible in production). + if s.expandDropStats == nil { + s.expandDropStats = &expand.DroppedEdgeStats{} + } + s.expandDropStats.RecordSourceMissing(srcEntitlementID) l.Debug("source entitlement not found, skipping edge", zap.String("src_entitlement_id", srcEntitlementID), zap.String("dst_entitlement_id", dstEntitlementID), @@ -2379,9 +2418,16 @@ func (s *syncer) syncGrantsForResource(ctx context.Context, action *Action) erro // groups and may also emit cross-type grants), so the per-resource // "N of M resources covered" accounting is meaningless here and // would trip the "more grant resources than resources" warning on - // healthy syncs. Count raw grant rows instead. + // healthy syncs. Count raw grant rows instead — response rows PLUS + // replayed rows: a warm 304 page carries zero response rows while + // landing the scope's whole replayed row set, and counting only + // the former reported near-zero progress for healthy warm syncs. + rows := len(grants) + if scPage != nil { + rows += int(scPage.replayRows) + } s.counts.SetGrantsCountOnly(resourceID.GetResourceType()) - s.counts.AddGrantsProgress(resourceID.GetResourceType(), len(grants)) + s.counts.AddGrantsProgress(resourceID.GetResourceType(), rows) s.counts.LogGrantsProgress(ctx, resourceID.GetResourceType()) } else if resp.GetNextPageToken() == "" && !action.Spawned { // A resource counts as covered exactly once: when its ORIGIN @@ -3201,7 +3247,11 @@ func (s *syncer) expandGrantsForEntitlements(ctx context.Context, action *Action // The expander needs Reader methods (on s.store) plus StoreExpandedGrants // (on s.store.Grants()). An inline adapter composes them so expand // stays decoupled from C1ZStore. + if s.expandDropStats == nil { + s.expandDropStats = &expand.DroppedEdgeStats{} + } expander := expand.NewExpander(expanderStoreAdapter{s.store}, graph) + expander.SetDropStats(s.expandDropStats) err = expander.RunSingleStep(ctx) if err != nil { l.Error("expandGrantsForEntitlements: error during expansion", zap.Error(err)) @@ -3215,6 +3265,9 @@ func (s *syncer) expandGrantsForEntitlements(ctx context.Context, action *Action if expander.IsDone(ctx) { l.Debug("expandGrantsForEntitlements: graph is expanded") + // The sync's one aggregated dropped-edge report (totals + + // distinct-id examples) — per-edge drops log at Debug only. + s.expandDropStats.LogSummary(ctx) s.state.FinishAction(ctx, action) } @@ -3437,8 +3490,10 @@ func WithExternalResourceC1ZPath(path string) SyncOpt { // multi-sync behavior), so existing callers are unaffected. // // The file is opened read-only and engine-agnostically (the magic byte -// selects SQLite or Pebble), so the previous-sync c1z may use either -// engine. +// selects SQLite or Pebble), but source-cache replay is served ONLY by +// Pebble artifacts: a SQLite previous sync opens fine and then degrades +// to a cold sync (every lookup misses), with a warning naming the +// reason. SQLite replay is an explicit non-goal. func WithPreviousSyncC1ZPath(path string) SyncOpt { return func(s *syncer) { s.previousSyncC1ZPath = path @@ -3637,8 +3692,11 @@ func NewSyncer(ctx context.Context, c types.ConnectorClient, opts ...SyncOpt) (S if s.previousSyncC1ZPath != "" { // Open the previous-sync c1z read-only and engine-agnostically - // (NewStore selects the engine from the file's magic byte), so a - // Pebble or SQLite prior run both work as a replay source. + // (NewStore selects the engine from the file's magic byte). Note + // that opening is NOT eligibility: only Pebble artifacts serve + // source-cache replay — a SQLite prior run opens fine and then + // degrades to cold in configureSourceCache (SQLite replay is an + // explicit non-goal). previousSyncStore, err := dotc1z.NewStore(ctx, s.previousSyncC1ZPath, dotc1z.WithReadOnly(true), dotc1z.WithTmpDir(s.tmpDir), diff --git a/pkg/sync/testdata/README-old-fold.md b/pkg/sync/testdata/README-old-fold.md new file mode 100644 index 000000000..c84162cdf --- /dev/null +++ b/pkg/sync/testdata/README-old-fold.md @@ -0,0 +1,33 @@ +# old-fold-mixed-version.c1z + +A REAL mixed-version compaction artifact: produced by the merge-base +SDK's fold compactor (pre-source-cache, pre-compacted-flag; commit +`604dae09`) running over two source-cache-stamped inputs written by the +CURRENT SDK. Frozen here so version-skew behavior is tested against +actual old-binary output bytes, not a synthesis of what we believe old +binaries did. + +Measured properties (which make it the dangerous shape): + +- run record: `type=full`, `compacted` UNSET (the old proto has no field); +- the base input's source-cache MANIFEST survived the fold intact, + validators and all — the old engine copies keyspaces it does not know; +- the sync token carries the compaction provenance section + (`CompactionTokenStats`), which every compactor version has written. + +So every record-level replay gate passes and only the token-provenance +gate refuses it. `source_cache_skew_fixture_test.go` pins that refusal. + +Regeneration (only if the fixture must be rebuilt): + +1. `BATON_GEN_SKEW_INPUTS=/tmp/skew-gen go test -run TestGenerateSkewInputs ./pkg/sync/` +2. `git worktree add /tmp/skew-base `; copy + `cmd/genskewfold` from the worktree history (or re-create per the + test's README reference) and run + `go run ./cmd/genskewfold /tmp/skew-gen/base.c1z /tmp/skew-gen/newer.c1z /tmp/skew-gen/out` + inside the worktree. +3. Copy the output over this file. + +Future releases should extend this corpus via a freeze-at-tag step +(generate artifacts with the just-released binary) rather than in CI, +which runs before tags exist. diff --git a/pkg/sync/testdata/old-fold-mixed-version.c1z b/pkg/sync/testdata/old-fold-mixed-version.c1z new file mode 100644 index 0000000000000000000000000000000000000000..07ced5062708dc4bcbd2218cf40903740ed208e7 GIT binary patch literal 25762 zcmd6P2|!cV((t`mAh}=&7m$dk5EVrsKoCS+0|bq-c>xuxO+#|QNJuaVD%P!asjX|R zTD5B3TU)Eu)~$8lp4GZkt95&IYhBvfYPH(hD*v2wZ#JN6-}k@&|7Z=IW#-J8GiPSb zoH@x%8kkHFM4uIue-&F=YGIS*iYPI2aV|-HM^a77ZMJd?n^0x9Ic=rYWeEmrjS3g& zAg+lkL6b6-5^Ah9bQ4PAtf^x4)pm=5j3V1g8GlqRwp-{>N<7?Psb&=ds1IbgLdI|^ z#AvHj)j^jb0b~Ve*aC839N$=Ovly)9Dx1CBLlHvmNT{&dMpzMbRi(=X!kz^g9|)TF z#I&n05+G33pS4)}pngRNq|^`3BblltsTLf5FQn#5*5Nc%R{h;%J5$mkS07qI`8!yn z&1!Ps84DR#5zW|Yu-Y7W#$tvmWK1F(p0do1(lv4Qb57jpqNS?zcar?49QZ*YA9L*@dYO8q| zFHn+nahbv5U>UxWj-;d@SXDNM*~uz|cmPiYTC^&I-Dx&h6e3Pc`C@t@C3Yg}=@xZ` zQkBa(9Ka%uCWm6yE-`=xs3TN998!5rp5bt%_oc*UG#$EGJ*3%Y)?#v@eDiNmdhvqc zFlBTYs4h1C9j*O27~ zr`u3LBUnR)ch>;iRfej)Is#T3mU8c|L0H)2T!UBVOezrc3X&i_Qd=N+h;dQ%80TR# z$Fu+*3MnpHLrGa{btQ*Ej4{+_y%u`Z#UepqJPOcI_TgqDtMJ1J+SDVVkZZ4t!v#lY z0u>{jtknT}Lg9~*w62E)OLkop9ymIc^5=NuP)P7bifjOdON}@fS7%yB$sM+8yOGr! zZ6;Q4wwBoxQjDPfmMz2`Gohajx-l{i&{d`zQ+|#btJ5%25s0+_U!v8NKWj20EvS$| zgG{ZcN1B#vXIveXfB8{Cb!OY340oo)WoXZ1trwtvPQGz4zKrKq?>P~8FdKNZ6PRFGSS^mef9tb&v0gnGQL!&a5nC3_44V`OeTxJ+kJ z6sSR48Ei_U1!N0NuRfRZQI|TH87T(L0yw&pM?C%iM!I|rf=5D&DVd!`dkk32%}k1k zP(5>I3=hUltdR=~H?NkIyF!mmD$-w4CeYO@56mnm2x_?5VJ zRF>K-{oEZHFQAA~l+2gI$ddtw$nL~O96T9ZpyiYh6fW=Gpg-vz9-av?&tmG0?2DN< zd~Vi7bfg;j@ZfoeR(>g>!WzU)51>N9QLs8$dzGC;|kVkQ)=ARzKgARY=Hq$PD2WQowlQqB~NOGS=X%hP73wF)N$a!<)@@ z0AtL!JJXd^;BeM%E;AcBqR=|X)M@pA8|0O$pBHGM)8J&&sp7nTT1J`W67S+&K&$t^koNP%Xkz%mOiBDRQ#Rw;x3rXwhcORhmL zkZmQ5yUev!!R%QRm@vMLEB6*jL~^mR0~K6vDs&K#s?E&NFq$G=ex975Wt7+KQYyDcvOLh@95DMoCEN5@7S#*kbhQ% z4u{G7Rt7IV7^sslGOn82o(79@vw#vIW}u}nDk;$Fv+|2Ga_t<0pvSPVhjWjUG*ZH_*_0Bssg4)D%h$7l<*nj&o$ z@XM>NZ?TSnI)Tzig*Iel8{9%(vVgQhfV#dcU|ihNdb~m9Y9STUfWO<|Al*G72&AcF z>+L3l2upTdzkrKYc)P5eeXH-XA{2FrqzBh?2soc9wLuE%46Z^beT*XG_5=RLF+h&g z(;Wk(3S{bk=ooaSg6q-p+y;6ANh$u0zYVUIN2?-<@MzV}9<{+OLj75|$Emv#%1tEI zxZ%@HgEVxb{3==J5SzTU?G*$tumNJ<5uhyg!1 z9ZdxeH4HcC`B(rSX$nAf1$=lQkqX6KV2-R^5000^VMReHWOt+tjoR{2Q>LqhPN70D zL?u{2bCreFqhN>g}))b6-s+ziBt#O#h&Vq9czGJv|Zxk)!8)y|su~E-jQ6Nxnu)`k?MGM?n zv{h8XNs*eP;= z(6WlwTvx6h`9`=J3AST|xyEk}p%r#MZuCF_o%ZU|8n=AJb9ACQmxTPs=b8W7 z=}nnV1$xUpEu}cg2M-4vx6J%3t-eq_aq*|-XnVoRAgB2&jdoyFY3FfJg*15fv&{2|~~&Vs~-fHN&q*N@%#aFjz;gh3E*;og%2r~6ZK zK9&p{6(6V_-~^!aN8Hn?<0H&j91s)4W8upVpS=d&=+s&R=6?3+p(1fuah=hKFj@$@rXe`NwC#wv)WC|(Ic#7shoGV_@%Fp*#r zIn|^u&}lQlN-9$*KTfY8^Xb+s0K^sfs0TK#sI@pZL(2ddNMoISNQY6ujJ8M%#x_4s zU!2z`FTXzu*ZEYtY%vMzD);m*%r9mzHPZx?yzZu?!>N!W4bxLwq{mSb4{E^S-H@%r zL82B^Q}0^s0HCQ5(Zp!-AUq?ei_aCe5h7B(At8F2lO-3`t$xvDilF4Hh%F=$uWYaFqttJg*IUe)k2ROtE?Tr|t`K znf_OClXF~@XJMn%jhEEn8xn*DeL`=q9x?y9=U3ON`*@u`R=ojZ#f2pPml`Xtou)iZ zQZoLq>ylV+jw$~M$9%o?=)YT${}+C*a)FR)>;t@+2UYOv3FcuHz7*zzG*RbNd&hh!)(x1(WQh`3;&%w3H;M)am6Es_&Xo+?}L5?m{SEQKekITATzI=l#yW4O2ZzP3q-^nz>6u9uw z7ViWjq|;>$<#xGmp7G@bI`Cw~VnX~dApx{sW=?*gHVcy>rA7G#+B~>$=ZEyP~TSP?6L4K|a@yOpVv>K~FkgEKrRW{H*u%!seY5=~wfPYY01pvmk5B{wx; zE&_b8&|~4a3-MGWWM+tkp3fJ@c1xGfDC-AUg9A` zhDP&%!uWDmh{(+n?iRTH(^$d>n%vBY@nS|O9h9TZ>si!GU%+Vb4IvQ2uSpO;mj#q2o4T9;fr>ey8Z=-mt^7q4(^@N2@flrPjUhgPSZKso*K~98QQ`kq{;FbPDUwO ztjWpA@9$v*g+}EgT_2KEC<~ceDBBA*8i)miy=oxq5bfv&59OYv&~k9Fc>$~tL1pM5 z&jJK=KKcaOU|qzRLjU=sWIicfP6pF(swv6Q>N#rhTFGcBu2UzfJ1A~pe%PI1lJ7t z!AO~*gBm_>)j1}@@fT?_zy}0!8V*jd{iv`86%pi6v~XRK>p41s#{4X}{P&_0FmW1* zqY`+7wH4siQ8Hh30`@1oxF|bK-%pdH%feYKzKA%T0uuy6k*<*ol_KCCsVt;|>L>LRXA1FqY87#M^ylMut-_j99=HN!qNSHFN*T)c;vzKE`+l&?Ce}kkr!PMSI0XX z5WwlQz8U~b{VBTsZl$BBNNpaVgJsRrz5OIN{N*?OmA?on+M~dDNq0afOTzCHl+=%6bzPpaZCzk04<_Gm z730Q+0yChF%$1PB%g0-u9hl((c$^r8VWbZV&My3YlB%qd1K?3a{0?{Ve@EIp_@J7>z_efeRCw;xjOTCj%fZ9B{lPNsEjTJEvx~#J#bZ0SqVD?E3 zCQ?T3YH)W>r<%C%n&8?FE|N{20Z{(1lnLiMWby4T*g9$YCj?Y5zD-50nmd6ym!b?- zs|_**k$=Q7um0Jh9-Kq&E|D8w%HyyPOvG)C0~}MZ6L-IA4{aBo4OH07n`lzVVQ$>u2OVjd)cd56!soDBDsJSFKwO z6SO}iEYuc(j>6hdrzr&Gv{OL&yEZlGTWyxEPQWt+;@*5Er#9si{lRv*D+`$XqI|Xd zdi}~|`3--_{0ek|gYS|I`mRn@2l*|3Fm-ZQuj0JIA`Qxqo0*5yxvM%!56Ndm`ux0{ z5}1HjgW9V>067BTE{IO>)6uJO@pQks&1qiEOX@dw@@jq;nnNJ)$Fm|=GeBMoKmP5C zDS2H#2DukLZV2K!QKuGr8ZZ^G@?L#8uS71$7~AjzExG6J6hF*x^dbkwKl6!}=W^EP zbyh^rI`7jMw#>pB(Ke#KR@Qxy1$hZJyXkLUWIgTmR)oUm{Z9gaP+$Pg{w6alloK00xEwb-vT#}XlYpWb zenx0qo=O-S1qYmBmnl^Dy|41Nb^Pc(9-${gBQFYk?99r3_z! zM8yD)UUq&+o# z1iJw=PVm_vdh5Us0+B^P$22L*Y04^>;Yh*+^DQW1g7{*oNsoT@ zkM_-Q`lLr99JDv*I71K5E^DktV5D+8q+7Vy{S<-V*7_L&!7aA8K-66n&hq*!N*x{` z&!r9zO3^hwqz-xb>-&VzLw;F?>Wx3zx1*#uy#|3q)RXN9Zy81@0jOJD5#Sy>{3jy7 z=Cjy}ULRoW^|s}IBmw9pW3?S3q397Fk`Gt$|J%BuL{f*Px*R7A!Yd5)H~8Vr z%aNYPUcJ-i=9WvXqJ&7KcOu-qifk@E^Ex`c1;|5FDiCDSWfq>ke;4{u0a&L$@6tmM zF~AEuzx>VWJj?Zwsj`gzAOVGx3?#q{G@_2OkBERb$yTGe%3x8s*55Vth_D9Q%qGU0 z6ckN$_i`VGeD`{RYXtBBGLnF*cq$NWc31VY1k*D>eeKO1nx1>>!g9kiL9pmJe$A== zR(05jSEjDnXFOo=C@RRxvL+M>cVR+SK$5}jIU43xr(M~&PDNDLC@#$s*W1l=$n^wqvDbi6;YtyMAl<>M0i7k#%XM@!DrL~0 zK%7^433^ZIyUdtx913nAJ^>|NkVd9Afh`h7`Qc6U^v!V%WL@KcoI!qbgu~Q0yT{Xl zp0FChd3YZ00Gp62LJs(ma%jt%(Xe?2N~FqTt&O6 z_p#{8y8^0FAeD#dUCtee@xhdm+RRQ}8xHOn0C!VQ_Wh;u%7eW5@3)TIp*Pu$T!MUqb;7 zF1IW2`CQ1j_6XE-5`?)0^}IZdq$1n{=)KH4C*zzG)VTp6@tU=YaZ#92;`fx`WY<=7lM{3;l?K=*EgZU3QO`bU5Rn<1|$IyVW zkq({)3ecs3ryA~La5i2c7djGxnrq@|<$?6Y1(3Z4Sq&mESxxIV!-azug~bZx;{WVB zPuySi^O}hI+h_>w`z<$0xcroPEiJfakceFG9P{4|Dkyi7RG_za_7hia>fP**&!-VpBO(fprvr z^k2Wr@7}#PZZ#!~N_leR_Ybrk)Ay#&8}!H6zOw7l>F+dq@dCSvJ^DxY2`9!C-t69D zf8>%bU+it884yf}R`wZoednB?`gA|Z9{J-3+nz55C4a1)IeW~q@uYg?gOl=?B1Rr< zo_BrWf-(6shwqZF+_to38^_UO1NB4KfAK}k{+QiWWtlIw%!|2ldf|eO_lI-|Yn9nN zP|^F+(tEW@izG>r0qqs>qCuLEOF!-}qSn1V>65rv-Bry`2k5@~;>>rV^DiZ5-gu?- zmrrkw-+!+8x{)i=cU)l)O(X9e=o8+)`2CzWY4*Liq`e2qDp=dH`zm%mvVn_&+b`>*#;r_4Tos`8~~=leW(o!C+^FuUujyV4MMNbm%-P_RuSBH%{KN z_EW)@Tk?$E73nV|Z2C0v$1MXFeAdNwW4T8C)4=;TZ|_cB^Ub-P?M5y3U$}YwzHut;K_`_SpKkw;qcB6FX?x3yLOLu-> z@a=cm-(E}3d+|W*qL;_Fxc|wVpzbYGCq20kx&PvnBXmsc&$|wV-_5M;bYh%nX{~>T zva!7nJ^6Utie76b?r9X?yI0(7vMe;P-QzERNeI1F`A)}oW?gu2uzBsH44)*-(DzWu1qTAijw~5EcrH_Bfs{JGRFMYK#TTEO%7XR|wt&(5e5o@R{ z?=LC4d1PvDB%h}He#^QOZ-?F)W`M7iBmIeB_$5-ohwwH@5Q3lKgQNveYzrS`Cj z1SyXsAo0e?3J7svXsNAKNR*TDiO@zQ5tS_ZP6|xX{>?m(6O{MDLFOGn4H>CozyWo)x>t{RFrgm@xREACYQ zK72WR|Bk-2+Kf`a;1B?Jd5xOhdBMwzf8SDPg26a^h8QE<3)n#Np1`+LSgNU;Ew|jU0${^_+vt>9YH9i{zUqd zjYMF2M=E~CTq3%)cupkkM++p70T{MOvUKUYVTSM7OboLNwc1f2N_Xzm4iczg0G~xOJpDPH8ba zobijJ1EDFgQR7Jin#m(w_FX%#w}NOE5{jh9Z0rV-FoxVv`BA9e2yu2j*;f2^@7D;? z%gW#{Bjc1#xYC8ny`hP5O2{3M1`uN!tx@d)W@t(A5AoF4Zba~n)m4O4eQ+qzO+$v+ z3H8{OL~uIcXm)OO3UU1dLgYv{^k`Q7Y91MVGUE}s`WDfhBx;Xx*RkEb{^t+EQ%9cc^nOw{RU6nCea-7rE0f?{>XJ8hMv@Rg2MRs~2@ z4tV%&w;3xOrczZYBtlfEN(~Oyq{2a875-)y73}P=R;`d!qOUBn*@IId8_3=}goxz7 zg2fTsG!Vs#)YXK@#FA$S6FEJsbtpWVhnpOA8-g&4-%zb15-VAFRHuTq2AyaM>)fny zvy~D^x{{6}O37R1W!>)vS2)S!7U%M52f&CQ1kgQA3<1&J!1ii^RW)Z;3m^UE){b9;iSeDJBU% zW2X;WQ`q`KR$H8h4ztgwak3-K4mLAo(tx<6)Yw9+p{lATJxQI|AwivzmK>L7BctQz zv`z1nl+Y;&uV`6MLYmoQ1aJvnp_IZ!T-W%d4xOXKN$+<}R(DM8*cmo0`W}Gtny!hR zqhKq6@;8F&hmjnpW|YHe(v#shbHmz1A!4l>U0V}s*(MbbF+}dHG*F-Dxw;kpfuesw z^{pUu@CMPrAc9=J@cV5bOGp-l$kGZs!3Rmutz63q*LA>YrdG~`)&_B!0&c(2;G`tD>_s~5A#Zwn6GNvzzxkKMDfajVa!${+44o2?iuFZnWJ-4Dvd-f!*n z|Cl^Q?jQ??uQgc?zUcR>rTdHRLj{+M2d)40e!KiNF~=WdZ5miL#;i72qR({dI(hy4 z@ENgd1lGJ&qQ$3Y{OdDW{`OM^Z#F6JpIrRbgb!nfG-KXdI?-Ny z?cCUC^PiJC^)0i$BAZciuUYSssI;==#Pu5@KFL`3hH2&Y_}ELS*~v#X4>4y36Pt%f zKU}xq8~ueRhkt5!pld(sPFq@S&uVokutjj?#5C4?m(3D|CVVtMCU)h>#KRZgh>T`B z%u!BXbTM$1CA)PeCShLVGr|4j?f

tG#7cmM)(-cw*2*L(9iURxJ8!(#`Z3_WSF# zP0Ih;HS^4GKhLcg^mgvcu_w~U^!VlGqSU3;&KU=9z1BVQouttRYsJ#yEgxNY@Z%Ec z(Y1TW4O)D=)8{%fIq~;*mal5stADrVex|?J9iMdVDHzm3us1X2{5^TUKrJ@(3u-f#JbrZ_mDZK`Sg zt=6-WANjqIwkvJm#`!6i^i#`LA8gq(F{h->YZvX9@cr^(qH$#Rye)r?Y@)!4^M8nee{CqvpOqEOj|~E zzVTtZnx!MMM6D#WDkyqc_Y)l@`FZMzAus*5XU(0w@7KK6G_1@tHf-Lp2a){?Du=DO z&YoJ2cTx-B0!?j>=lG;MGSn1=1_EL3?Y{-mkq< z_*nmS#=&_}wT1ngzxdO`af?cRe|P+yUBvsfV>amI8PerrANKVhyyVl;g)3*BWaqs8 z`mzb8(j5iU4vd}nQQz-ZG^zP_=||a3!xFx{)#QViMbmr54Lk7UOkjBP-AfHpS*^hT zNXOmBiKZ9Z+C#Dry%nblZ`N|+mA$5bh?j?ayWMnuK`yg@wQAv#{j1w-xTUj(rg`-{ zQ@%SmVyt5R-iINPTaO)&o4Yzj_xYX|dq`s>I%7oNwoOv{cih#gGCM>1*|3i*n|#%_ zq=ed5bnC+MKPFYaDGFcFx5xF;sIO+v_Dk&X`q?wLzn?OGYp*H9-Ls6ox1hQ7lcDQQ z7hU_tS+Oo8bMn!*R=$}v=jQD3ms`oBT5AgZ&n%BxJbQ5PASV9$-d48OE7sCKO)!Lr z27bKt@=Ed4fV^6B)VAp12gZf0%e?LvrBW=+K0W*12ul6Z{P<59%+YHwWe|+=$ zZynD4wBowohYM{NsikAgfeA-P5AAw6vi;(nisC~3Z0Vvuz8=-~vE8yWsmr}{w}y8d zG&91oWQ z8HKwOivDcn#f^^+S|*E6q;E)E(>g5i&NuX*2s| zN0R|B=Qnxv2fvHoMy~%MIId?-Y|FXkK7DQ!t=#tOZ0hYY%j!=$9Nawgr_DPTUbq>& z;ExYSUpPBDH~j4Kw6t|#IansIWYd>Fnh&xI4fo>SJTbiMk>p8lo_O%vhL5Oye^kHn z;!8;(^M#FxYx{Q0BT_`>!o8pGkMGW^>G20YyY=wM?JLs#o2M;XJBTi&b9d`uIKf5;&FzoQe z&-Z2)JIcQ;8o0$jzxhKy>~h?Zo^C>%Ongn*qvC#7$){}+R`#Pe{Lxh$d%vJWnU}Eh zv2FddukM$WTr6A^Rrufdae3xU2t~A8r|%X?M8pTSG#+}d_SGC^Y$MnzOeYmEx+tr5j>t; zdv-?lO9gLqdVg_~OHYQ|C&!F=ToHRcGNb7qeRqY2{JiqQ+-SwQyR(v8W~^)a$A;fD z>n}Yj&%E%~nQ^T?>ho&Kx%m~bt2@~gs|M#NT7Pl8!;}l(FoKHl8$#ZyD9Jq*i+(C(T{_B=en$FPd+f_{OQwX)A^dJk`9o`+PePnUs0FB9vNE? zykwXch2sq&?XziFm7nsI*j2Ft-e7eCswMtJ0-xW(BOu_p8M zSEH@%r=84y{PFQY2{C{Eynn+oc63YNl67Qt?VTZ`&(s{wUTJ$U@PqF5@08s)U)vY7 zY}1@mhmXx(DSmwCt7>JhE(MmJJ#XC3(Irae{n+8xAusQlc`rk=XmtnOzJ>36Yk7F^ zo%@wzBF^g!W8!p=&M<>Jr+&A+&4|$zv1NVdWerp8uNM7PdGOkndG^RGP2-2x@BbKo zd&J@;n~L`3QyTpeU40izi4BRgD>n3 z9MxyCA^B$OCQXm`Sg>VrGxqH8vcpHhI+tY;7Z$BvaHF*U%ulXu>CooHi7a5robt7AlZvI{ zsCWHhQ2_AkaXl`oqeh(0>AU!dB4yhv6;rm9jCKxhXC5W~?y&Ys#L)`-*UKy1C{i(s zFBg%DSLIRngJRJMB4+%Y&(%?TL#Az*dU5E_6;jcF`DeO{*1jDq%uS@n=SlmelY7r4 zQC&GIlGG}Ttz3R=EvaY}i_RP!FJ%9sj=J{o{*3v5MTRc?^Zua)wr#E2tkZmFdgra2 zm2(#~9m#Ahc2SU&j48MP;O_@Tt)XL)I-Gl1GgTcm;AYcFK}&jcXgPStfnJe8?^UXU zYNmI-_r@oyjtu*<2lFG94ex0*dAe-Ki;l+(=_H@9om3nlquvn2B3%^IO}kSawdmeU zwZ$>_mR+qa)9m?ye0ArUysu8*yk*+3`JiOb-F^2cEhXc=ShEnLNlKC*+d(Q0`9<9c zi$zO8oLc#jI_l25#Ef-AH$+}110S?LpVa!~vG=xzU5g5-$@{)v(g%08QiT*8E$tPo zUB$%=MQ;sYWe0Kiiz>jJMYm; i>3~VAxZ`GNBE&)ZRemk|SN8EQnaretx~u_7#Qy=T1oS@u literal 0 HcmV?d00001 diff --git a/pkg/sync/type_scoped.go b/pkg/sync/type_scoped.go index cc89efe8b..65c140b7f 100644 --- a/pkg/sync/type_scoped.go +++ b/pkg/sync/type_scoped.go @@ -114,6 +114,7 @@ func (s *syncer) collectEnqueuedPageTokens(ctx context.Context, phase string, op phase, len(spawn.GetPageTokens()), maxEnqueuePageTokensPerResponse) } spawned := make([]Action, 0, len(spawn.GetPageTokens())) + totalTokenBytes := 0 for _, tok := range spawn.GetPageTokens() { if tok == "" { // An empty token would spawn a PLANNER call, not a page — @@ -126,6 +127,15 @@ func (s *syncer) collectEnqueuedPageTokens(ctx context.Context, phase string, op "%s: EnqueuePageTokens page token is %d bytes (max %d)", phase, len(tok), maxEnqueuedPageTokenBytes) } + totalTokenBytes += len(tok) + if totalTokenBytes > maxEnqueuedPageTokenTotalBytes { + // The per-item caps alone admit ~1GiB of tokens per legal + // response, all persisted into every checkpoint until the + // actions drain — bound the aggregate too. + return nil, fmt.Errorf( + "%s: EnqueuePageTokens carries more than %d total page-token bytes; shrink the tokens or chain additional spawns across pages", + phase, maxEnqueuedPageTokenTotalBytes) + } spawned = append(spawned, Action{ Op: op, ResourceTypeID: action.ResourceTypeID, diff --git a/pkg/tasks/c1api/full_sync.go b/pkg/tasks/c1api/full_sync.go index 935c57b5d..5e5058b7a 100644 --- a/pkg/tasks/c1api/full_sync.go +++ b/pkg/tasks/c1api/full_sync.go @@ -9,6 +9,7 @@ import ( "os" "path/filepath" "slices" + stdsync "sync" "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" "go.uber.org/zap" @@ -53,6 +54,30 @@ type fullSyncTaskHandler struct { // host a full c1z of disk. Computed once by the task manager via // previousSyncSparePath(). previousSyncSparePath string + + // previousSyncC1ZPath is an OPERATOR-supplied previous-sync c1z + // (--previous-sync-c1z), taking precedence over the SDK-owned spare. + // Unlike the spare it is never rotated or retired: deleting an + // operator's file is not ours to do (same contract as the local task + // manager), so a replay-integrity retry re-runs cold but leaves the + // file in place. + previousSyncC1ZPath string + + // spareMu serializes every mutation of the spare slot (promotion and + // retirement) across this manager's CONCURRENT full-sync handlers, + // which share one deterministic spare path. Without it, a failing + // task's replay-integrity retirement can race another task's + // promotion and delete a fresh, uninvolved spare. Shared from the + // task manager; nil in tests that don't exercise rotation. + spareMu *stdsync.Mutex + + // openedSpareSyncID records the sync id of the spare THIS task + // actually opened as its replay source (read at open time, before + // any concurrent rotation can swap the file). Retirement compares it + // against the CURRENT spare's id and only removes on a match — the + // failing sync may only retire the artifact that failed it, never a + // newer spare promoted underneath it. + openedSpareSyncID string } // previousSyncSparePrefix is the fixed filename prefix of the retained @@ -144,6 +169,70 @@ func fileExists(path string) bool { return err == nil && fi.Mode().IsRegular() } +// retireSpareIfStillImplicated removes the spare after a +// replay-integrity failure, but ONLY when it is still the artifact the +// failing sync replayed from. Concurrent full-sync tasks share one +// deterministic spare path: while this task was syncing (minutes to +// hours), another task may have finished and promoted a FRESH spare into +// the slot — that artifact is uninvolved in this failure and must +// survive, or a slow failing task permanently blinds replay that a +// successful sibling just re-armed. +// +// Identity is the sync id read from the c1z manifest header at OPEN time +// vs now. An unreadable id on either side degrades to removal (the old, +// conservative behavior): an extra cold sync is safe; replaying from an +// implicated spare is not. Callers must hold the manager's spare mutex +// so the check-then-remove cannot interleave with a promotion. +func retireSpareIfStillImplicated(ctx context.Context, sparePath, openedSyncID string) { + l := ctxzap.Extract(ctx) + if openedSyncID != "" { + if currentID := c1zSyncIDBestEffort(sparePath); currentID != "" && currentID != openedSyncID { + l.Info("previous-sync spare was rotated by a concurrent task since this sync opened it; leaving the fresh spare in place", + zap.String("spare_path", sparePath), + zap.String("implicated_sync_id", openedSyncID), + zap.String("current_spare_sync_id", currentID)) + return + } + } + if rmErr := os.Remove(sparePath); rmErr != nil && !os.IsNotExist(rmErr) { + l.Warn("failed to remove implicated previous-sync spare", zap.Error(rmErr)) + } +} + +// withSpareLock runs fn holding the manager-wide spare mutex (no-op +// when unset — tests that don't exercise rotation). +func (c *fullSyncTaskHandler) withSpareLock(fn func()) { + if c.spareMu != nil { + c.spareMu.Lock() + defer c.spareMu.Unlock() + } + fn() +} + +// resolveStorageEngine returns the engine this task's sync writes: +// runner config wins, then the task's own engine, then "" (the SDK +// default — SQLite). +func (c *fullSyncTaskHandler) resolveStorageEngine() c1zstore.Engine { + if c.storageEngine != "" { + return c.storageEngine + } + if taskEngine := c.task.GetSyncFull().GetStorageEngine(); taskEngine != "" { + return c1zstore.Engine(taskEngine) + } + return "" +} + +// spareRetentionEligible reports whether an artifact written with engine +// can ever serve source-cache replay. Only Pebble implements +// dotc1z.SourceCacheStore (SQLite replay is an explicit non-goal), so +// retaining a non-Pebble spare costs a full c1z of disk per rotation for +// a file every future sync would refuse — the retention is skipped, with +// a warning so an opted-in operator learns WHY replay never engages +// (set --storage-engine pebble) instead of silently paying for nothing. +func spareRetentionEligible(engine c1zstore.Engine) bool { + return engine == c1zstore.EnginePebble +} + func (c *fullSyncTaskHandler) sync(ctx context.Context, c1zPath string) error { ctx, span := tracer.Start(ctx, "fullSyncTaskHandler.sync") var err error @@ -159,11 +248,7 @@ func (c *fullSyncTaskHandler) sync(ctx context.Context, c1zPath string) error { sdkSync.WithTmpDir(c.helpers.TempDir()), sdkSync.WithWorkerCount(c.workerCount), } - engine := c.storageEngine - if engine == "" && c.task.GetSyncFull().GetStorageEngine() != "" { - engine = c1zstore.Engine(c.task.GetSyncFull().GetStorageEngine()) - } - if engine != "" { + if engine := c.resolveStorageEngine(); engine != "" { syncOpts = append(syncOpts, sdkSync.WithStorageEngine(engine)) } @@ -189,19 +274,30 @@ func (c *fullSyncTaskHandler) sync(ctx context.Context, c1zPath string) error { syncOpts = append(syncOpts, sdkSync.WithExternalResourceC1ZPath(c.externalResourceC1ZPath)) } - // source-cache replay (opt-in): feed the spare retained from the last - // successful upload as the previous-sync replay source. Optional - // semantics — a missing/corrupt/stale-format spare degrades to a - // plain sync and is replaced by this task's rotation on success, - // so a bad spare self-heals and can never fail the sync. Both - // branches log: a persistently-absent spare on an opted-in - // connector means rotation is broken, and that must be visible - // rather than silently disabling source-cache replay forever. - if c.previousSyncSparePath != "" { + // source-cache replay (opt-in): an operator-supplied + // --previous-sync-c1z path wins over the SDK-owned spare retained + // from the last successful upload. The spare uses optional semantics + // — a missing/corrupt/stale-format spare degrades to a plain sync + // and is replaced by this task's rotation on success, so a bad spare + // self-heals and can never fail the sync. Both spare branches log: a + // persistently-absent spare on an opted-in connector means rotation + // is broken, and that must be visible rather than silently disabling + // source-cache replay forever. + switch { + case c.previousSyncC1ZPath != "": + l.Info("using operator-supplied previous-sync c1z as the source-cache replay source", + zap.String("previous_sync_c1z", c.previousSyncC1ZPath), + zap.String("previous_sync_id", c1zSyncIDBestEffort(c.previousSyncC1ZPath))) + syncOpts = append(syncOpts, sdkSync.WithPreviousSyncC1ZPath(c.previousSyncC1ZPath)) + case c.previousSyncSparePath != "": if fileExists(c.previousSyncSparePath) { + // Record which artifact THIS sync replays from, so a + // replay-integrity retirement can never remove a fresh spare + // a concurrent task promoted mid-sync. + c.openedSpareSyncID = c1zSyncIDBestEffort(c.previousSyncSparePath) l.Info("previous-sync spare found; source-cache replay enabled for this sync", zap.String("spare_path", c.previousSyncSparePath), - zap.String("spare_sync_id", c1zSyncIDBestEffort(c.previousSyncSparePath))) + zap.String("spare_sync_id", c.openedSpareSyncID)) syncOpts = append(syncOpts, sdkSync.WithOptionalPreviousSyncC1ZPath(c.previousSyncSparePath)) } else { l.Info("no previous-sync spare on disk; source-cache replay inactive for this sync (expected on the first sync after enabling, or after a failed rotation)", @@ -271,9 +367,12 @@ func (c *fullSyncTaskHandler) sync(ctx context.Context, c1zPath string) error { return errors.Join(err, rmErr) } if c.previousSyncSparePath != "" { - if rmErr := os.Remove(c.previousSyncSparePath); rmErr != nil && !os.IsNotExist(rmErr) { - l.Warn("failed to remove implicated previous-sync spare", zap.Error(rmErr)) - } + // Under the spare lock, and only the exact artifact this sync + // opened — a concurrent task's freshly promoted spare is not + // implicated and must survive. + c.withSpareLock(func() { + retireSpareIfStillImplicated(ctx, c.previousSyncSparePath, c.openedSpareSyncID) + }) } err = runSync(append(slices.Clone(syncOpts), sdkSync.WithoutPreviousSync())) } @@ -348,7 +447,21 @@ func (c *fullSyncTaskHandler) HandleTask(ctx context.Context) error { // retained previous sync always corresponds to an artifact C1 has. uploaded := false defer func() { - promoteOrRemoveC1Z(ctx, c1zPath, c.previousSyncSparePath, c.previousSyncSparePath != "" && uploaded) + keep := c.previousSyncSparePath != "" && uploaded + if keep && !spareRetentionEligible(c.resolveStorageEngine()) { + // Retaining a spare that no future sync can replay from is + // pure disk cost; see spareRetentionEligible. + l.Warn("keep-previous-sync-c1z is enabled but this sync's storage engine cannot serve source-cache replay; not retaining a spare "+ + "(set --storage-engine pebble to activate replay)", + zap.String("storage_engine", string(c.resolveStorageEngine()))) + keep = false + } + // Under the spare lock: promotion renames onto the shared slot + // and must not interleave with another task's identity-checked + // retirement. + c.withSpareLock(func() { + promoteOrRemoveC1Z(ctx, c1zPath, c.previousSyncSparePath, keep) + }) }() defer func(f *os.File) { if closeErr := f.Close(); closeErr != nil { @@ -383,6 +496,8 @@ func newFullSyncTaskHandler( workerCount int, storageEngine c1zstore.Engine, previousSyncSparePath string, + previousSyncC1ZPath string, + spareMu *stdsync.Mutex, ) tasks.TaskHandler { return &fullSyncTaskHandler{ task: task, @@ -395,6 +510,8 @@ func newFullSyncTaskHandler( workerCount: workerCount, storageEngine: storageEngine, previousSyncSparePath: previousSyncSparePath, + previousSyncC1ZPath: previousSyncC1ZPath, + spareMu: spareMu, } } diff --git a/pkg/tasks/c1api/full_sync_spare_race_test.go b/pkg/tasks/c1api/full_sync_spare_race_test.go new file mode 100644 index 000000000..79dcf6f02 --- /dev/null +++ b/pkg/tasks/c1api/full_sync_spare_race_test.go @@ -0,0 +1,124 @@ +package c1api + +// Regression tests for the spare-slot lifecycle race: concurrent +// full-sync handlers share ONE deterministic spare path, so a failing +// task's replay-integrity retirement can interleave with a successful +// sibling's promotion. Retirement must therefore be identity-checked — +// it may only remove the exact artifact the failing sync replayed from, +// never a fresh spare promoted underneath it — and both mutations are +// serialized by the manager's spare mutex. + +import ( + "context" + "os" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + + v1 "github.com/conductorone/baton-sdk/pb/c1/connectorapi/baton/v1" + "github.com/conductorone/baton-sdk/pkg/connectorstore" + "github.com/conductorone/baton-sdk/pkg/dotc1z" + "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" + "github.com/conductorone/baton-sdk/pkg/logging" +) + +// writeSpareC1Z writes a minimal finished pebble c1z at path and returns +// its sync id (readable via c1zSyncIDBestEffort — the same identity the +// production retirement compares). +func writeSpareC1Z(ctx context.Context, t *testing.T, path, tmpDir string) string { + t.Helper() + store, err := dotc1z.NewStore(ctx, path, + dotc1z.WithEngine(c1zstore.EnginePebble), + dotc1z.WithTmpDir(tmpDir), + ) + require.NoError(t, err) + syncID, err := store.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + require.NoError(t, store.EndSync(ctx)) + require.NoError(t, store.Close(ctx)) + require.Equal(t, syncID, c1zSyncIDBestEffort(path), + "fixture: the spare's manifest header must carry the sync id the identity check reads") + return syncID +} + +func TestRetireSpareIfStillImplicated(t *testing.T) { + ctx, err := logging.Init(context.Background()) + require.NoError(t, err) + tmpDir := t.TempDir() + sparePath := filepath.Join(tmpDir, "baton-previous-sync-test.c1z") + + t.Run("rotated spare survives a stale task's retirement", func(t *testing.T) { + // Task A opens spare X, syncs for a long time. Meanwhile task B + // finishes and promotes a FRESH spare Y onto the same path. A's + // replay-integrity failure implicates X — not Y. + implicatedID := writeSpareC1Z(ctx, t, sparePath, tmpDir) + + freshPath := filepath.Join(tmpDir, "fresh.c1z") + freshID := writeSpareC1Z(ctx, t, freshPath, tmpDir) + require.NotEqual(t, implicatedID, freshID) + require.NoError(t, os.Rename(freshPath, sparePath)) // B's promotion + + retireSpareIfStillImplicated(ctx, sparePath, implicatedID) + + require.FileExists(t, sparePath, + "a fresh spare promoted by a concurrent task is not implicated and must survive") + require.Equal(t, freshID, c1zSyncIDBestEffort(sparePath)) + }) + + t.Run("unrotated spare is removed", func(t *testing.T) { + implicatedID := writeSpareC1Z(ctx, t, sparePath, tmpDir) + retireSpareIfStillImplicated(ctx, sparePath, implicatedID) + require.NoFileExists(t, sparePath, + "the artifact the failing sync replayed from must be retired") + }) + + t.Run("unknown opened identity degrades to removal", func(t *testing.T) { + // The failing sync could not identify what it opened (unreadable + // header): conservative direction is retirement — an extra cold + // sync is safe, replaying from an implicated spare is not. + writeSpareC1Z(ctx, t, sparePath, tmpDir) + retireSpareIfStillImplicated(ctx, sparePath, "") + require.NoFileExists(t, sparePath) + }) + + t.Run("unreadable current spare degrades to removal", func(t *testing.T) { + // The slot holds something the identity check can't read (e.g. a + // truncated file): it can't be positively identified as a fresh + // promotion, so it is retired. + require.NoError(t, os.WriteFile(sparePath, []byte("not a c1z"), 0o600)) + retireSpareIfStillImplicated(ctx, sparePath, "some-sync-id") + require.NoFileExists(t, sparePath) + }) + + t.Run("missing spare is a no-op", func(t *testing.T) { + retireSpareIfStillImplicated(ctx, filepath.Join(tmpDir, "ghost.c1z"), "some-sync-id") + }) +} + +// TestSpareRetentionEligibility pins the engine gate on spare rotation: +// only Pebble artifacts can serve source-cache replay (SQLite replay is +// an explicit non-goal), so a keep-previous-sync-c1z opt-in running on +// the DEFAULT engine (empty = SQLite) must not retain a spare — a full +// c1z of disk per rotation for a file every future sync would refuse. +func TestSpareRetentionEligibility(t *testing.T) { + require.True(t, spareRetentionEligible(c1zstore.EnginePebble)) + require.False(t, spareRetentionEligible(c1zstore.EngineSQLite)) + require.False(t, spareRetentionEligible(""), "the empty engine resolves to the SQLite default and must not retain") + + // Engine resolution precedence: runner config > task > default. + h := &fullSyncTaskHandler{task: fullSyncTask(t, "")} + require.Equal(t, c1zstore.Engine(""), h.resolveStorageEngine(), "no config, no task engine: SDK default (SQLite)") + h = &fullSyncTaskHandler{task: fullSyncTask(t, "pebble")} + require.Equal(t, c1zstore.EnginePebble, h.resolveStorageEngine(), "task-supplied engine applies") + h = &fullSyncTaskHandler{task: fullSyncTask(t, "sqlite"), storageEngine: c1zstore.EnginePebble} + require.Equal(t, c1zstore.EnginePebble, h.resolveStorageEngine(), "runner config wins over the task") +} + +// fullSyncTask builds a minimal full-sync task carrying storageEngine. +func fullSyncTask(t *testing.T, storageEngine string) *v1.Task { + t.Helper() + syncFull := &v1.Task_SyncFullTask{} + syncFull.SetStorageEngine(storageEngine) + return v1.Task_builder{SyncFull: syncFull}.Build() +} diff --git a/pkg/tasks/c1api/manager.go b/pkg/tasks/c1api/manager.go index b88b6b096..032c9a956 100644 --- a/pkg/tasks/c1api/manager.go +++ b/pkg/tasks/c1api/manager.go @@ -6,6 +6,7 @@ import ( "errors" "os" "strconv" + stdsync "sync" "sync/atomic" "time" @@ -76,6 +77,17 @@ type c1ApiTaskManager struct { // construction so every full-sync task agrees on the same spare. previousSyncSparePath string + // previousSyncC1ZPath is an operator-supplied previous-sync c1z + // (--previous-sync-c1z); takes precedence over the spare and is + // never rotated or deleted by the SDK. + previousSyncC1ZPath string + + // spareLifecycleMu serializes spare-slot mutations (promotion, + // replay-integrity retirement) across concurrent full-sync handlers, + // which all share previousSyncSparePath. See + // retireSpareIfStillImplicated. + spareLifecycleMu stdsync.Mutex + // runnerShouldDebug is flipped by the StartDebugging task handler (which // runs on a task-processing goroutine) and read by the runner loop via // ShouldDebug(). It is atomic to avoid a data race between those two. @@ -434,6 +446,8 @@ func (c *c1ApiTaskManager) Process(ctx context.Context, task *v1.Task, cc types. c.workerCount, c.storageEngine, c.previousSyncSparePath, + c.previousSyncC1ZPath, + &c.spareLifecycleMu, ) case taskTypes.HelloType: handler = newHelloTaskHandler(task, tHelpers) @@ -503,6 +517,7 @@ func NewC1TaskManager( storageEngine c1zstore.Engine, taskConcurrency int, keepPreviousSyncC1Z bool, + previousSyncC1Z string, ) (BootstrappingTaskManager, error) { serviceClient, err := newServiceClient(ctx, clientID, clientSecret) if err != nil { @@ -530,5 +545,6 @@ func NewC1TaskManager( workerCount: workerCount, storageEngine: storageEngine, previousSyncSparePath: sparePath, + previousSyncC1ZPath: previousSyncC1Z, }, nil } From 231f2d05f2f17379ec656aeda6c373484859b040 Mon Sep 17 00:00:00 2001 From: Matt Kaniaris Date: Thu, 16 Jul 2026 09:29:03 -0600 Subject: [PATCH 13/17] expanded harness --- .gitignore | 2 + cmd/baton/source_cache.go | 99 +++++- .../v2/annotation_source_cache.pb.go | 80 ++++- .../v2/annotation_source_cache.pb.validate.go | 30 ++ .../annotation_source_cache_protoopaque.pb.go | 61 +++- .../annotation_type_scoped_entitlements.pb.go | 11 +- ...type_scoped_entitlements_protoopaque.pb.go | 11 +- .../v2/annotation_type_scoped_grants.pb.go | 9 +- ...ation_type_scoped_grants_protoopaque.pb.go | 9 +- pb/c1/storage/v3/records.pb.go | 281 +++++++++++++++--- pb/c1/storage/v3/records.pb.validate.go | 114 +++++++ pb/c1/storage/v3/records_protoopaque.pb.go | 259 +++++++++++++--- pkg/dotc1z/engine/pebble/source_cache.go | 120 ++++---- .../pebble/source_cache_preclear_test.go | 166 +++++++++++ pkg/dotc1z/source_cache.go | 5 + pkg/sync/expand/drop_stats_test.go | 33 ++ .../expand/topological_merge_projection.go | 10 +- pkg/sync/ingest_invariants.go | 157 ++++++---- pkg/sync/ingest_invariants_probe_test.go | 97 ++++++ pkg/sync/source_cache.go | 49 +-- pkg/tasks/c1api/full_sync.go | 104 ++++++- pkg/tasks/c1api/full_sync_spare_race_test.go | 72 +++++ .../v2/annotation_source_cache.proto | 29 ++ .../annotation_type_scoped_entitlements.proto | 11 +- .../v2/annotation_type_scoped_grants.proto | 9 +- proto/c1/storage/v3/records.proto | 47 +++ 26 files changed, 1586 insertions(+), 289 deletions(-) create mode 100644 pkg/dotc1z/engine/pebble/source_cache_preclear_test.go create mode 100644 pkg/sync/ingest_invariants_probe_test.go diff --git a/.gitignore b/.gitignore index cf8abcdf3..6fbb2fc74 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,8 @@ *.dylib build/ *.c1z +# Frozen test fixtures are the exception (version-skew corpus etc.). +!pkg/sync/testdata/*.c1z *.db *.csv *.xlsx diff --git a/cmd/baton/source_cache.go b/cmd/baton/source_cache.go index eb82539b4..1c086e361 100644 --- a/cmd/baton/source_cache.go +++ b/cmd/baton/source_cache.go @@ -15,6 +15,7 @@ import ( "github.com/conductorone/baton-sdk/pkg/dotc1z" "github.com/conductorone/baton-sdk/pkg/logging" "github.com/conductorone/baton-sdk/pkg/sourcecache" + sdkSync "github.com/conductorone/baton-sdk/pkg/sync" ) // source-cache: the attribution surface for source-cache replay. Rows in @@ -25,7 +26,12 @@ import ( // how many rows does it hold, and is its index/manifest state consistent. // // Exits non-zero when any orphan-index scope is found (ingestion -// invariant I6): such a file would poison a future sync's replay. +// invariant I6) on a file that is ELIGIBLE as a replay source: such a +// file would poison a future sync's replay. Files the metadata gate +// already refuses (compaction folds — which legitimately clear the +// manifest while keeping scope stamps — and partial/derived syncs) can +// never be replayed from, so their index/manifest divergence is reported +// as "stale-index" without failing the audit. func sourceCacheCmd() *cobra.Command { cmd := &cobra.Command{ Use: "source-cache", @@ -42,10 +48,15 @@ type sourceCacheScopeReport struct { Validator string `json:"cache_validator,omitempty"` DiscoveredAt time.Time `json:"discovered_at,omitempty"` // Status: "ok" (manifest + rows), "zero-row" (manifest entry with no - // stamped rows — legal: the scope legitimately returned nothing), or - // "orphan-index" (stamped rows with NO manifest entry — an invariant - // violation that would poison the next sync's replay; see ingestion - // invariant I6). + // stamped rows — legal: the scope legitimately returned nothing), + // "invalidated" (the dangling-reference drops removed rows from this + // scope, so its entry is marked to miss and re-fetch cold next sync — + // legal, informational), "orphan-index" (stamped rows with NO + // manifest entry on a replay-eligible file — an invariant violation + // that would poison the next sync's replay; see ingestion invariant + // I6), or "stale-index" (the same shape on a file the replay metadata + // gate already refuses, e.g. a compaction fold whose manifest was + // deliberately cleared — informational, never a failure). Status string `json:"status"` } @@ -53,12 +64,16 @@ type sourceCacheAuditReport struct { SyncID string `json:"sync_id,omitempty"` // SyncType / Compacted / UsableAsReplaySource describe whether a // FUTURE sync may use this file as its replay source (the metadata - // gate; see c1zstore.SyncRun.UsableAsReplaySource). + // gate; see c1zstore.SyncRun.UsableAsReplaySource, plus the + // compaction-provenance token check the syncer applies for artifacts + // produced by pre-compacted-flag compactors). SyncType string `json:"sync_type,omitempty"` Compacted bool `json:"compacted"` UsableAsReplaySource bool `json:"usable_as_replay_source"` + ReplayUnusableReason string `json:"replay_unusable_reason,omitempty"` Scopes []sourceCacheScopeReport `json:"scopes"` OrphanIndexScopes int `json:"orphan_index_scopes"` + StaleIndexScopes int `json:"stale_index_scopes,omitempty"` } func runSourceCacheAudit(cmd *cobra.Command, args []string) error { @@ -87,11 +102,44 @@ func runSourceCacheAudit(cmd *cobra.Command, args []string) error { } report := sourceCacheAuditReport{} - if latest, err := store.SyncMeta().LatestFullSync(ctx); err == nil && latest != nil { + // The newest FINISHED sync of any type: it is the sync the store + // opened (see openReadOnlyC1ZStore's SetCurrentSync) and the one the + // replay metadata gate would evaluate — LatestFullSync could describe + // an older sync than the state being inspected, or nothing at all on + // partial/derived artifacts. + // + // A metadata READ FAILURE is fatal: replay eligibility gates whether + // orphan indexes are a hard failure or informational, so silently + // defaulting eligibility to false would let the audit exit zero on a + // file it could not actually judge. + latest, err := store.SyncMeta().LatestFinishedSyncOfAnyType(ctx) + if err != nil { + return fmt.Errorf("reading sync run metadata (needed to judge replay eligibility): %w", err) + } + if latest != nil { report.SyncID = latest.ID report.SyncType = string(latest.Type) report.Compacted = latest.Compacted - report.UsableAsReplaySource = latest.UsableAsReplaySource() + // The syncer's own metadata gate (type, compacted flag, + // compaction-provenance token) — shared, so audit and syncer can + // never drift. + report.ReplayUnusableReason = sdkSync.ReplaySourceRunUnusableReason(latest) + report.UsableAsReplaySource = report.ReplayUnusableReason == "" + } + if report.UsableAsReplaySource { + // The replay-compatibility gate's offline-decidable half: key + // PRESENCE. An exact-match verdict depends on the NEXT run's + // connector declarations (cache generation, config fingerprint, + // selection), which no offline audit can know — but an ABSENT + // key can never match any future key, so the syncer will always + // run cold from this file (pre-compat-key SDKs, compaction + // folds). Reporting such a file usable=true would be wrong. + if _, compatFound, compatErr := inspector.GetSourceCacheCompat(ctx); compatErr != nil { + return fmt.Errorf("reading replay-compatibility key: %w", compatErr) + } else if !compatFound { + report.UsableAsReplaySource = false + report.ReplayUnusableReason = "no replay-compatibility key recorded (pre-key SDK or compaction fold); the syncer treats an absent key as a mismatch and always runs cold" + } } manifest, err := inspector.SourceCacheManifestSnapshot(ctx) @@ -107,7 +155,7 @@ func runSourceCacheAudit(cmd *cobra.Command, args []string) error { return fmt.Errorf("checking scope consistency: %w", err) } - for key := range manifest { + for key, snapEntry := range manifest { kind, scope, found := strings.Cut(key, "\x00") if !found { return fmt.Errorf("malformed manifest snapshot key %q", key) @@ -121,19 +169,38 @@ func runSourceCacheAudit(cmd *cobra.Command, args []string) error { rep.Validator = entry.CacheValidator rep.DiscoveredAt = entry.DiscoveredAt } - if rep.Rows == 0 { + switch { + case snapEntry.Invalidated: + // The lookup surface reports invalidated entries as misses; + // surface the retained (stale) validator for forensics. + rep.Validator = snapEntry.CacheValidator + rep.Status = "invalidated" + case rep.Rows == 0: rep.Status = "zero-row" - } else { + default: rep.Status = "ok" } report.Scopes = append(report.Scopes, rep) } for kind, scopes := range orphans { for _, scope := range scopes { + status := "orphan-index" + if !report.UsableAsReplaySource { + // Replay from this file is already impossible (compaction + // fold, partial/derived sync): the divergence cannot poison + // anything. Compaction folds in particular ALWAYS look like + // this — the fold clears the manifest and keeps scope + // stamps by design (see pebble.ClearSourceCacheEntries). + status = "stale-index" + } report.Scopes = append(report.Scopes, sourceCacheScopeReport{ - RowKind: kind, ScopeKey: scope, Rows: indexCounts[kind][scope], Status: "orphan-index", + RowKind: kind, ScopeKey: scope, Rows: indexCounts[kind][scope], Status: status, }) - report.OrphanIndexScopes++ + if status == "orphan-index" { + report.OrphanIndexScopes++ + } else { + report.StaleIndexScopes++ + } } } sort.Slice(report.Scopes, func(i, j int) bool { @@ -160,8 +227,12 @@ func runSourceCacheAudit(cmd *cobra.Command, args []string) error { func printSourceCacheAudit(report sourceCacheAuditReport) error { if report.SyncID != "" { - fmt.Fprintf(os.Stdout, "sync %s type=%s compacted=%v usable-as-replay-source=%v\n\n", + fmt.Fprintf(os.Stdout, "sync %s type=%s compacted=%v usable-as-replay-source=%v\n", report.SyncID, report.SyncType, report.Compacted, report.UsableAsReplaySource) + if report.ReplayUnusableReason != "" { + fmt.Fprintf(os.Stdout, " (not usable: %s)\n", report.ReplayUnusableReason) + } + fmt.Fprintln(os.Stdout) } if len(report.Scopes) == 0 { fmt.Fprintln(os.Stdout, "no source-cache state (cold artifact: every scope will fetch fresh next sync)") diff --git a/pb/c1/connector/v2/annotation_source_cache.pb.go b/pb/c1/connector/v2/annotation_source_cache.pb.go index db954c3cc..f11ddfe0b 100644 --- a/pb/c1/connector/v2/annotation_source_cache.pb.go +++ b/pb/c1/connector/v2/annotation_source_cache.pb.go @@ -71,10 +71,31 @@ func (x SourceCacheCapability_Mode) Number() protoreflect.EnumNumber { // annotations to opt in to source-cache replay. Absent or any mode other // than MODE_READ_WRITE means all source-cache annotations are ignored. type SourceCacheCapability struct { - state protoimpl.MessageState `protogen:"hybrid.v1"` - Mode SourceCacheCapability_Mode `protobuf:"varint,1,opt,name=mode,proto3,enum=c1.connector.v2.SourceCacheCapability_Mode" json:"mode,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"hybrid.v1"` + Mode SourceCacheCapability_Mode `protobuf:"varint,1,opt,name=mode,proto3,enum=c1.connector.v2.SourceCacheCapability_Mode" json:"mode,omitempty"` + // cache_generation is the connector's replay-compatibility generation: + // bump it whenever the connector changes how it computes scopes, + // validators, or rows in a way that makes a PRIOR artifact's cached + // rows unsafe to replay (schema of emitted rows, scope partitioning, + // id construction). The SDK never interprets the value — replay is + // permitted only when the previous artifact recorded a byte-identical + // generation (any mismatch runs the sync cold). Never derive this from + // a semver: compatibility is an explicit declaration, not an + // inference. Empty is a valid (constant) generation. + CacheGeneration string `protobuf:"bytes,2,opt,name=cache_generation,json=cacheGeneration,proto3" json:"cache_generation,omitempty"` + // config_fingerprint is an opaque connector-computed digest of every + // configuration and permission input that changes what upstream data + // the connector CAN see (credentials scope, tenant/org selection, + // API-permission grants, include/exclude config). Replay is permitted + // only when the previous artifact recorded a byte-identical + // fingerprint: a config change means cached scopes may cover a + // different universe, and replaying them would resurrect rows the new + // configuration can no longer observe (or hide rows it now can). + // Empty means "connector declares no config sensitivity" and matches + // only empty. + ConfigFingerprint string `protobuf:"bytes,3,opt,name=config_fingerprint,json=configFingerprint,proto3" json:"config_fingerprint,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *SourceCacheCapability) Reset() { @@ -109,14 +130,57 @@ func (x *SourceCacheCapability) GetMode() SourceCacheCapability_Mode { return SourceCacheCapability_MODE_UNSPECIFIED } +func (x *SourceCacheCapability) GetCacheGeneration() string { + if x != nil { + return x.CacheGeneration + } + return "" +} + +func (x *SourceCacheCapability) GetConfigFingerprint() string { + if x != nil { + return x.ConfigFingerprint + } + return "" +} + func (x *SourceCacheCapability) SetMode(v SourceCacheCapability_Mode) { x.Mode = v } +func (x *SourceCacheCapability) SetCacheGeneration(v string) { + x.CacheGeneration = v +} + +func (x *SourceCacheCapability) SetConfigFingerprint(v string) { + x.ConfigFingerprint = v +} + type SourceCacheCapability_builder struct { _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. Mode SourceCacheCapability_Mode + // cache_generation is the connector's replay-compatibility generation: + // bump it whenever the connector changes how it computes scopes, + // validators, or rows in a way that makes a PRIOR artifact's cached + // rows unsafe to replay (schema of emitted rows, scope partitioning, + // id construction). The SDK never interprets the value — replay is + // permitted only when the previous artifact recorded a byte-identical + // generation (any mismatch runs the sync cold). Never derive this from + // a semver: compatibility is an explicit declaration, not an + // inference. Empty is a valid (constant) generation. + CacheGeneration string + // config_fingerprint is an opaque connector-computed digest of every + // configuration and permission input that changes what upstream data + // the connector CAN see (credentials scope, tenant/org selection, + // API-permission grants, include/exclude config). Replay is permitted + // only when the previous artifact recorded a byte-identical + // fingerprint: a config change means cached scopes may cover a + // different universe, and replaying them would resurrect rows the new + // configuration can no longer observe (or hide rows it now can). + // Empty means "connector declares no config sensitivity" and matches + // only empty. + ConfigFingerprint string } func (b0 SourceCacheCapability_builder) Build() *SourceCacheCapability { @@ -124,6 +188,8 @@ func (b0 SourceCacheCapability_builder) Build() *SourceCacheCapability { b, x := &b0, m0 _, _ = b, x x.Mode = b.Mode + x.CacheGeneration = b.CacheGeneration + x.ConfigFingerprint = b.ConfigFingerprint return m0 } @@ -837,9 +903,11 @@ var File_c1_connector_v2_annotation_source_cache_proto protoreflect.FileDescript const file_c1_connector_v2_annotation_source_cache_proto_rawDesc = "" + "\n" + - "-c1/connector/v2/annotation_source_cache.proto\x12\x0fc1.connector.v2\x1a\x17validate/validate.proto\"\x9e\x01\n" + + "-c1/connector/v2/annotation_source_cache.proto\x12\x0fc1.connector.v2\x1a\x17validate/validate.proto\"\x92\x02\n" + "\x15SourceCacheCapability\x12?\n" + - "\x04mode\x18\x01 \x01(\x0e2+.c1.connector.v2.SourceCacheCapability.ModeR\x04mode\"D\n" + + "\x04mode\x18\x01 \x01(\x0e2+.c1.connector.v2.SourceCacheCapability.ModeR\x04mode\x126\n" + + "\x10cache_generation\x18\x02 \x01(\tB\v\xfaB\br\x06(\x80\x02\xd0\x01\x01R\x0fcacheGeneration\x12:\n" + + "\x12config_fingerprint\x18\x03 \x01(\tB\v\xfaB\br\x06(\x80\x02\xd0\x01\x01R\x11configFingerprint\"D\n" + "\x04Mode\x12\x14\n" + "\x10MODE_UNSPECIFIED\x10\x00\x12\x11\n" + "\rMODE_DISABLED\x10\x01\x12\x13\n" + diff --git a/pb/c1/connector/v2/annotation_source_cache.pb.validate.go b/pb/c1/connector/v2/annotation_source_cache.pb.validate.go index a60200dce..80fb38aa0 100644 --- a/pb/c1/connector/v2/annotation_source_cache.pb.validate.go +++ b/pb/c1/connector/v2/annotation_source_cache.pb.validate.go @@ -59,6 +59,36 @@ func (m *SourceCacheCapability) validate(all bool) error { // no validation rules for Mode + if m.GetCacheGeneration() != "" { + + if len(m.GetCacheGeneration()) > 256 { + err := SourceCacheCapabilityValidationError{ + field: "CacheGeneration", + reason: "value length must be at most 256 bytes", + } + if !all { + return err + } + errors = append(errors, err) + } + + } + + if m.GetConfigFingerprint() != "" { + + if len(m.GetConfigFingerprint()) > 256 { + err := SourceCacheCapabilityValidationError{ + field: "ConfigFingerprint", + reason: "value length must be at most 256 bytes", + } + if !all { + return err + } + errors = append(errors, err) + } + + } + if len(errors) > 0 { return SourceCacheCapabilityMultiError(errors) } diff --git a/pb/c1/connector/v2/annotation_source_cache_protoopaque.pb.go b/pb/c1/connector/v2/annotation_source_cache_protoopaque.pb.go index e44b120f5..25a3a06ab 100644 --- a/pb/c1/connector/v2/annotation_source_cache_protoopaque.pb.go +++ b/pb/c1/connector/v2/annotation_source_cache_protoopaque.pb.go @@ -71,10 +71,12 @@ func (x SourceCacheCapability_Mode) Number() protoreflect.EnumNumber { // annotations to opt in to source-cache replay. Absent or any mode other // than MODE_READ_WRITE means all source-cache annotations are ignored. type SourceCacheCapability struct { - state protoimpl.MessageState `protogen:"opaque.v1"` - xxx_hidden_Mode SourceCacheCapability_Mode `protobuf:"varint,1,opt,name=mode,proto3,enum=c1.connector.v2.SourceCacheCapability_Mode"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + state protoimpl.MessageState `protogen:"opaque.v1"` + xxx_hidden_Mode SourceCacheCapability_Mode `protobuf:"varint,1,opt,name=mode,proto3,enum=c1.connector.v2.SourceCacheCapability_Mode"` + xxx_hidden_CacheGeneration string `protobuf:"bytes,2,opt,name=cache_generation,json=cacheGeneration,proto3"` + xxx_hidden_ConfigFingerprint string `protobuf:"bytes,3,opt,name=config_fingerprint,json=configFingerprint,proto3"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *SourceCacheCapability) Reset() { @@ -109,14 +111,57 @@ func (x *SourceCacheCapability) GetMode() SourceCacheCapability_Mode { return SourceCacheCapability_MODE_UNSPECIFIED } +func (x *SourceCacheCapability) GetCacheGeneration() string { + if x != nil { + return x.xxx_hidden_CacheGeneration + } + return "" +} + +func (x *SourceCacheCapability) GetConfigFingerprint() string { + if x != nil { + return x.xxx_hidden_ConfigFingerprint + } + return "" +} + func (x *SourceCacheCapability) SetMode(v SourceCacheCapability_Mode) { x.xxx_hidden_Mode = v } +func (x *SourceCacheCapability) SetCacheGeneration(v string) { + x.xxx_hidden_CacheGeneration = v +} + +func (x *SourceCacheCapability) SetConfigFingerprint(v string) { + x.xxx_hidden_ConfigFingerprint = v +} + type SourceCacheCapability_builder struct { _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. Mode SourceCacheCapability_Mode + // cache_generation is the connector's replay-compatibility generation: + // bump it whenever the connector changes how it computes scopes, + // validators, or rows in a way that makes a PRIOR artifact's cached + // rows unsafe to replay (schema of emitted rows, scope partitioning, + // id construction). The SDK never interprets the value — replay is + // permitted only when the previous artifact recorded a byte-identical + // generation (any mismatch runs the sync cold). Never derive this from + // a semver: compatibility is an explicit declaration, not an + // inference. Empty is a valid (constant) generation. + CacheGeneration string + // config_fingerprint is an opaque connector-computed digest of every + // configuration and permission input that changes what upstream data + // the connector CAN see (credentials scope, tenant/org selection, + // API-permission grants, include/exclude config). Replay is permitted + // only when the previous artifact recorded a byte-identical + // fingerprint: a config change means cached scopes may cover a + // different universe, and replaying them would resurrect rows the new + // configuration can no longer observe (or hide rows it now can). + // Empty means "connector declares no config sensitivity" and matches + // only empty. + ConfigFingerprint string } func (b0 SourceCacheCapability_builder) Build() *SourceCacheCapability { @@ -124,6 +169,8 @@ func (b0 SourceCacheCapability_builder) Build() *SourceCacheCapability { b, x := &b0, m0 _, _ = b, x x.xxx_hidden_Mode = b.Mode + x.xxx_hidden_CacheGeneration = b.CacheGeneration + x.xxx_hidden_ConfigFingerprint = b.ConfigFingerprint return m0 } @@ -779,9 +826,11 @@ var File_c1_connector_v2_annotation_source_cache_proto protoreflect.FileDescript const file_c1_connector_v2_annotation_source_cache_proto_rawDesc = "" + "\n" + - "-c1/connector/v2/annotation_source_cache.proto\x12\x0fc1.connector.v2\x1a\x17validate/validate.proto\"\x9e\x01\n" + + "-c1/connector/v2/annotation_source_cache.proto\x12\x0fc1.connector.v2\x1a\x17validate/validate.proto\"\x92\x02\n" + "\x15SourceCacheCapability\x12?\n" + - "\x04mode\x18\x01 \x01(\x0e2+.c1.connector.v2.SourceCacheCapability.ModeR\x04mode\"D\n" + + "\x04mode\x18\x01 \x01(\x0e2+.c1.connector.v2.SourceCacheCapability.ModeR\x04mode\x126\n" + + "\x10cache_generation\x18\x02 \x01(\tB\v\xfaB\br\x06(\x80\x02\xd0\x01\x01R\x0fcacheGeneration\x12:\n" + + "\x12config_fingerprint\x18\x03 \x01(\tB\v\xfaB\br\x06(\x80\x02\xd0\x01\x01R\x11configFingerprint\"D\n" + "\x04Mode\x12\x14\n" + "\x10MODE_UNSPECIFIED\x10\x00\x12\x11\n" + "\rMODE_DISABLED\x10\x01\x12\x13\n" + diff --git a/pb/c1/connector/v2/annotation_type_scoped_entitlements.pb.go b/pb/c1/connector/v2/annotation_type_scoped_entitlements.pb.go index 81fad6a4a..e5f9a8b45 100644 --- a/pb/c1/connector/v2/annotation_type_scoped_entitlements.pb.go +++ b/pb/c1/connector/v2/annotation_type_scoped_entitlements.pb.go @@ -70,11 +70,12 @@ const ( // leaves type-scoped entitlement enumeration to a full entitlements // phase. // -// ROUTING MARKER ONLY: TypeScopedEntitlements is not a stream-coupled -// side-effect annotation. It is absent from the ingestion-invariant -// side-effect coverage map for the same reason as TypeScopedGrants — -// it routes the ListEntitlements call; it does not imply post-ingest -// repair or validation beyond ordinary entitlement handling. +// ROUTING MARKER: TypeScopedEntitlements routes the ListEntitlements +// call; it implies no repair machinery of its own. It IS listed in the +// ingestion-invariant side-effect coverage map, covered by invariant I7 +// (entitlement→resource referential integrity): type-granularity scopes +// can carry entitlement rows for resources that vanished between +// enumerations, and the post-collection check is what catches that. type TypeScopedEntitlements struct { state protoimpl.MessageState `protogen:"hybrid.v1"` unknownFields protoimpl.UnknownFields diff --git a/pb/c1/connector/v2/annotation_type_scoped_entitlements_protoopaque.pb.go b/pb/c1/connector/v2/annotation_type_scoped_entitlements_protoopaque.pb.go index d65f6c11c..c5773acd6 100644 --- a/pb/c1/connector/v2/annotation_type_scoped_entitlements_protoopaque.pb.go +++ b/pb/c1/connector/v2/annotation_type_scoped_entitlements_protoopaque.pb.go @@ -70,11 +70,12 @@ const ( // leaves type-scoped entitlement enumeration to a full entitlements // phase. // -// ROUTING MARKER ONLY: TypeScopedEntitlements is not a stream-coupled -// side-effect annotation. It is absent from the ingestion-invariant -// side-effect coverage map for the same reason as TypeScopedGrants — -// it routes the ListEntitlements call; it does not imply post-ingest -// repair or validation beyond ordinary entitlement handling. +// ROUTING MARKER: TypeScopedEntitlements routes the ListEntitlements +// call; it implies no repair machinery of its own. It IS listed in the +// ingestion-invariant side-effect coverage map, covered by invariant I7 +// (entitlement→resource referential integrity): type-granularity scopes +// can carry entitlement rows for resources that vanished between +// enumerations, and the post-collection check is what catches that. type TypeScopedEntitlements struct { state protoimpl.MessageState `protogen:"opaque.v1"` unknownFields protoimpl.UnknownFields diff --git a/pb/c1/connector/v2/annotation_type_scoped_grants.pb.go b/pb/c1/connector/v2/annotation_type_scoped_grants.pb.go index 609293177..c5e1bf688 100644 --- a/pb/c1/connector/v2/annotation_type_scoped_grants.pb.go +++ b/pb/c1/connector/v2/annotation_type_scoped_grants.pb.go @@ -65,9 +65,12 @@ const ( // SyncGrantsOp for a type-scoped type. Type-scoped grant enumeration is // left to a full grants phase. // -// ROUTING MARKER ONLY: TypeScopedGrants is absent from the ingestion- -// invariant side-effect coverage map — it routes ListGrants; it does not -// imply post-ingest repair beyond ordinary grant handling. +// ROUTING MARKER: TypeScopedGrants routes ListGrants; it implies no +// repair machinery of its own. It IS listed in the ingestion-invariant +// side-effect coverage map, covered by invariant I8 (grant→entitlement +// referential integrity): independently-fresh type scopes can strand +// grant references against a refreshed entitlement set, and the +// post-collection check is what catches that. type TypeScopedGrants struct { state protoimpl.MessageState `protogen:"hybrid.v1"` unknownFields protoimpl.UnknownFields diff --git a/pb/c1/connector/v2/annotation_type_scoped_grants_protoopaque.pb.go b/pb/c1/connector/v2/annotation_type_scoped_grants_protoopaque.pb.go index 862fc1fc6..4a30650a3 100644 --- a/pb/c1/connector/v2/annotation_type_scoped_grants_protoopaque.pb.go +++ b/pb/c1/connector/v2/annotation_type_scoped_grants_protoopaque.pb.go @@ -65,9 +65,12 @@ const ( // SyncGrantsOp for a type-scoped type. Type-scoped grant enumeration is // left to a full grants phase. // -// ROUTING MARKER ONLY: TypeScopedGrants is absent from the ingestion- -// invariant side-effect coverage map — it routes ListGrants; it does not -// imply post-ingest repair beyond ordinary grant handling. +// ROUTING MARKER: TypeScopedGrants routes ListGrants; it implies no +// repair machinery of its own. It IS listed in the ingestion-invariant +// side-effect coverage map, covered by invariant I8 (grant→entitlement +// referential integrity): independently-fresh type scopes can strand +// grant references against a refreshed entitlement set, and the +// post-collection check is what catches that. type TypeScopedGrants struct { state protoimpl.MessageState `protogen:"opaque.v1"` unknownFields protoimpl.UnknownFields diff --git a/pb/c1/storage/v3/records.pb.go b/pb/c1/storage/v3/records.pb.go index e54fd02c2..7f80444f8 100644 --- a/pb/c1/storage/v3/records.pb.go +++ b/pb/c1/storage/v3/records.pb.go @@ -1770,8 +1770,16 @@ type SourceCacheEntryRecord struct { // Opaque upstream validator: HTTP ETag, delta token, etc. CacheValidator string `protobuf:"bytes,3,opt,name=cache_validator,json=cacheValidator,proto3" json:"cache_validator,omitempty"` DiscoveredAt *timestamppb.Timestamp `protobuf:"bytes,4,opt,name=discovered_at,json=discoveredAt,proto3" json:"discovered_at,omitempty"` - unknownFields protoimpl.UnknownFields - sizeCache protoimpl.SizeCache + // Set when the dangling-reference drops (ingestion invariants I7/I8/I9, + // pkg/sync/ingest_invariants.go) deleted rows stamped with this scope: + // the artifact no longer holds the full row set the validator vouches + // for, so a future sync must not 304-replay it. Lookups treat an + // invalidated entry as a miss (the scope re-fetches cold and converges + // with a cold sync); the entry itself is kept so the scope's surviving + // stamped rows do not read as an I6 orphan (lost manifest write). + Invalidated bool `protobuf:"varint,5,opt,name=invalidated,proto3" json:"invalidated,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache } func (x *SourceCacheEntryRecord) Reset() { @@ -1827,6 +1835,13 @@ func (x *SourceCacheEntryRecord) GetDiscoveredAt() *timestamppb.Timestamp { return nil } +func (x *SourceCacheEntryRecord) GetInvalidated() bool { + if x != nil { + return x.Invalidated + } + return false +} + func (x *SourceCacheEntryRecord) SetRowKind(v string) { x.RowKind = v } @@ -1843,6 +1858,10 @@ func (x *SourceCacheEntryRecord) SetDiscoveredAt(v *timestamppb.Timestamp) { x.DiscoveredAt = v } +func (x *SourceCacheEntryRecord) SetInvalidated(v bool) { + x.Invalidated = v +} + func (x *SourceCacheEntryRecord) HasDiscoveredAt() bool { if x == nil { return false @@ -1866,6 +1885,14 @@ type SourceCacheEntryRecord_builder struct { // Opaque upstream validator: HTTP ETag, delta token, etc. CacheValidator string DiscoveredAt *timestamppb.Timestamp + // Set when the dangling-reference drops (ingestion invariants I7/I8/I9, + // pkg/sync/ingest_invariants.go) deleted rows stamped with this scope: + // the artifact no longer holds the full row set the validator vouches + // for, so a future sync must not 304-replay it. Lookups treat an + // invalidated entry as a miss (the scope re-fetches cold and converges + // with a cold sync); the entry itself is kept so the scope's surviving + // stamped rows do not read as an I6 orphan (lost manifest write). + Invalidated bool } func (b0 SourceCacheEntryRecord_builder) Build() *SourceCacheEntryRecord { @@ -1876,6 +1903,153 @@ func (b0 SourceCacheEntryRecord_builder) Build() *SourceCacheEntryRecord { x.ScopeKey = b.ScopeKey x.CacheValidator = b.CacheValidator x.DiscoveredAt = b.DiscoveredAt + x.Invalidated = b.Invalidated + return m0 +} + +// SourceCacheCompatRecord is the sync's replay-compatibility key: the +// exact conditions under which this artifact's source-cache manifest was +// recorded. A future sync may replay from this artifact ONLY when its +// own key matches byte-exactly on every component; an absent, unreadable, +// or mismatched record degrades that sync to cold (no-op lookup). +// Compatibility is never inferred from versions — each component is an +// explicit declaration by its owner. +// +// One record per sync, written when the write side of the source cache +// enables (before any manifest entry), cleared by compaction folds with +// the manifest. +type SourceCacheCompatRecord struct { + state protoimpl.MessageState `protogen:"hybrid.v1"` + // Fixed singleton id ("compat"); present so the record satisfies the + // table shape. + Id string `protobuf:"bytes,1,opt,name=id,proto3" json:"id,omitempty"` + // Connector-declared components (SourceCacheCapability annotations): + // the connector's cache generation and configuration/permission + // fingerprint. + ConnectorCacheGeneration string `protobuf:"bytes,2,opt,name=connector_cache_generation,json=connectorCacheGeneration,proto3" json:"connector_cache_generation,omitempty"` + ConnectorConfigFingerprint string `protobuf:"bytes,3,opt,name=connector_config_fingerprint,json=connectorConfigFingerprint,proto3" json:"connector_config_fingerprint,omitempty"` + // SDK materialization-policy generation: bumped when the SDK changes + // how response rows or their side effects materialize into the store + // in a replay-visible way (sourcecache.MaterializationPolicyGeneration). + SdkMaterializationGeneration string `protobuf:"bytes,4,opt,name=sdk_materialization_generation,json=sdkMaterializationGeneration,proto3" json:"sdk_materialization_generation,omitempty"` + // SDK sync-selection fingerprint: digest of the sync-shaping inputs + // (enabled resource types, skip flags) — a selection change means the + // prior artifact's scopes cover a different row universe. + SyncSelectionFingerprint string `protobuf:"bytes,5,opt,name=sync_selection_fingerprint,json=syncSelectionFingerprint,proto3" json:"sync_selection_fingerprint,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SourceCacheCompatRecord) Reset() { + *x = SourceCacheCompatRecord{} + mi := &file_c1_storage_v3_records_proto_msgTypes[11] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SourceCacheCompatRecord) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SourceCacheCompatRecord) ProtoMessage() {} + +func (x *SourceCacheCompatRecord) ProtoReflect() protoreflect.Message { + mi := &file_c1_storage_v3_records_proto_msgTypes[11] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *SourceCacheCompatRecord) GetId() string { + if x != nil { + return x.Id + } + return "" +} + +func (x *SourceCacheCompatRecord) GetConnectorCacheGeneration() string { + if x != nil { + return x.ConnectorCacheGeneration + } + return "" +} + +func (x *SourceCacheCompatRecord) GetConnectorConfigFingerprint() string { + if x != nil { + return x.ConnectorConfigFingerprint + } + return "" +} + +func (x *SourceCacheCompatRecord) GetSdkMaterializationGeneration() string { + if x != nil { + return x.SdkMaterializationGeneration + } + return "" +} + +func (x *SourceCacheCompatRecord) GetSyncSelectionFingerprint() string { + if x != nil { + return x.SyncSelectionFingerprint + } + return "" +} + +func (x *SourceCacheCompatRecord) SetId(v string) { + x.Id = v +} + +func (x *SourceCacheCompatRecord) SetConnectorCacheGeneration(v string) { + x.ConnectorCacheGeneration = v +} + +func (x *SourceCacheCompatRecord) SetConnectorConfigFingerprint(v string) { + x.ConnectorConfigFingerprint = v +} + +func (x *SourceCacheCompatRecord) SetSdkMaterializationGeneration(v string) { + x.SdkMaterializationGeneration = v +} + +func (x *SourceCacheCompatRecord) SetSyncSelectionFingerprint(v string) { + x.SyncSelectionFingerprint = v +} + +type SourceCacheCompatRecord_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + // Fixed singleton id ("compat"); present so the record satisfies the + // table shape. + Id string + // Connector-declared components (SourceCacheCapability annotations): + // the connector's cache generation and configuration/permission + // fingerprint. + ConnectorCacheGeneration string + ConnectorConfigFingerprint string + // SDK materialization-policy generation: bumped when the SDK changes + // how response rows or their side effects materialize into the store + // in a replay-visible way (sourcecache.MaterializationPolicyGeneration). + SdkMaterializationGeneration string + // SDK sync-selection fingerprint: digest of the sync-shaping inputs + // (enabled resource types, skip flags) — a selection change means the + // prior artifact's scopes cover a different row universe. + SyncSelectionFingerprint string +} + +func (b0 SourceCacheCompatRecord_builder) Build() *SourceCacheCompatRecord { + m0 := &SourceCacheCompatRecord{} + b, x := &b0, m0 + _, _ = b, x + x.Id = b.Id + x.ConnectorCacheGeneration = b.ConnectorCacheGeneration + x.ConnectorConfigFingerprint = b.ConnectorConfigFingerprint + x.SdkMaterializationGeneration = b.SdkMaterializationGeneration + x.SyncSelectionFingerprint = b.SyncSelectionFingerprint return m0 } @@ -1999,13 +2173,21 @@ const file_c1_storage_v3_records_proto_rawDesc = "" + "\async_id\x18\x01 \x01(\tR\x06syncId\x12\x10\n" + "\x03key\x18\x02 \x01(\tR\x03key\x12\x14\n" + "\x05value\x18\x03 \x01(\fR\x05value:\x1c\x82\xf9+\x18\n" + - "\bsessions\x12\async_id\x12\x03key\"\xeb\x01\n" + + "\bsessions\x12\async_id\x12\x03key\"\x8d\x02\n" + "\x16SourceCacheEntryRecord\x12\x19\n" + "\brow_kind\x18\x01 \x01(\tR\arowKind\x12\x1b\n" + "\tscope_key\x18\x02 \x01(\tR\bscopeKey\x12'\n" + "\x0fcache_validator\x18\x03 \x01(\tR\x0ecacheValidator\x12?\n" + - "\rdiscovered_at\x18\x04 \x01(\v2\x1a.google.protobuf.TimestampR\fdiscoveredAt:/\x82\xf9++\n" + - "\x14source_cache_entries\x12\brow_kind\x12\tscope_key*\xae\x01\n" + + "\rdiscovered_at\x18\x04 \x01(\v2\x1a.google.protobuf.TimestampR\fdiscoveredAt\x12 \n" + + "\vinvalidated\x18\x05 \x01(\bR\vinvalidated:/\x82\xf9++\n" + + "\x14source_cache_entries\x12\brow_kind\x12\tscope_key\"\xcc\x02\n" + + "\x17SourceCacheCompatRecord\x12\x0e\n" + + "\x02id\x18\x01 \x01(\tR\x02id\x12<\n" + + "\x1aconnector_cache_generation\x18\x02 \x01(\tR\x18connectorCacheGeneration\x12@\n" + + "\x1cconnector_config_fingerprint\x18\x03 \x01(\tR\x1aconnectorConfigFingerprint\x12D\n" + + "\x1esdk_materialization_generation\x18\x04 \x01(\tR\x1csdkMaterializationGeneration\x12<\n" + + "\x1async_selection_fingerprint\x18\x05 \x01(\tR\x18syncSelectionFingerprint:\x1d\x82\xf9+\x19\n" + + "\x13source_cache_compat\x12\x02id*\xae\x01\n" + "\bSyncType\x12\x19\n" + "\x15SYNC_TYPE_UNSPECIFIED\x10\x00\x12\x12\n" + "\x0eSYNC_TYPE_FULL\x10\x01\x12\x15\n" + @@ -2015,54 +2197,55 @@ const file_c1_storage_v3_records_proto_rawDesc = "" + "\x1bSYNC_TYPE_PARTIAL_DELETIONS\x10\x05B4Z2github.com/conductorone/baton-sdk/pb/c1/storage/v3b\x06proto3" var file_c1_storage_v3_records_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_c1_storage_v3_records_proto_msgTypes = make([]protoimpl.MessageInfo, 15) +var file_c1_storage_v3_records_proto_msgTypes = make([]protoimpl.MessageInfo, 16) var file_c1_storage_v3_records_proto_goTypes = []any{ - (SyncType)(0), // 0: c1.storage.v3.SyncType - (*GrantExpandableRecord)(nil), // 1: c1.storage.v3.GrantExpandableRecord - (*GrantSourceRecord)(nil), // 2: c1.storage.v3.GrantSourceRecord - (*ResourceTypeRecord)(nil), // 3: c1.storage.v3.ResourceTypeRecord - (*ResourceRecord)(nil), // 4: c1.storage.v3.ResourceRecord - (*EntitlementRecord)(nil), // 5: c1.storage.v3.EntitlementRecord - (*GrantRecord)(nil), // 6: c1.storage.v3.GrantRecord - (*AssetRecord)(nil), // 7: c1.storage.v3.AssetRecord - (*SyncRunRecord)(nil), // 8: c1.storage.v3.SyncRunRecord - (*SyncStatsRecord)(nil), // 9: c1.storage.v3.SyncStatsRecord - (*SessionRecord)(nil), // 10: c1.storage.v3.SessionRecord - (*SourceCacheEntryRecord)(nil), // 11: c1.storage.v3.SourceCacheEntryRecord - nil, // 12: c1.storage.v3.GrantRecord.SourcesEntry - nil, // 13: c1.storage.v3.SyncStatsRecord.ResourcesByResourceTypeEntry - nil, // 14: c1.storage.v3.SyncStatsRecord.GrantsByEntitlementResourceTypeEntry - nil, // 15: c1.storage.v3.SyncStatsRecord.EntitlementsByResourceTypeEntry - (*anypb.Any)(nil), // 16: google.protobuf.Any - (*timestamppb.Timestamp)(nil), // 17: google.protobuf.Timestamp - (*ResourceRef)(nil), // 18: c1.storage.v3.ResourceRef - (*EntitlementRef)(nil), // 19: c1.storage.v3.EntitlementRef - (*PrincipalRef)(nil), // 20: c1.storage.v3.PrincipalRef + (SyncType)(0), // 0: c1.storage.v3.SyncType + (*GrantExpandableRecord)(nil), // 1: c1.storage.v3.GrantExpandableRecord + (*GrantSourceRecord)(nil), // 2: c1.storage.v3.GrantSourceRecord + (*ResourceTypeRecord)(nil), // 3: c1.storage.v3.ResourceTypeRecord + (*ResourceRecord)(nil), // 4: c1.storage.v3.ResourceRecord + (*EntitlementRecord)(nil), // 5: c1.storage.v3.EntitlementRecord + (*GrantRecord)(nil), // 6: c1.storage.v3.GrantRecord + (*AssetRecord)(nil), // 7: c1.storage.v3.AssetRecord + (*SyncRunRecord)(nil), // 8: c1.storage.v3.SyncRunRecord + (*SyncStatsRecord)(nil), // 9: c1.storage.v3.SyncStatsRecord + (*SessionRecord)(nil), // 10: c1.storage.v3.SessionRecord + (*SourceCacheEntryRecord)(nil), // 11: c1.storage.v3.SourceCacheEntryRecord + (*SourceCacheCompatRecord)(nil), // 12: c1.storage.v3.SourceCacheCompatRecord + nil, // 13: c1.storage.v3.GrantRecord.SourcesEntry + nil, // 14: c1.storage.v3.SyncStatsRecord.ResourcesByResourceTypeEntry + nil, // 15: c1.storage.v3.SyncStatsRecord.GrantsByEntitlementResourceTypeEntry + nil, // 16: c1.storage.v3.SyncStatsRecord.EntitlementsByResourceTypeEntry + (*anypb.Any)(nil), // 17: google.protobuf.Any + (*timestamppb.Timestamp)(nil), // 18: google.protobuf.Timestamp + (*ResourceRef)(nil), // 19: c1.storage.v3.ResourceRef + (*EntitlementRef)(nil), // 20: c1.storage.v3.EntitlementRef + (*PrincipalRef)(nil), // 21: c1.storage.v3.PrincipalRef } var file_c1_storage_v3_records_proto_depIdxs = []int32{ - 16, // 0: c1.storage.v3.ResourceTypeRecord.annotations:type_name -> google.protobuf.Any - 17, // 1: c1.storage.v3.ResourceTypeRecord.discovered_at:type_name -> google.protobuf.Timestamp - 18, // 2: c1.storage.v3.ResourceRecord.parent:type_name -> c1.storage.v3.ResourceRef - 16, // 3: c1.storage.v3.ResourceRecord.annotations:type_name -> google.protobuf.Any - 17, // 4: c1.storage.v3.ResourceRecord.discovered_at:type_name -> google.protobuf.Timestamp - 18, // 5: c1.storage.v3.EntitlementRecord.resource:type_name -> c1.storage.v3.ResourceRef - 16, // 6: c1.storage.v3.EntitlementRecord.annotations:type_name -> google.protobuf.Any - 17, // 7: c1.storage.v3.EntitlementRecord.discovered_at:type_name -> google.protobuf.Timestamp - 19, // 8: c1.storage.v3.GrantRecord.entitlement:type_name -> c1.storage.v3.EntitlementRef - 20, // 9: c1.storage.v3.GrantRecord.principal:type_name -> c1.storage.v3.PrincipalRef - 17, // 10: c1.storage.v3.GrantRecord.discovered_at:type_name -> google.protobuf.Timestamp + 17, // 0: c1.storage.v3.ResourceTypeRecord.annotations:type_name -> google.protobuf.Any + 18, // 1: c1.storage.v3.ResourceTypeRecord.discovered_at:type_name -> google.protobuf.Timestamp + 19, // 2: c1.storage.v3.ResourceRecord.parent:type_name -> c1.storage.v3.ResourceRef + 17, // 3: c1.storage.v3.ResourceRecord.annotations:type_name -> google.protobuf.Any + 18, // 4: c1.storage.v3.ResourceRecord.discovered_at:type_name -> google.protobuf.Timestamp + 19, // 5: c1.storage.v3.EntitlementRecord.resource:type_name -> c1.storage.v3.ResourceRef + 17, // 6: c1.storage.v3.EntitlementRecord.annotations:type_name -> google.protobuf.Any + 18, // 7: c1.storage.v3.EntitlementRecord.discovered_at:type_name -> google.protobuf.Timestamp + 20, // 8: c1.storage.v3.GrantRecord.entitlement:type_name -> c1.storage.v3.EntitlementRef + 21, // 9: c1.storage.v3.GrantRecord.principal:type_name -> c1.storage.v3.PrincipalRef + 18, // 10: c1.storage.v3.GrantRecord.discovered_at:type_name -> google.protobuf.Timestamp 1, // 11: c1.storage.v3.GrantRecord.expansion:type_name -> c1.storage.v3.GrantExpandableRecord - 16, // 12: c1.storage.v3.GrantRecord.annotations:type_name -> google.protobuf.Any - 12, // 13: c1.storage.v3.GrantRecord.sources:type_name -> c1.storage.v3.GrantRecord.SourcesEntry - 17, // 14: c1.storage.v3.AssetRecord.discovered_at:type_name -> google.protobuf.Timestamp + 17, // 12: c1.storage.v3.GrantRecord.annotations:type_name -> google.protobuf.Any + 13, // 13: c1.storage.v3.GrantRecord.sources:type_name -> c1.storage.v3.GrantRecord.SourcesEntry + 18, // 14: c1.storage.v3.AssetRecord.discovered_at:type_name -> google.protobuf.Timestamp 0, // 15: c1.storage.v3.SyncRunRecord.type:type_name -> c1.storage.v3.SyncType - 17, // 16: c1.storage.v3.SyncRunRecord.started_at:type_name -> google.protobuf.Timestamp - 17, // 17: c1.storage.v3.SyncRunRecord.ended_at:type_name -> google.protobuf.Timestamp - 13, // 18: c1.storage.v3.SyncStatsRecord.resources_by_resource_type:type_name -> c1.storage.v3.SyncStatsRecord.ResourcesByResourceTypeEntry - 14, // 19: c1.storage.v3.SyncStatsRecord.grants_by_entitlement_resource_type:type_name -> c1.storage.v3.SyncStatsRecord.GrantsByEntitlementResourceTypeEntry - 15, // 20: c1.storage.v3.SyncStatsRecord.entitlements_by_resource_type:type_name -> c1.storage.v3.SyncStatsRecord.EntitlementsByResourceTypeEntry - 17, // 21: c1.storage.v3.SyncStatsRecord.written_at:type_name -> google.protobuf.Timestamp - 17, // 22: c1.storage.v3.SourceCacheEntryRecord.discovered_at:type_name -> google.protobuf.Timestamp + 18, // 16: c1.storage.v3.SyncRunRecord.started_at:type_name -> google.protobuf.Timestamp + 18, // 17: c1.storage.v3.SyncRunRecord.ended_at:type_name -> google.protobuf.Timestamp + 14, // 18: c1.storage.v3.SyncStatsRecord.resources_by_resource_type:type_name -> c1.storage.v3.SyncStatsRecord.ResourcesByResourceTypeEntry + 15, // 19: c1.storage.v3.SyncStatsRecord.grants_by_entitlement_resource_type:type_name -> c1.storage.v3.SyncStatsRecord.GrantsByEntitlementResourceTypeEntry + 16, // 20: c1.storage.v3.SyncStatsRecord.entitlements_by_resource_type:type_name -> c1.storage.v3.SyncStatsRecord.EntitlementsByResourceTypeEntry + 18, // 21: c1.storage.v3.SyncStatsRecord.written_at:type_name -> google.protobuf.Timestamp + 18, // 22: c1.storage.v3.SourceCacheEntryRecord.discovered_at:type_name -> google.protobuf.Timestamp 2, // 23: c1.storage.v3.GrantRecord.SourcesEntry.value:type_name -> c1.storage.v3.GrantSourceRecord 24, // [24:24] is the sub-list for method output_type 24, // [24:24] is the sub-list for method input_type @@ -2084,7 +2267,7 @@ func file_c1_storage_v3_records_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_c1_storage_v3_records_proto_rawDesc), len(file_c1_storage_v3_records_proto_rawDesc)), NumEnums: 1, - NumMessages: 15, + NumMessages: 16, NumExtensions: 0, NumServices: 0, }, diff --git a/pb/c1/storage/v3/records.pb.validate.go b/pb/c1/storage/v3/records.pb.validate.go index bc3827e0a..cd79694c1 100644 --- a/pb/c1/storage/v3/records.pb.validate.go +++ b/pb/c1/storage/v3/records.pb.validate.go @@ -1749,6 +1749,8 @@ func (m *SourceCacheEntryRecord) validate(all bool) error { } } + // no validation rules for Invalidated + if len(errors) > 0 { return SourceCacheEntryRecordMultiError(errors) } @@ -1828,3 +1830,115 @@ var _ interface { Cause() error ErrorName() string } = SourceCacheEntryRecordValidationError{} + +// Validate checks the field values on SourceCacheCompatRecord with the rules +// defined in the proto definition for this message. If any rules are +// violated, the first error encountered is returned, or nil if there are no violations. +func (m *SourceCacheCompatRecord) Validate() error { + return m.validate(false) +} + +// ValidateAll checks the field values on SourceCacheCompatRecord with the +// rules defined in the proto definition for this message. If any rules are +// violated, the result is a list of violation errors wrapped in +// SourceCacheCompatRecordMultiError, or nil if none found. +func (m *SourceCacheCompatRecord) ValidateAll() error { + return m.validate(true) +} + +func (m *SourceCacheCompatRecord) validate(all bool) error { + if m == nil { + return nil + } + + var errors []error + + // no validation rules for Id + + // no validation rules for ConnectorCacheGeneration + + // no validation rules for ConnectorConfigFingerprint + + // no validation rules for SdkMaterializationGeneration + + // no validation rules for SyncSelectionFingerprint + + if len(errors) > 0 { + return SourceCacheCompatRecordMultiError(errors) + } + + return nil +} + +// SourceCacheCompatRecordMultiError is an error wrapping multiple validation +// errors returned by SourceCacheCompatRecord.ValidateAll() if the designated +// constraints aren't met. +type SourceCacheCompatRecordMultiError []error + +// Error returns a concatenation of all the error messages it wraps. +func (m SourceCacheCompatRecordMultiError) Error() string { + msgs := make([]string, 0, len(m)) + for _, err := range m { + msgs = append(msgs, err.Error()) + } + return strings.Join(msgs, "; ") +} + +// AllErrors returns a list of validation violation errors. +func (m SourceCacheCompatRecordMultiError) AllErrors() []error { return m } + +// SourceCacheCompatRecordValidationError is the validation error returned by +// SourceCacheCompatRecord.Validate if the designated constraints aren't met. +type SourceCacheCompatRecordValidationError struct { + field string + reason string + cause error + key bool +} + +// Field function returns field value. +func (e SourceCacheCompatRecordValidationError) Field() string { return e.field } + +// Reason function returns reason value. +func (e SourceCacheCompatRecordValidationError) Reason() string { return e.reason } + +// Cause function returns cause value. +func (e SourceCacheCompatRecordValidationError) Cause() error { return e.cause } + +// Key function returns key value. +func (e SourceCacheCompatRecordValidationError) Key() bool { return e.key } + +// ErrorName returns error name. +func (e SourceCacheCompatRecordValidationError) ErrorName() string { + return "SourceCacheCompatRecordValidationError" +} + +// Error satisfies the builtin error interface +func (e SourceCacheCompatRecordValidationError) Error() string { + cause := "" + if e.cause != nil { + cause = fmt.Sprintf(" | caused by: %v", e.cause) + } + + key := "" + if e.key { + key = "key for " + } + + return fmt.Sprintf( + "invalid %sSourceCacheCompatRecord.%s: %s%s", + key, + e.field, + e.reason, + cause) +} + +var _ error = SourceCacheCompatRecordValidationError{} + +var _ interface { + Field() string + Reason() string + Key() bool + Cause() error + ErrorName() string +} = SourceCacheCompatRecordValidationError{} diff --git a/pb/c1/storage/v3/records_protoopaque.pb.go b/pb/c1/storage/v3/records_protoopaque.pb.go index 28a95a3c5..f4ce080cb 100644 --- a/pb/c1/storage/v3/records_protoopaque.pb.go +++ b/pb/c1/storage/v3/records_protoopaque.pb.go @@ -1712,6 +1712,7 @@ type SourceCacheEntryRecord struct { xxx_hidden_ScopeKey string `protobuf:"bytes,2,opt,name=scope_key,json=scopeKey,proto3"` xxx_hidden_CacheValidator string `protobuf:"bytes,3,opt,name=cache_validator,json=cacheValidator,proto3"` xxx_hidden_DiscoveredAt *timestamppb.Timestamp `protobuf:"bytes,4,opt,name=discovered_at,json=discoveredAt,proto3"` + xxx_hidden_Invalidated bool `protobuf:"varint,5,opt,name=invalidated,proto3"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -1769,6 +1770,13 @@ func (x *SourceCacheEntryRecord) GetDiscoveredAt() *timestamppb.Timestamp { return nil } +func (x *SourceCacheEntryRecord) GetInvalidated() bool { + if x != nil { + return x.xxx_hidden_Invalidated + } + return false +} + func (x *SourceCacheEntryRecord) SetRowKind(v string) { x.xxx_hidden_RowKind = v } @@ -1785,6 +1793,10 @@ func (x *SourceCacheEntryRecord) SetDiscoveredAt(v *timestamppb.Timestamp) { x.xxx_hidden_DiscoveredAt = v } +func (x *SourceCacheEntryRecord) SetInvalidated(v bool) { + x.xxx_hidden_Invalidated = v +} + func (x *SourceCacheEntryRecord) HasDiscoveredAt() bool { if x == nil { return false @@ -1808,6 +1820,14 @@ type SourceCacheEntryRecord_builder struct { // Opaque upstream validator: HTTP ETag, delta token, etc. CacheValidator string DiscoveredAt *timestamppb.Timestamp + // Set when the dangling-reference drops (ingestion invariants I7/I8/I9, + // pkg/sync/ingest_invariants.go) deleted rows stamped with this scope: + // the artifact no longer holds the full row set the validator vouches + // for, so a future sync must not 304-replay it. Lookups treat an + // invalidated entry as a miss (the scope re-fetches cold and converges + // with a cold sync); the entry itself is kept so the scope's surviving + // stamped rows do not read as an I6 orphan (lost manifest write). + Invalidated bool } func (b0 SourceCacheEntryRecord_builder) Build() *SourceCacheEntryRecord { @@ -1818,6 +1838,142 @@ func (b0 SourceCacheEntryRecord_builder) Build() *SourceCacheEntryRecord { x.xxx_hidden_ScopeKey = b.ScopeKey x.xxx_hidden_CacheValidator = b.CacheValidator x.xxx_hidden_DiscoveredAt = b.DiscoveredAt + x.xxx_hidden_Invalidated = b.Invalidated + return m0 +} + +// SourceCacheCompatRecord is the sync's replay-compatibility key: the +// exact conditions under which this artifact's source-cache manifest was +// recorded. A future sync may replay from this artifact ONLY when its +// own key matches byte-exactly on every component; an absent, unreadable, +// or mismatched record degrades that sync to cold (no-op lookup). +// Compatibility is never inferred from versions — each component is an +// explicit declaration by its owner. +// +// One record per sync, written when the write side of the source cache +// enables (before any manifest entry), cleared by compaction folds with +// the manifest. +type SourceCacheCompatRecord struct { + state protoimpl.MessageState `protogen:"opaque.v1"` + xxx_hidden_Id string `protobuf:"bytes,1,opt,name=id,proto3"` + xxx_hidden_ConnectorCacheGeneration string `protobuf:"bytes,2,opt,name=connector_cache_generation,json=connectorCacheGeneration,proto3"` + xxx_hidden_ConnectorConfigFingerprint string `protobuf:"bytes,3,opt,name=connector_config_fingerprint,json=connectorConfigFingerprint,proto3"` + xxx_hidden_SdkMaterializationGeneration string `protobuf:"bytes,4,opt,name=sdk_materialization_generation,json=sdkMaterializationGeneration,proto3"` + xxx_hidden_SyncSelectionFingerprint string `protobuf:"bytes,5,opt,name=sync_selection_fingerprint,json=syncSelectionFingerprint,proto3"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SourceCacheCompatRecord) Reset() { + *x = SourceCacheCompatRecord{} + mi := &file_c1_storage_v3_records_proto_msgTypes[11] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SourceCacheCompatRecord) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SourceCacheCompatRecord) ProtoMessage() {} + +func (x *SourceCacheCompatRecord) ProtoReflect() protoreflect.Message { + mi := &file_c1_storage_v3_records_proto_msgTypes[11] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +func (x *SourceCacheCompatRecord) GetId() string { + if x != nil { + return x.xxx_hidden_Id + } + return "" +} + +func (x *SourceCacheCompatRecord) GetConnectorCacheGeneration() string { + if x != nil { + return x.xxx_hidden_ConnectorCacheGeneration + } + return "" +} + +func (x *SourceCacheCompatRecord) GetConnectorConfigFingerprint() string { + if x != nil { + return x.xxx_hidden_ConnectorConfigFingerprint + } + return "" +} + +func (x *SourceCacheCompatRecord) GetSdkMaterializationGeneration() string { + if x != nil { + return x.xxx_hidden_SdkMaterializationGeneration + } + return "" +} + +func (x *SourceCacheCompatRecord) GetSyncSelectionFingerprint() string { + if x != nil { + return x.xxx_hidden_SyncSelectionFingerprint + } + return "" +} + +func (x *SourceCacheCompatRecord) SetId(v string) { + x.xxx_hidden_Id = v +} + +func (x *SourceCacheCompatRecord) SetConnectorCacheGeneration(v string) { + x.xxx_hidden_ConnectorCacheGeneration = v +} + +func (x *SourceCacheCompatRecord) SetConnectorConfigFingerprint(v string) { + x.xxx_hidden_ConnectorConfigFingerprint = v +} + +func (x *SourceCacheCompatRecord) SetSdkMaterializationGeneration(v string) { + x.xxx_hidden_SdkMaterializationGeneration = v +} + +func (x *SourceCacheCompatRecord) SetSyncSelectionFingerprint(v string) { + x.xxx_hidden_SyncSelectionFingerprint = v +} + +type SourceCacheCompatRecord_builder struct { + _ [0]func() // Prevents comparability and use of unkeyed literals for the builder. + + // Fixed singleton id ("compat"); present so the record satisfies the + // table shape. + Id string + // Connector-declared components (SourceCacheCapability annotations): + // the connector's cache generation and configuration/permission + // fingerprint. + ConnectorCacheGeneration string + ConnectorConfigFingerprint string + // SDK materialization-policy generation: bumped when the SDK changes + // how response rows or their side effects materialize into the store + // in a replay-visible way (sourcecache.MaterializationPolicyGeneration). + SdkMaterializationGeneration string + // SDK sync-selection fingerprint: digest of the sync-shaping inputs + // (enabled resource types, skip flags) — a selection change means the + // prior artifact's scopes cover a different row universe. + SyncSelectionFingerprint string +} + +func (b0 SourceCacheCompatRecord_builder) Build() *SourceCacheCompatRecord { + m0 := &SourceCacheCompatRecord{} + b, x := &b0, m0 + _, _ = b, x + x.xxx_hidden_Id = b.Id + x.xxx_hidden_ConnectorCacheGeneration = b.ConnectorCacheGeneration + x.xxx_hidden_ConnectorConfigFingerprint = b.ConnectorConfigFingerprint + x.xxx_hidden_SdkMaterializationGeneration = b.SdkMaterializationGeneration + x.xxx_hidden_SyncSelectionFingerprint = b.SyncSelectionFingerprint return m0 } @@ -1941,13 +2097,21 @@ const file_c1_storage_v3_records_proto_rawDesc = "" + "\async_id\x18\x01 \x01(\tR\x06syncId\x12\x10\n" + "\x03key\x18\x02 \x01(\tR\x03key\x12\x14\n" + "\x05value\x18\x03 \x01(\fR\x05value:\x1c\x82\xf9+\x18\n" + - "\bsessions\x12\async_id\x12\x03key\"\xeb\x01\n" + + "\bsessions\x12\async_id\x12\x03key\"\x8d\x02\n" + "\x16SourceCacheEntryRecord\x12\x19\n" + "\brow_kind\x18\x01 \x01(\tR\arowKind\x12\x1b\n" + "\tscope_key\x18\x02 \x01(\tR\bscopeKey\x12'\n" + "\x0fcache_validator\x18\x03 \x01(\tR\x0ecacheValidator\x12?\n" + - "\rdiscovered_at\x18\x04 \x01(\v2\x1a.google.protobuf.TimestampR\fdiscoveredAt:/\x82\xf9++\n" + - "\x14source_cache_entries\x12\brow_kind\x12\tscope_key*\xae\x01\n" + + "\rdiscovered_at\x18\x04 \x01(\v2\x1a.google.protobuf.TimestampR\fdiscoveredAt\x12 \n" + + "\vinvalidated\x18\x05 \x01(\bR\vinvalidated:/\x82\xf9++\n" + + "\x14source_cache_entries\x12\brow_kind\x12\tscope_key\"\xcc\x02\n" + + "\x17SourceCacheCompatRecord\x12\x0e\n" + + "\x02id\x18\x01 \x01(\tR\x02id\x12<\n" + + "\x1aconnector_cache_generation\x18\x02 \x01(\tR\x18connectorCacheGeneration\x12@\n" + + "\x1cconnector_config_fingerprint\x18\x03 \x01(\tR\x1aconnectorConfigFingerprint\x12D\n" + + "\x1esdk_materialization_generation\x18\x04 \x01(\tR\x1csdkMaterializationGeneration\x12<\n" + + "\x1async_selection_fingerprint\x18\x05 \x01(\tR\x18syncSelectionFingerprint:\x1d\x82\xf9+\x19\n" + + "\x13source_cache_compat\x12\x02id*\xae\x01\n" + "\bSyncType\x12\x19\n" + "\x15SYNC_TYPE_UNSPECIFIED\x10\x00\x12\x12\n" + "\x0eSYNC_TYPE_FULL\x10\x01\x12\x15\n" + @@ -1957,54 +2121,55 @@ const file_c1_storage_v3_records_proto_rawDesc = "" + "\x1bSYNC_TYPE_PARTIAL_DELETIONS\x10\x05B4Z2github.com/conductorone/baton-sdk/pb/c1/storage/v3b\x06proto3" var file_c1_storage_v3_records_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_c1_storage_v3_records_proto_msgTypes = make([]protoimpl.MessageInfo, 15) +var file_c1_storage_v3_records_proto_msgTypes = make([]protoimpl.MessageInfo, 16) var file_c1_storage_v3_records_proto_goTypes = []any{ - (SyncType)(0), // 0: c1.storage.v3.SyncType - (*GrantExpandableRecord)(nil), // 1: c1.storage.v3.GrantExpandableRecord - (*GrantSourceRecord)(nil), // 2: c1.storage.v3.GrantSourceRecord - (*ResourceTypeRecord)(nil), // 3: c1.storage.v3.ResourceTypeRecord - (*ResourceRecord)(nil), // 4: c1.storage.v3.ResourceRecord - (*EntitlementRecord)(nil), // 5: c1.storage.v3.EntitlementRecord - (*GrantRecord)(nil), // 6: c1.storage.v3.GrantRecord - (*AssetRecord)(nil), // 7: c1.storage.v3.AssetRecord - (*SyncRunRecord)(nil), // 8: c1.storage.v3.SyncRunRecord - (*SyncStatsRecord)(nil), // 9: c1.storage.v3.SyncStatsRecord - (*SessionRecord)(nil), // 10: c1.storage.v3.SessionRecord - (*SourceCacheEntryRecord)(nil), // 11: c1.storage.v3.SourceCacheEntryRecord - nil, // 12: c1.storage.v3.GrantRecord.SourcesEntry - nil, // 13: c1.storage.v3.SyncStatsRecord.ResourcesByResourceTypeEntry - nil, // 14: c1.storage.v3.SyncStatsRecord.GrantsByEntitlementResourceTypeEntry - nil, // 15: c1.storage.v3.SyncStatsRecord.EntitlementsByResourceTypeEntry - (*anypb.Any)(nil), // 16: google.protobuf.Any - (*timestamppb.Timestamp)(nil), // 17: google.protobuf.Timestamp - (*ResourceRef)(nil), // 18: c1.storage.v3.ResourceRef - (*EntitlementRef)(nil), // 19: c1.storage.v3.EntitlementRef - (*PrincipalRef)(nil), // 20: c1.storage.v3.PrincipalRef + (SyncType)(0), // 0: c1.storage.v3.SyncType + (*GrantExpandableRecord)(nil), // 1: c1.storage.v3.GrantExpandableRecord + (*GrantSourceRecord)(nil), // 2: c1.storage.v3.GrantSourceRecord + (*ResourceTypeRecord)(nil), // 3: c1.storage.v3.ResourceTypeRecord + (*ResourceRecord)(nil), // 4: c1.storage.v3.ResourceRecord + (*EntitlementRecord)(nil), // 5: c1.storage.v3.EntitlementRecord + (*GrantRecord)(nil), // 6: c1.storage.v3.GrantRecord + (*AssetRecord)(nil), // 7: c1.storage.v3.AssetRecord + (*SyncRunRecord)(nil), // 8: c1.storage.v3.SyncRunRecord + (*SyncStatsRecord)(nil), // 9: c1.storage.v3.SyncStatsRecord + (*SessionRecord)(nil), // 10: c1.storage.v3.SessionRecord + (*SourceCacheEntryRecord)(nil), // 11: c1.storage.v3.SourceCacheEntryRecord + (*SourceCacheCompatRecord)(nil), // 12: c1.storage.v3.SourceCacheCompatRecord + nil, // 13: c1.storage.v3.GrantRecord.SourcesEntry + nil, // 14: c1.storage.v3.SyncStatsRecord.ResourcesByResourceTypeEntry + nil, // 15: c1.storage.v3.SyncStatsRecord.GrantsByEntitlementResourceTypeEntry + nil, // 16: c1.storage.v3.SyncStatsRecord.EntitlementsByResourceTypeEntry + (*anypb.Any)(nil), // 17: google.protobuf.Any + (*timestamppb.Timestamp)(nil), // 18: google.protobuf.Timestamp + (*ResourceRef)(nil), // 19: c1.storage.v3.ResourceRef + (*EntitlementRef)(nil), // 20: c1.storage.v3.EntitlementRef + (*PrincipalRef)(nil), // 21: c1.storage.v3.PrincipalRef } var file_c1_storage_v3_records_proto_depIdxs = []int32{ - 16, // 0: c1.storage.v3.ResourceTypeRecord.annotations:type_name -> google.protobuf.Any - 17, // 1: c1.storage.v3.ResourceTypeRecord.discovered_at:type_name -> google.protobuf.Timestamp - 18, // 2: c1.storage.v3.ResourceRecord.parent:type_name -> c1.storage.v3.ResourceRef - 16, // 3: c1.storage.v3.ResourceRecord.annotations:type_name -> google.protobuf.Any - 17, // 4: c1.storage.v3.ResourceRecord.discovered_at:type_name -> google.protobuf.Timestamp - 18, // 5: c1.storage.v3.EntitlementRecord.resource:type_name -> c1.storage.v3.ResourceRef - 16, // 6: c1.storage.v3.EntitlementRecord.annotations:type_name -> google.protobuf.Any - 17, // 7: c1.storage.v3.EntitlementRecord.discovered_at:type_name -> google.protobuf.Timestamp - 19, // 8: c1.storage.v3.GrantRecord.entitlement:type_name -> c1.storage.v3.EntitlementRef - 20, // 9: c1.storage.v3.GrantRecord.principal:type_name -> c1.storage.v3.PrincipalRef - 17, // 10: c1.storage.v3.GrantRecord.discovered_at:type_name -> google.protobuf.Timestamp + 17, // 0: c1.storage.v3.ResourceTypeRecord.annotations:type_name -> google.protobuf.Any + 18, // 1: c1.storage.v3.ResourceTypeRecord.discovered_at:type_name -> google.protobuf.Timestamp + 19, // 2: c1.storage.v3.ResourceRecord.parent:type_name -> c1.storage.v3.ResourceRef + 17, // 3: c1.storage.v3.ResourceRecord.annotations:type_name -> google.protobuf.Any + 18, // 4: c1.storage.v3.ResourceRecord.discovered_at:type_name -> google.protobuf.Timestamp + 19, // 5: c1.storage.v3.EntitlementRecord.resource:type_name -> c1.storage.v3.ResourceRef + 17, // 6: c1.storage.v3.EntitlementRecord.annotations:type_name -> google.protobuf.Any + 18, // 7: c1.storage.v3.EntitlementRecord.discovered_at:type_name -> google.protobuf.Timestamp + 20, // 8: c1.storage.v3.GrantRecord.entitlement:type_name -> c1.storage.v3.EntitlementRef + 21, // 9: c1.storage.v3.GrantRecord.principal:type_name -> c1.storage.v3.PrincipalRef + 18, // 10: c1.storage.v3.GrantRecord.discovered_at:type_name -> google.protobuf.Timestamp 1, // 11: c1.storage.v3.GrantRecord.expansion:type_name -> c1.storage.v3.GrantExpandableRecord - 16, // 12: c1.storage.v3.GrantRecord.annotations:type_name -> google.protobuf.Any - 12, // 13: c1.storage.v3.GrantRecord.sources:type_name -> c1.storage.v3.GrantRecord.SourcesEntry - 17, // 14: c1.storage.v3.AssetRecord.discovered_at:type_name -> google.protobuf.Timestamp + 17, // 12: c1.storage.v3.GrantRecord.annotations:type_name -> google.protobuf.Any + 13, // 13: c1.storage.v3.GrantRecord.sources:type_name -> c1.storage.v3.GrantRecord.SourcesEntry + 18, // 14: c1.storage.v3.AssetRecord.discovered_at:type_name -> google.protobuf.Timestamp 0, // 15: c1.storage.v3.SyncRunRecord.type:type_name -> c1.storage.v3.SyncType - 17, // 16: c1.storage.v3.SyncRunRecord.started_at:type_name -> google.protobuf.Timestamp - 17, // 17: c1.storage.v3.SyncRunRecord.ended_at:type_name -> google.protobuf.Timestamp - 13, // 18: c1.storage.v3.SyncStatsRecord.resources_by_resource_type:type_name -> c1.storage.v3.SyncStatsRecord.ResourcesByResourceTypeEntry - 14, // 19: c1.storage.v3.SyncStatsRecord.grants_by_entitlement_resource_type:type_name -> c1.storage.v3.SyncStatsRecord.GrantsByEntitlementResourceTypeEntry - 15, // 20: c1.storage.v3.SyncStatsRecord.entitlements_by_resource_type:type_name -> c1.storage.v3.SyncStatsRecord.EntitlementsByResourceTypeEntry - 17, // 21: c1.storage.v3.SyncStatsRecord.written_at:type_name -> google.protobuf.Timestamp - 17, // 22: c1.storage.v3.SourceCacheEntryRecord.discovered_at:type_name -> google.protobuf.Timestamp + 18, // 16: c1.storage.v3.SyncRunRecord.started_at:type_name -> google.protobuf.Timestamp + 18, // 17: c1.storage.v3.SyncRunRecord.ended_at:type_name -> google.protobuf.Timestamp + 14, // 18: c1.storage.v3.SyncStatsRecord.resources_by_resource_type:type_name -> c1.storage.v3.SyncStatsRecord.ResourcesByResourceTypeEntry + 15, // 19: c1.storage.v3.SyncStatsRecord.grants_by_entitlement_resource_type:type_name -> c1.storage.v3.SyncStatsRecord.GrantsByEntitlementResourceTypeEntry + 16, // 20: c1.storage.v3.SyncStatsRecord.entitlements_by_resource_type:type_name -> c1.storage.v3.SyncStatsRecord.EntitlementsByResourceTypeEntry + 18, // 21: c1.storage.v3.SyncStatsRecord.written_at:type_name -> google.protobuf.Timestamp + 18, // 22: c1.storage.v3.SourceCacheEntryRecord.discovered_at:type_name -> google.protobuf.Timestamp 2, // 23: c1.storage.v3.GrantRecord.SourcesEntry.value:type_name -> c1.storage.v3.GrantSourceRecord 24, // [24:24] is the sub-list for method output_type 24, // [24:24] is the sub-list for method input_type @@ -2026,7 +2191,7 @@ func file_c1_storage_v3_records_proto_init() { GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_c1_storage_v3_records_proto_rawDesc), len(file_c1_storage_v3_records_proto_rawDesc)), NumEnums: 1, - NumMessages: 15, + NumMessages: 16, NumExtensions: 0, NumServices: 0, }, diff --git a/pkg/dotc1z/engine/pebble/source_cache.go b/pkg/dotc1z/engine/pebble/source_cache.go index 5de9a0591..3f450a1db 100644 --- a/pkg/dotc1z/engine/pebble/source_cache.go +++ b/pkg/dotc1z/engine/pebble/source_cache.go @@ -30,7 +30,9 @@ import ( // replayBatchRows bounds how many rows accumulate in one pebble.Batch // before an intermediate commit. Replay of a delta-query collection can // be the whole previous row set, so the batch must not grow unbounded. -const replayBatchRows = 10_000 +// A var (not const) so tests can shrink it to exercise the +// partial-commit failure windows deterministically. +var replayBatchRows = 10_000 // ReplayedParentResource identifies one replayed resource whose // annotations carry ChildResourceType — the syncer must schedule child @@ -880,6 +882,37 @@ func (e *Engine) ReplaySourceCacheGrants(ctx context.Context, prev *Engine, scop if err := e.requireCurrentSync(); err != nil { return err } + // Consumed in a defer keyed on rows whose commit LANDED, not on + // success: a replay that fails after an intermediate commit + // (large scope, error or cancellation mid-copy) has already + // populated the keyspace, and while the failing action unwinds, + // concurrently draining workers can still write — a first + // PutGrantRecords taking the empty-keyspace fast path over + // partially replayed identities would skip index cleanup. + // + // clearedRows counts pre-clear deletes the same way: the + // pre-clear commits intermediate batches too, so a pre-clear + // that fails partway has already mutated the keyspace. + // Registered BEFORE the pre-clear so its partial-failure path is + // covered. + // + // sawRelatedResource: InsertResourceGrants copies write resource + // rows too, so the resources empty-keyspace fast path must also + // be disarmed. Related copies always ride a batch that carries + // at least their own grant row, so committedRows > 0 covers + // "any batch that may hold a resource write landed". + committedRows := 0 + clearedRows := int64(0) + sawRelatedResource := false + defer func() { + if committedRows > 0 || clearedRows > 0 { + _ = e.takeFreshGrantsEmpty() + if sawRelatedResource { + _ = e.takeFreshResourcesEmpty() + } + } + }() + // Resume idempotency: drop any destination rows a crashed earlier // attempt already committed under this scope (replay copy and/or // overlay rows), so the copy below is the scope's first writer @@ -893,6 +926,7 @@ func (e *Engine) ReplaySourceCacheGrants(ctx context.Context, prev *Engine, scop return e.deleteGrantIndexesRaw(batch, "", val) }, ) + clearedRows = cleared if err != nil { return fmt.Errorf("source cache replay: pre-clear grants scope %q: %w", scopeKey, err) } @@ -914,30 +948,6 @@ func (e *Engine) ReplaySourceCacheGrants(ctx context.Context, prev *Engine, scop batch := e.db.NewBatch() defer func() { _ = batch.Close() }() rowsInBatch := 0 - - // Consumed in a defer keyed on rows whose commit LANDED, not on - // success: a replay that fails after an intermediate commit - // (large scope, error or cancellation mid-copy) has already - // populated the keyspace, and while the failing action unwinds, - // concurrently draining workers can still write — a first - // PutGrantRecords taking the empty-keyspace fast path over - // partially replayed identities would skip index cleanup. - // - // sawRelatedResource: InsertResourceGrants copies write resource - // rows too, so the resources empty-keyspace fast path must also - // be disarmed. Related copies always ride a batch that carries - // at least their own grant row, so committedRows > 0 covers - // "any batch that may hold a resource write landed". - committedRows := 0 - sawRelatedResource := false - defer func() { - if committedRows > 0 { - _ = e.takeFreshGrantsEmpty() - if sawRelatedResource { - _ = e.takeFreshResourcesEmpty() - } - } - }() // relatedCopied dedupes InsertResourceGrants resource copies: // many grants in one scope typically reference few resources. var relatedCopied map[string]struct{} @@ -1192,9 +1202,28 @@ func (e *Engine) ReplaySourceCacheEntitlements(ctx context.Context, prev *Engine if err := e.requireCurrentSync(); err != nil { return err } + // Keyed on rows whose commit LANDED, not on success (see the + // grants replay): a partial replay has already mutated the + // entitlement keyspace, so the bare-id lookup map must be + // invalidated and the empty-keyspace fast path disarmed even when + // the replay itself fails — draining workers can still resolve + // tombstones and write entitlements while the failure unwinds. + // Pre-clear deletes mutate the keyspace the same way, so they + // arm the invalidation too — registered BEFORE the pre-clear so + // a pre-clear that commits a batch and then fails is covered + // (clearedRows is assigned even on the error path). + committedRows := 0 + clearedRows := int64(0) + defer func() { + if committedRows > 0 || clearedRows > 0 { + e.noteEntitlementKeyspaceWrite() + _ = e.takeFreshEntitlementsEmpty() + } + }() + // Resume idempotency: see the grants replay. The entitlement // bare-id lookup map is invalidated for the deletes exactly as - // for the copies (the defer below keys on either landing). + // for the copies (the defer above keys on either landing). cleared, err := e.clearReplayScopeRowsLocked(ctx, scopeKey, prefix, primaryHeader, scanEntitlementSourceScopeRaw, func(batch *pebble.Batch, priKey []byte, val []byte) error { @@ -1203,6 +1232,7 @@ func (e *Engine) ReplaySourceCacheEntitlements(ctx context.Context, prev *Engine return nil }, ) + clearedRows = cleared if err != nil { return fmt.Errorf("source cache replay: pre-clear entitlements scope %q: %w", scopeKey, err) } @@ -1225,22 +1255,6 @@ func (e *Engine) ReplaySourceCacheEntitlements(ctx context.Context, prev *Engine defer func() { _ = batch.Close() }() rowsInBatch := 0 - // Keyed on rows whose commit LANDED, not on success (see the - // grants replay): a partial replay has already mutated the - // entitlement keyspace, so the bare-id lookup map must be - // invalidated and the empty-keyspace fast path disarmed even when - // the replay itself fails — draining workers can still resolve - // tombstones and write entitlements while the failure unwinds. - // Pre-clear deletes mutate the keyspace the same way, so they - // arm the invalidation too. - committedRows := 0 - defer func() { - if committedRows > 0 || res.ResumedRowsCleared > 0 { - e.noteEntitlementKeyspaceWrite() - _ = e.takeFreshEntitlementsEmpty() - } - }() - for iter.First(); iter.Valid(); iter.Next() { if err := ctx.Err(); err != nil { return err @@ -1353,6 +1367,18 @@ func (e *Engine) ReplaySourceCacheResources(ctx context.Context, prev *Engine, s if err := e.requireCurrentSync(); err != nil { return err } + // Keyed on rows whose commit LANDED, not on success — see the + // grants replay for rationale. Registered BEFORE the pre-clear + // and keyed on its deletes too (clearedRows is assigned even on + // the error path). + committedRows := 0 + clearedRows := int64(0) + defer func() { + if committedRows > 0 || clearedRows > 0 { + _ = e.takeFreshResourcesEmpty() + } + }() + // Resume idempotency: see the grants replay. cleared, err := e.clearReplayScopeRowsLocked(ctx, scopeKey, prefix, primaryHeader, func(val []byte) (string, error) { @@ -1367,6 +1393,7 @@ func (e *Engine) ReplaySourceCacheResources(ctx context.Context, prev *Engine, s return e.deleteResourceIndexesRaw(batch, rt, rid, val) }, ) + clearedRows = cleared if err != nil { return fmt.Errorf("source cache replay: pre-clear resources scope %q: %w", scopeKey, err) } @@ -1389,15 +1416,6 @@ func (e *Engine) ReplaySourceCacheResources(ctx context.Context, prev *Engine, s defer func() { _ = batch.Close() }() rowsInBatch := 0 - // Keyed on rows whose commit LANDED, not on success — see the - // grants replay for rationale. - committedRows := 0 - defer func() { - if committedRows > 0 { - _ = e.takeFreshResourcesEmpty() - } - }() - for iter.First(); iter.Valid(); iter.Next() { if err := ctx.Err(); err != nil { return err diff --git a/pkg/dotc1z/engine/pebble/source_cache_preclear_test.go b/pkg/dotc1z/engine/pebble/source_cache_preclear_test.go new file mode 100644 index 000000000..79b7b98a5 --- /dev/null +++ b/pkg/dotc1z/engine/pebble/source_cache_preclear_test.go @@ -0,0 +1,166 @@ +package pebble + +// Pins the pre-clear failure window of the replay functions: the +// resume-idempotency pre-clear commits intermediate batches, so a +// pre-clear that fails partway (error or cancellation) has already +// mutated the keyspace — the empty-keyspace fast-path flags must be +// disarmed and (for entitlements) the bare-id lookup map invalidated +// even though the replay itself returned an error. Before the fix, the +// entitlement invalidation defer was registered AFTER the pre-clear's +// error return, and the grants/resources defers ignored pre-clear +// deletes entirely. + +import ( + "context" + "fmt" + "testing" + + "github.com/stretchr/testify/require" + + v3 "github.com/conductorone/baton-sdk/pb/c1/storage/v3" + "github.com/conductorone/baton-sdk/pkg/connectorstore" +) + +// failAfterNChecks is a context whose Err() reports Canceled after the +// first n calls succeed — a deterministic mid-loop cancellation for +// paths that poll ctx.Err() once per iteration. +type failAfterNChecks struct { + context.Context + remaining int +} + +func (c *failAfterNChecks) Err() error { + if c.remaining <= 0 { + return context.Canceled + } + c.remaining-- + return nil +} + +// plantScopedEntitlement writes a scope-stamped entitlement row and its +// scope index key DIRECTLY into the db, bypassing PutEntitlementRecords — +// simulating rows a crashed earlier replay attempt committed, without +// consuming the fresh-empty flag the way a live writer would. +func plantScopedEntitlement(t *testing.T, e *Engine, scope, rid string) { + t.Helper() + rec := v3.EntitlementRecord_builder{ + ExternalId: "group:" + rid + ":member", + Resource: v3.ResourceRef_builder{ResourceTypeId: "group", ResourceId: rid}.Build(), + SourceScopeKey: scope, + }.Build() + id, err := entitlementIdentityFromRecord(rec) + require.NoError(t, err) + val, err := marshalRecord(rec) + require.NoError(t, err) + require.NoError(t, e.db.Set(encodeEntitlementIdentityKey(id), val, nil)) + require.NoError(t, e.db.Set(encodeEntitlementBySourceScopeIndexKey(scope, id), nil, nil)) +} + +func plantScopedGrant(t *testing.T, e *Engine, scope, principalID string) { + t.Helper() + rec := v3.GrantRecord_builder{ + ExternalId: "group:g1:member:user:" + principalID, + Entitlement: v3.EntitlementRef_builder{ + ResourceTypeId: "group", ResourceId: "g1", EntitlementId: "group:g1:member", + }.Build(), + Principal: v3.PrincipalRef_builder{ResourceTypeId: "user", ResourceId: principalID}.Build(), + SourceScopeKey: scope, + }.Build() + id, err := grantIdentityFromRecord(rec) + require.NoError(t, err) + val, err := marshalRecord(rec) + require.NoError(t, err) + require.NoError(t, e.db.Set(encodeGrantIdentityKey(id), val, nil)) + require.NoError(t, e.db.Set(encodeGrantBySourceScopeIndexKey(scope, id), nil, nil)) +} + +func plantScopedResource(t *testing.T, e *Engine, scope, rid string) { + t.Helper() + rec := v3.ResourceRecord_builder{ + ResourceTypeId: "user", + ResourceId: rid, + SourceScopeKey: scope, + }.Build() + val, err := marshalRecord(rec) + require.NoError(t, err) + require.NoError(t, e.db.Set(encodeResourceKey("user", rid), val, nil)) + require.NoError(t, e.db.Set(encodeResourceBySourceScopeIndexKey(scope, "user", rid), nil, nil)) +} + +func TestReplayPreClearPartialFailureDisarmsFastPaths(t *testing.T) { + ctx := context.Background() + old := replayBatchRows + replayBatchRows = 1 + t.Cleanup(func() { replayBatchRows = old }) + + scope := "scope-preclear-test" + + // One shared, empty previous file: the copy loop has nothing to do; + // everything under test happens in the pre-clear. + prev := newAdapter(t) + _, err := prev.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + + t.Run("entitlements", func(t *testing.T) { + cur := newAdapter(t) + _, err := cur.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + e := cur.PebbleEngine() + for i := 0; i < 3; i++ { + plantScopedEntitlement(t, e, scope, fmt.Sprintf("g%d", i)) + } + // One iteration succeeds (its single-row batch commits), the + // second sees the cancellation: a partial pre-clear. + cctx := &failAfterNChecks{Context: ctx, remaining: 1} + _, err = e.ReplaySourceCacheEntitlements(cctx, prev.PebbleEngine(), scope) + require.Error(t, err, "the partial pre-clear must surface its failure") + require.False(t, e.takeFreshEntitlementsEmpty(), + "a pre-clear that committed deletes mutated the keyspace; the empty-keyspace fast path must be disarmed even on failure") + }) + + t.Run("grants", func(t *testing.T) { + cur := newAdapter(t) + _, err := cur.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + e := cur.PebbleEngine() + for i := 0; i < 3; i++ { + plantScopedGrant(t, e, scope, fmt.Sprintf("u%d", i)) + } + cctx := &failAfterNChecks{Context: ctx, remaining: 1} + _, err = e.ReplaySourceCacheGrants(cctx, prev.PebbleEngine(), scope) + require.Error(t, err) + require.False(t, e.takeFreshGrantsEmpty(), + "grants pre-clear deletes must disarm the fast path even on failure") + }) + + t.Run("resources", func(t *testing.T) { + cur := newAdapter(t) + _, err := cur.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + e := cur.PebbleEngine() + for i := 0; i < 3; i++ { + plantScopedResource(t, e, scope, fmt.Sprintf("u%d", i)) + } + cctx := &failAfterNChecks{Context: ctx, remaining: 1} + _, err = e.ReplaySourceCacheResources(cctx, prev.PebbleEngine(), scope) + require.Error(t, err) + require.False(t, e.takeFreshResourcesEmpty(), + "resources pre-clear deletes must disarm the fast path even on failure") + }) + + // Control: a replay that fails BEFORE anything commits leaves the + // fast path armed — the disarm is keyed on landed mutations, not on + // the mere attempt. + t.Run("no mutation, no disarm", func(t *testing.T) { + cur := newAdapter(t) + _, err := cur.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + e := cur.PebbleEngine() + // Nothing planted: pre-clear iterates zero rows, copy loop + // copies zero rows (prev is empty), no commits with content. + _, err = e.ReplaySourceCacheEntitlements(ctx, prev.PebbleEngine(), scope) + require.NoError(t, err) + require.True(t, e.takeFreshEntitlementsEmpty(), + "an empty replay must not burn the first-put fast path") + }) +} diff --git a/pkg/dotc1z/source_cache.go b/pkg/dotc1z/source_cache.go index 042b4a5cd..666c2411f 100644 --- a/pkg/dotc1z/source_cache.go +++ b/pkg/dotc1z/source_cache.go @@ -92,6 +92,11 @@ type SourceCacheInspector interface { SourceCacheOrphanScopes(ctx context.Context) (map[string][]string, error) // LookupSourceCacheEntry: see SourceCacheStore. LookupSourceCacheEntry(ctx context.Context, kind sourcecache.RowKind, scopeKey string) (sourcecache.Entry, bool, error) + // GetSourceCacheCompat: see SourceCacheStore. Tooling uses key + // PRESENCE (the offline-decidable half of the compat gate — a key's + // exact match depends on the next run's connector declarations, which + // no offline audit can know; an ABSENT key can never match anything). + GetSourceCacheCompat(ctx context.Context) (sourcecache.CompatKey, bool, error) } var _ SourceCacheInspector = (*pebbleStore)(nil) diff --git a/pkg/sync/expand/drop_stats_test.go b/pkg/sync/expand/drop_stats_test.go index e7cde6252..9cee3fa8c 100644 --- a/pkg/sync/expand/drop_stats_test.go +++ b/pkg/sync/expand/drop_stats_test.go @@ -9,6 +9,7 @@ package expand // exists to report. import ( + "context" "testing" "github.com/stretchr/testify/require" @@ -46,3 +47,35 @@ func TestDroppedEdgeStatsExamplesDedupeAcrossRepeats(t *testing.T) { require.Equal(t, int64(6), s.sourceMissing, "repeat drops of one id still count their edges") require.Equal(t, []string{"ent-repeat"}, s.examples, "examples list one entry per distinct id") } + +// TestTopologicalSourceMissingRecordedOnDropStats pins the topological +// reduce's SOURCE-missing path onto the aggregate: a graph edge whose +// source entitlement has no row must complete (drop-don't-fail), be +// counted on DroppedEdgeStats, and not fail the run — the migration +// that moved the destination-side to the aggregate had left this path +// as a per-edge Warn invisible to LogSummary. +func TestTopologicalSourceMissingRecordedOnDropStats(t *testing.T) { + ctx := context.Background() + store := NewMockExpanderStore() + + group := makeResource("group", "org") + source := makeEntitlement("ent:source", group) + dest := makeEntitlement("ent:dest", group) + // The DESTINATION row exists; the SOURCE row deliberately does not. + store.AddEntitlement(dest) + + graph := NewEntitlementGraph(ctx) + graph.AddEntitlementID(source.GetId()) + graph.AddEntitlementID(dest.GetId()) + require.NoError(t, graph.AddEdge(ctx, source.GetId(), dest.GetId(), false, nil)) + + e := NewExpander(store, graph) + stats := &DroppedEdgeStats{} + e.SetDropStats(stats) + require.NoError(t, e.RunTopologicalMergeStreaming(ctx), "a missing source must complete, not error") + + stats.mu.Lock() + defer stats.mu.Unlock() + require.Equal(t, int64(1), stats.sourceMissing, "the dropped source edge must land on the aggregate") + require.Contains(t, stats.seen, source.GetId()) +} diff --git a/pkg/sync/expand/topological_merge_projection.go b/pkg/sync/expand/topological_merge_projection.go index d2081d935..a8cf4b05a 100644 --- a/pkg/sync/expand/topological_merge_projection.go +++ b/pkg/sync/expand/topological_merge_projection.go @@ -162,9 +162,13 @@ func (e *Expander) mergeDestinationStreams( for _, sourceID := range sortedCopy(sourceNode.EntitlementIDs) { sourceEntitlement := entitlements[sourceID] if sourceEntitlement == nil { - // Drop-don't-fail, but loudly (see the destination-side - // warning in driveTopologicalLayer). - ctxzap.Extract(ctx).Warn("topological expansion: source entitlement not in store; its contributions are skipped", + // Drop-don't-fail, recorded on the sync-wide aggregate + // (one warning per sync — see DroppedEdgeStats and the + // destination-side twin in driveTopologicalLayer); + // per-edge logging stays at Debug so a large dangling + // family can't flood the logs. + e.dropStats.RecordSourceMissing(sourceID) + ctxzap.Extract(ctx).Debug("topological expansion: source entitlement not in store; its contributions are skipped", zap.String("entitlement_id", sourceID)) continue } diff --git a/pkg/sync/ingest_invariants.go b/pkg/sync/ingest_invariants.go index f66715c3e..faad7d4ba 100644 --- a/pkg/sync/ingest_invariants.go +++ b/pkg/sync/ingest_invariants.go @@ -70,19 +70,30 @@ package sync //nolint:revive,nolintlint // we can't change the package name for // are evidence that no external resource file was configured, not // bad data (the match op deletes carriers when it runs). // -// I7/I8/I9 violations are connector-attributable bad data. The -// platform's ingestion provably discards such rows (grants that resolve -// no principal or entitlement are skipped at uplift; grants under a -// missing entitlement are never even read), so by DEFAULT the -// syncer DROPS the rows and emits one aggregated warning per invariant -// (total dropped, distinct dangling refs, capped id examples — never -// per-row logs; the expansion path's per-edge warnings produce millions -// of lines a week in production). FAIL-FAST mode (tests, the harness) -// hard-fails so engineered violations are named. Deliberately NOT -// ErrReplayIntegrity: a dishonest validator makes the warm run fail but -// the cold run CLEAN, so the cold-retry ladder would never re-attribute -// to the connector and every sync would pay warm-fail-plus-retry -// forever. +// I7/I8/I9 verdicts split on whether the referent's resource TYPE was +// synced (docs/tasks/dangling-reference-drops.md): +// +// - DISABLED-TYPE danglings (type never synced — a config gap) are +// DROPPED with one aggregated warning per invariant (total dropped, +// distinct dangling refs, capped id examples — never per-row logs; +// the expansion path's per-edge warnings produce millions of lines +// a week in production). The platform's ingestion provably discards +// such rows anyway. The DROP arm is deliberately NOT +// ErrReplayIntegrity: a dishonest validator makes the warm run fail +// but the cold run CLEAN, so the cold-retry ladder would never +// re-attribute to the connector and every sync would pay +// warm-fail-plus-retry forever. +// - ENABLED-TYPE danglings (type IS synced) FAIL the sync — never +// sealed away. A WARM sync's failure wraps ErrReplayIntegrity so +// the runners discard the output and retry cold; a COLD sync with +// the same evidence fails plainly, attributed to the connector +// (see unexpectedDanglingError). The compaction expand pass +// (onlyExpandGrants) is exempt from the FAIL arm: keep-newer merges +// manufacture dangling refs by design, and its drops are attributed +// separately in the warning. +// +// FAIL-FAST mode (tests, the harness) hard-fails every dangling so +// engineered violations are named. // // Evidence is monotone (existence bits, set-union scheduling records), // so parallel workers and mixed stream/replay arrival orders yield @@ -98,6 +109,8 @@ import ( "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" "go.uber.org/zap" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" reader_v2 "github.com/conductorone/baton-sdk/pb/c1/reader/v2" @@ -504,19 +517,28 @@ type danglingTypeProbe struct { known map[string]bool } -func (p *danglingTypeProbe) typeExists(ctx context.Context, resourceTypeID string) bool { +func (p *danglingTypeProbe) typeExists(ctx context.Context, resourceTypeID string) (bool, error) { if v, ok := p.known[resourceTypeID]; ok { - return v - } - if p.known == nil { - p.known = map[string]bool{} + return v, nil } _, err := p.s.store.GetResourceType(ctx, reader_v2.ResourceTypesReaderServiceGetResourceTypeRequest_builder{ ResourceTypeId: resourceTypeID, }.Build()) + if err != nil && status.Code(err) != codes.NotFound { + // A read failure is NOT "type never synced": that verdict picks + // the DROP arm, so an IO error or canceled context here would + // seal a sanitized artifact where policy demands a loud failure. + // Propagate, and memoize nothing — only definitive answers may + // be cached (a memoized transient error would poison every later + // reference to the type). + return false, fmt.Errorf("resource-type probe for %q: %w", resourceTypeID, err) + } + if p.known == nil { + p.known = map[string]bool{} + } exists := err == nil p.known[resourceTypeID] = exists - return exists + return exists, nil } // unexpectedDanglingError renders the ENABLED-TYPE dangling verdict of @@ -542,25 +564,31 @@ func (s *syncer) unexpectedDanglingError(format string, args ...any) error { return err } -// danglingAttribution renders the config-gap vs bug split for the -// aggregated warnings. -func danglingAttribution(unsyncedTypeCount, missingRowCount int) string { +// danglingAttribution renders the attribution split for the aggregated +// drop warnings. The drop arm only ever sees two shapes: references +// into never-synced types (config gaps, any mode) and enabled-type +// references reached under the compaction expand pass (keep-newer +// merges manufacture dangling refs no input contained — by design, not +// a connector bug; enabled-type danglings in a NORMAL sync take the +// FAIL arm and never reach a warning). +func danglingAttribution(unsyncedTypeCount, mergeManufacturedCount int) string { switch { - case unsyncedTypeCount > 0 && missingRowCount == 0: + case mergeManufacturedCount == 0: return "referenced resource type(s) never synced — likely disabled on the connector (config gap), not a code bug" - case unsyncedTypeCount == 0 && missingRowCount > 0: - return "referenced type(s) ARE synced but the rows are missing — check the connector's magic-id construction" + case unsyncedTypeCount == 0: + return "references target SYNCED types, dropped during the compaction expand pass — keep-newer merges manufacture these by design; not a connector bug" default: - return "mixed: some references target never-synced types (config gap), some target missing rows in synced types (magic-id bug)" + return "mixed: some references target never-synced types (config gap), some are compaction-merge-manufactured (expand pass)" } } // checkEntitlementResourceReferences is invariant I7: every distinct // resource referenced by an entitlement row must exist as a resource // row. One seek per distinct resource among entitlements — bounded by -// the entitlement count. Default mode DROPS the dangling entitlement -// rows (uplift skips them anyway — see the header) and warns once with -// aggregates; fail-fast mode hard-fails. Never ErrReplayIntegrity. +// the entitlement count. Default mode DROPS disabled-type danglings +// (uplift skips them anyway — see the header) and warns once with +// aggregates; enabled-type danglings FAIL (warm → ErrReplayIntegrity; +// see unexpectedDanglingError); fail-fast mode hard-fails everything. func (s *syncer) checkEntitlementResourceReferences(ctx context.Context) error { if s.syncType != connectorstore.SyncTypeFull { return nil @@ -570,7 +598,7 @@ func (s *syncer) checkEntitlementResourceReferences(ctx context.Context) error { return nil // engine without the inspection surface } probe := &danglingTypeProbe{s: s} - danglingResources, unsyncedType, missingRow := 0, 0, 0 + danglingResources, unsyncedType, mergeManufactured := 0, 0, 0 var droppedRows int64 var resourceExamples, entIDExamples []string err := facts.ForEachDistinctEntitlementResource(ctx, func(rt, rid string) error { @@ -581,13 +609,17 @@ func (s *syncer) checkEntitlementResourceReferences(ctx context.Context) error { if exists { return nil } + typeSynced, probeErr := probe.typeExists(ctx, rt) + if probeErr != nil { + return fmt.Errorf("ingest invariant I7: %w", probeErr) + } if s.failFastInvariants { return fmt.Errorf( "ingest invariant I7 violated: entitlements reference resource %s/%s but no resource row exists "+ "(resource type synced: %t — false means a config gap, true a magic-id construction bug)", - rt, rid, probe.typeExists(ctx, rt)) + rt, rid, typeSynced) } - if !s.onlyExpandGrants && probe.typeExists(ctx, rt) { + if !s.onlyExpandGrants && typeSynced { // Enabled-type dangling: the referent's type IS synced, so the // missing row is unexpected — never sealed away (replay // policy; see unexpectedDanglingError). The compaction @@ -604,8 +636,11 @@ func (s *syncer) checkEntitlementResourceReferences(ctx context.Context) error { } danglingResources++ droppedRows += n - if probe.typeExists(ctx, rt) { - missingRow++ + if typeSynced { + // Reachable only under onlyExpandGrants (a normal sync FAILs + // enabled-type danglings above): a keep-newer merge + // manufactured this gap, not a connector bug. + mergeManufactured++ } else { unsyncedType++ } @@ -623,8 +658,8 @@ func (s *syncer) checkEntitlementResourceReferences(ctx context.Context) error { zap.Int64("dropped_entitlements", droppedRows), zap.Int("dangling_resources", danglingResources), zap.Int("refs_into_unsynced_types", unsyncedType), - zap.Int("refs_missing_rows_in_synced_types", missingRow), - zap.String("attribution", danglingAttribution(unsyncedType, missingRow)), + zap.Int("refs_manufactured_by_compaction_merge", mergeManufactured), + zap.String("attribution", danglingAttribution(unsyncedType, mergeManufactured)), zap.Strings("resource_examples", resourceExamples), zap.Strings("entitlement_id_examples", entIDExamples), ) @@ -648,10 +683,11 @@ func (s *syncer) checkEntitlementResourceReferences(ctx context.Context) error { // resource- or entitlement-level exemption would let one IRG grant // launder every dangling reference that shares its resource. // -// Default mode DROPS the dangling grants (uplift reads grants BY +// Default mode DROPS disabled-type danglings (uplift reads grants BY // entitlement, so rows under a missing entitlement are never even read -// platform-side) and warns once with aggregates; fail-fast mode -// hard-fails. Never ErrReplayIntegrity (see the header). +// platform-side) and warns once with aggregates; enabled-type danglings +// FAIL (warm → ErrReplayIntegrity; see unexpectedDanglingError and the +// header); fail-fast mode hard-fails everything. func (s *syncer) checkGrantEntitlementReferences(ctx context.Context) error { if s.syncType != connectorstore.SyncTypeFull { return nil @@ -661,10 +697,14 @@ func (s *syncer) checkGrantEntitlementReferences(ctx context.Context) error { return nil // engine without the inspection surface } probe := &danglingTypeProbe{s: s} - danglingEnts, unsyncedType, missingRow := 0, 0, 0 + danglingEnts, unsyncedType, mergeManufactured := 0, 0, 0 var droppedRows, insertFactKept int64 var entIDExamples []string err := facts.ForEachDanglingGrantEntitlement(ctx, func(entitlementID, rt, rid string) error { + typeSynced, probeErr := probe.typeExists(ctx, rt) + if probeErr != nil { + return fmt.Errorf("ingest invariant I8: %w", probeErr) + } if s.failFastInvariants { allCarry, err := facts.GrantsForEntitlementAllCarryInsertFact(ctx, entitlementID, rt, rid) if err != nil { @@ -677,9 +717,9 @@ func (s *syncer) checkGrantEntitlementReferences(ctx context.Context) error { "ingest invariant I8 violated: grants without InsertResourceGrants reference entitlement %q (resource %s/%s) but no entitlement row exists "+ "(resource type synced: %t — false means a config gap, true a magic-id construction bug "+ "or a grants scope replayed against a refreshed entitlement set)", - entitlementID, rt, rid, probe.typeExists(ctx, rt)) + entitlementID, rt, rid, typeSynced) } - if !s.onlyExpandGrants && probe.typeExists(ctx, rt) { + if !s.onlyExpandGrants && typeSynced { // Enabled-type dangling (replay policy; see I7's twin arm). // The per-grant IRG exemption still applies: an entitlement // whose grants ALL carry InsertResourceGrants is the @@ -706,8 +746,9 @@ func (s *syncer) checkGrantEntitlementReferences(ctx context.Context) error { } danglingEnts++ droppedRows += n - if probe.typeExists(ctx, rt) { - missingRow++ + if typeSynced { + // Only reachable under onlyExpandGrants — see I7's twin arm. + mergeManufactured++ } else { unsyncedType++ } @@ -725,8 +766,8 @@ func (s *syncer) checkGrantEntitlementReferences(ctx context.Context) error { zap.Int("dangling_entitlements", danglingEnts), zap.Int64("insert_resource_grants_kept", insertFactKept), zap.Int("refs_into_unsynced_types", unsyncedType), - zap.Int("refs_missing_rows_in_synced_types", missingRow), - zap.String("attribution", danglingAttribution(unsyncedType, missingRow)), + zap.Int("refs_manufactured_by_compaction_merge", mergeManufactured), + zap.String("attribution", danglingAttribution(unsyncedType, mergeManufactured)), zap.Strings("entitlement_id_examples", entIDExamples), ) } @@ -746,9 +787,10 @@ func (s *syncer) checkGrantEntitlementReferences(ctx context.Context) error { // are evidence of a config gap, not bad data, and dropping them would // make a misconfigured deployment look clean. // -// Default mode DROPS the dangling grants (uplift skips grants whose +// Default mode DROPS disabled-type danglings (uplift skips grants whose // principal resolves no platform object) and warns once with -// aggregates; fail-fast mode hard-fails. Never ErrReplayIntegrity. +// aggregates; enabled-type danglings FAIL (warm → ErrReplayIntegrity; +// see unexpectedDanglingError); fail-fast mode hard-fails everything. func (s *syncer) checkGrantPrincipalReferences(ctx context.Context) error { if s.syncType != connectorstore.SyncTypeFull { return nil @@ -761,7 +803,7 @@ func (s *syncer) checkGrantPrincipalReferences(ctx context.Context) error { return fmt.Errorf("ingest invariant I9: ensuring grant indexes: %w", err) } probe := &danglingTypeProbe{s: s} - danglingPrincipals, unsyncedType, missingRow := 0, 0, 0 + danglingPrincipals, unsyncedType, mergeManufactured := 0, 0, 0 var droppedRows, matchCarriersKept int64 var principalExamples []string // matchCarriersKept counts GRANTS uniformly: fully-annotated @@ -772,14 +814,18 @@ func (s *syncer) checkGrantPrincipalReferences(ctx context.Context) error { matchCarriersKept += carrierGrants return nil } + typeSynced, probeErr := probe.typeExists(ctx, rt) + if probeErr != nil { + return fmt.Errorf("ingest invariant I9: %w", probeErr) + } if s.failFastInvariants { return fmt.Errorf( "ingest invariant I9 violated: grants reference principal %s/%s but no resource row exists "+ "(resource type synced: %t — false means a config gap, true means the connector "+ "emitted grants for a principal it never synced)", - rt, rid, probe.typeExists(ctx, rt)) + rt, rid, typeSynced) } - if !s.onlyExpandGrants && probe.typeExists(ctx, rt) { + if !s.onlyExpandGrants && typeSynced { // Enabled-type dangling (replay policy; see I7's twin arm). // Mixed populations land here too: exempt carriers were // already excluded above (matchAnnotatedOnly), so a plain @@ -796,8 +842,9 @@ func (s *syncer) checkGrantPrincipalReferences(ctx context.Context) error { danglingPrincipals++ droppedRows += deleted matchCarriersKept += skipped - if probe.typeExists(ctx, rt) { - missingRow++ + if typeSynced { + // Only reachable under onlyExpandGrants — see I7's twin arm. + mergeManufactured++ } else { unsyncedType++ } @@ -815,8 +862,8 @@ func (s *syncer) checkGrantPrincipalReferences(ctx context.Context) error { zap.Int64("dropped_grants", droppedRows), zap.Int("dangling_principals", danglingPrincipals), zap.Int("refs_into_unsynced_types", unsyncedType), - zap.Int("refs_missing_rows_in_synced_types", missingRow), - zap.String("attribution", danglingAttribution(unsyncedType, missingRow)), + zap.Int("refs_manufactured_by_compaction_merge", mergeManufactured), + zap.String("attribution", danglingAttribution(unsyncedType, mergeManufactured)), zap.Strings("principal_examples", principalExamples), ) } diff --git a/pkg/sync/ingest_invariants_probe_test.go b/pkg/sync/ingest_invariants_probe_test.go new file mode 100644 index 000000000..ba23e6b28 --- /dev/null +++ b/pkg/sync/ingest_invariants_probe_test.go @@ -0,0 +1,97 @@ +package sync //nolint:revive,nolintlint // we can't change the package name for backwards compatibility + +// Pins the dangling-type probe's error handling: a resource-type read +// failure during the I7/I8/I9 sweep must FAIL the check, never +// masquerade as "type never synced" — that verdict picks the DROP arm, +// so an IO error or canceled context would seal a sanitized artifact +// where the replay policy demands a loud failure. + +import ( + "context" + "errors" + "path/filepath" + "testing" + + "github.com/stretchr/testify/require" + + reader_v2 "github.com/conductorone/baton-sdk/pb/c1/reader/v2" + "github.com/conductorone/baton-sdk/pkg/connectorstore" + "github.com/conductorone/baton-sdk/pkg/dotc1z" + "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" + "github.com/conductorone/baton-sdk/pkg/logging" + et "github.com/conductorone/baton-sdk/pkg/types/entitlement" + rs "github.com/conductorone/baton-sdk/pkg/types/resource" +) + +// typeProbeFaultStore wraps a real store and fails every GetResourceType +// with a chosen non-NotFound error, leaving the rest of the invariant +// surface (HasResourceRecord, the sweep iterators, the delete paths) +// intact — the exact seam the danglingTypeProbe reads. +type typeProbeFaultStore struct { + c1zstore.Store + dotc1z.IngestInvariantStore + err error +} + +func (s typeProbeFaultStore) GetResourceType( + context.Context, *reader_v2.ResourceTypesReaderServiceGetResourceTypeRequest, +) (*reader_v2.ResourceTypesReaderServiceGetResourceTypeResponse, error) { + return nil, s.err +} + +func TestDanglingTypeProbeErrorFailsInsteadOfDropping(t *testing.T) { + ctx, err := logging.Init(t.Context()) + require.NoError(t, err) + tmpDir := t.TempDir() + + repo, err := rs.NewResource("Repo r1", equivRepoRT, "r1") + require.NoError(t, err) + ent := et.NewAssignmentEntitlement(repo, "admin") + + // A dangling entitlement (no resource row, no type row): with a + // healthy probe this is a disabled-type gap and gets DROPPED. With + // the probe FAILING, the sweep must fail — before the fix, the + // probe's error was collapsed into "type never synced" and the row + // was silently dropped. + cur := newRepairTestStore(ctx, t, filepath.Join(tmpDir, "cur.c1z"), tmpDir) + _, err = cur.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + require.NoError(t, cur.PutEntitlements(ctx, ent)) + + inv, ok := cur.(dotc1z.IngestInvariantStore) + require.True(t, ok) + probeFault := errors.New("injected: resource-type read failed") + s := &syncer{ + store: typeProbeFaultStore{Store: cur, IngestInvariantStore: inv, err: probeFault}, + syncType: connectorstore.SyncTypeFull, + } + + err = s.checkEntitlementResourceReferences(ctx) + require.Error(t, err, "a probe read failure must fail the sweep, not pick a drop verdict") + require.ErrorIs(t, err, probeFault) + + _, err = cur.GetEntitlement(ctx, reader_v2.EntitlementsReaderServiceGetEntitlementRequest_builder{ + EntitlementId: ent.GetId(), + }.Build()) + require.NoError(t, err, "no row may be dropped on a probe failure") + + // Same seam for I8: a grant whose entitlement has no row. + err = s.checkGrantEntitlementReferences(ctx) + require.NoError(t, err, "no dangling grants planted; I8 must pass without probing") + + require.NoError(t, cur.Close(ctx)) +} + +// TestDanglingAttributionBuckets pins the drop-arm attribution wording: +// the drop arm only ever sees config gaps and compaction-manufactured +// gaps (enabled-type danglings in a normal sync FAIL before any drop), +// so no drop warning may blame connector magic-id construction. +func TestDanglingAttributionBuckets(t *testing.T) { + require.Contains(t, danglingAttribution(3, 0), "config gap") + require.Contains(t, danglingAttribution(0, 2), "compaction") + require.Contains(t, danglingAttribution(1, 1), "mixed") + for _, s := range []string{danglingAttribution(3, 0), danglingAttribution(0, 2), danglingAttribution(1, 1)} { + require.NotContains(t, s, "magic-id", + "drop-arm attribution must not blame connector id construction — those cases fail the sync instead") + } +} diff --git a/pkg/sync/source_cache.go b/pkg/sync/source_cache.go index 65e39d2b2..69d4be2f5 100644 --- a/pkg/sync/source_cache.go +++ b/pkg/sync/source_cache.go @@ -286,24 +286,15 @@ func replayCompatMismatchReason(ctx context.Context, current sourcecache.CompatK return current.MismatchReason(prevKey) } -// previousSyncReplayUnusableReason inspects the previous-sync reader's -// newest finished run record against the replay-source predicate -// (c1zstore.SyncRun.UsableAsReplaySource) and returns a human-readable -// reason when it fails, or "" when the sync qualifies. A reader without a -// SyncMeta surface degrades to "" — the manifest-based guards still -// apply — but a readable record is authoritative. -func previousSyncReplayUnusableReason(ctx context.Context, reader connectorstore.Reader) (string, error) { - metaHolder, ok := reader.(interface{ SyncMeta() c1zstore.SyncMeta }) - if !ok { - return "", nil - } - run, err := metaHolder.SyncMeta().LatestFinishedSyncOfAnyType(ctx) - if err != nil { - return "", err - } +// ReplaySourceRunUnusableReason evaluates the sync-run metadata gate on +// one finished-run record: "" when the run qualifies as a replay source, +// or a human-readable reason. Exported as the single source of truth for +// the gate — the syncer's read-side setup and the `baton source-cache` +// audit command both consume it, so the two can never drift. +func ReplaySourceRunUnusableReason(run *c1zstore.SyncRun) string { switch { case run == nil: - return "previous-sync c1z has no finished sync run", nil + return "previous-sync c1z has no finished sync run" case run.UsableAsReplaySource(): // The record can lie by omission: a compacted artifact produced // by an SDK that predates the run record's compacted flag carries @@ -316,14 +307,32 @@ func previousSyncReplayUnusableReason(ctx context.Context, reader connectorstore // artifact cannot omit. A token that fails to parse proves // nothing either way and is ignored (the primary gates stand). if comp, tokenErr := CompactionStatsFromToken(run.SyncToken); tokenErr == nil && comp != nil { - return "previous sync's token carries compaction provenance; the artifact is a merge no upstream validator describes (produced by a pre-compacted-flag compactor)", nil + return "previous sync's token carries compaction provenance; the artifact is a merge no upstream validator describes (produced by a pre-compacted-flag compactor)" } - return "", nil + return "" case run.Compacted: - return "previous sync is a compaction artifact; upstream validators do not describe its merged contents", nil + return "previous sync is a compaction artifact; upstream validators do not describe its merged contents" default: - return fmt.Sprintf("previous sync is type %q, not a full sync; its rows do not cover the scopes any validator vouches for", run.Type), nil + return fmt.Sprintf("previous sync is type %q, not a full sync; its rows do not cover the scopes any validator vouches for", run.Type) + } +} + +// previousSyncReplayUnusableReason inspects the previous-sync reader's +// newest finished run record against the replay-source predicate +// (ReplaySourceRunUnusableReason) and returns a human-readable reason +// when it fails, or "" when the sync qualifies. A reader without a +// SyncMeta surface degrades to "" — the manifest-based guards still +// apply — but a readable record is authoritative. +func previousSyncReplayUnusableReason(ctx context.Context, reader connectorstore.Reader) (string, error) { + metaHolder, ok := reader.(interface{ SyncMeta() c1zstore.SyncMeta }) + if !ok { + return "", nil + } + run, err := metaHolder.SyncMeta().LatestFinishedSyncOfAnyType(ctx) + if err != nil { + return "", err } + return ReplaySourceRunUnusableReason(run), nil } // clearSourceCacheLookup detaches the per-sync lookup so a late RPC from diff --git a/pkg/tasks/c1api/full_sync.go b/pkg/tasks/c1api/full_sync.go index 5e5058b7a..7a8e66518 100644 --- a/pkg/tasks/c1api/full_sync.go +++ b/pkg/tasks/c1api/full_sync.go @@ -71,6 +71,13 @@ type fullSyncTaskHandler struct { // task manager; nil in tests that don't exercise rotation. spareMu *stdsync.Mutex + // openedSpare reports that THIS task selected the SDK spare as its + // replay source. Retirement is gated on it: a replay-integrity + // failure may only retire the spare when the spare was actually + // opened — an operator-supplied --previous-sync-c1z takes precedence + // over the spare, and a failure on the operator's file must never + // delete an SDK spare that sat unopened on disk. + openedSpare bool // openedSpareSyncID records the sync id of the spare THIS task // actually opened as its replay source (read at open time, before // any concurrent rotation can swap the file). Retirement compares it @@ -78,6 +85,10 @@ type fullSyncTaskHandler struct { // failing sync may only retire the artifact that failed it, never a // newer spare promoted underneath it. openedSpareSyncID string + // openedSparePrivatePath is the task-private hardlink pinning the + // exact inode this sync replays from (see openSpareForReplay). + // Removed when the sync finishes. + openedSparePrivatePath string } // previousSyncSparePrefix is the fixed filename prefix of the retained @@ -209,6 +220,80 @@ func (c *fullSyncTaskHandler) withSpareLock(fn func()) { fn() } +// openSpareForReplay selects the SDK spare as this task's replay source +// and returns the path the syncer should open, or "" when no spare is +// on disk. Under the spare lock it captures the spare's sync id AND pins +// the exact inode with a task-private hardlink; the syncer opens the +// link, so a concurrent promotion (an os.Rename onto the shared slot) +// can neither swap the file between identity capture and open nor +// desync the recorded id from what the sync actually read. Without the +// pin, a promotion in that gap would make a later replay-integrity +// retirement compare the WRONG id and preserve the implicated artifact. +// +// The link lives next to the spare (same directory, task-digest suffix +// — constant shape, never caller-influenced) and is removed by +// releaseSpareSnapshot when the sync finishes. A failed link degrades +// to opening the shared path directly: the tiny selection/open race +// returns, costing at worst one preserved-implicated-spare cycle that +// the next successful rotation heals. +func (c *fullSyncTaskHandler) openSpareForReplay(ctx context.Context) string { + l := ctxzap.Extract(ctx) + openPath := "" + c.withSpareLock(func() { + if !fileExists(c.previousSyncSparePath) { + return + } + taskDigest := sha256.Sum256([]byte(c.task.GetId())) + private := fmt.Sprintf("%s.open-%x", c.previousSyncSparePath, taskDigest[:8]) + _ = os.Remove(private) // stale link from a crashed run of this task id + if linkErr := os.Link(c.previousSyncSparePath, private); linkErr == nil { + c.openedSparePrivatePath = private + openPath = private + } else { + l.Warn("failed to pin previous-sync spare with a task-private link; opening the shared path directly", + zap.String("spare_path", c.previousSyncSparePath), zap.Error(linkErr)) + openPath = c.previousSyncSparePath + } + // Read the id from the path the sync will actually open: with + // the pin this is race-free; without it, the same best-effort + // semantics as before. + c.openedSpareSyncID = c1zSyncIDBestEffort(openPath) + c.openedSpare = true + }) + return openPath +} + +// releaseSpareSnapshot removes the task-private hardlink created by +// openSpareForReplay. Safe to call when none was created. +func (c *fullSyncTaskHandler) releaseSpareSnapshot(ctx context.Context) { + if c.openedSparePrivatePath == "" { + return + } + if err := os.Remove(c.openedSparePrivatePath); err != nil && !errors.Is(err, os.ErrNotExist) { + ctxzap.Extract(ctx).Warn("failed to remove task-private spare link", zap.Error(err), + zap.String("path", c.openedSparePrivatePath)) + } + c.openedSparePrivatePath = "" +} + +// retireImplicatedSpare handles the spare side of a replay-integrity +// failure. Gated on openedSpare: only a sync that actually replayed +// from the spare may retire it — a failure on an operator-supplied +// previous file (which wins selection over the spare) must leave the +// unopened spare in place, and a sync that ran cold has nothing to +// retire. +func (c *fullSyncTaskHandler) retireImplicatedSpare(ctx context.Context) { + if !c.openedSpare || c.previousSyncSparePath == "" { + return + } + // Under the spare lock, and only the exact artifact this sync + // opened — a concurrent task's freshly promoted spare is not + // implicated and must survive. + c.withSpareLock(func() { + retireSpareIfStillImplicated(ctx, c.previousSyncSparePath, c.openedSpareSyncID) + }) +} + // resolveStorageEngine returns the engine this task's sync writes: // runner config wins, then the task's own engine, then "" (the SDK // default — SQLite). @@ -290,15 +375,13 @@ func (c *fullSyncTaskHandler) sync(ctx context.Context, c1zPath string) error { zap.String("previous_sync_id", c1zSyncIDBestEffort(c.previousSyncC1ZPath))) syncOpts = append(syncOpts, sdkSync.WithPreviousSyncC1ZPath(c.previousSyncC1ZPath)) case c.previousSyncSparePath != "": - if fileExists(c.previousSyncSparePath) { - // Record which artifact THIS sync replays from, so a - // replay-integrity retirement can never remove a fresh spare - // a concurrent task promoted mid-sync. - c.openedSpareSyncID = c1zSyncIDBestEffort(c.previousSyncSparePath) + if openPath := c.openSpareForReplay(ctx); openPath != "" { + defer c.releaseSpareSnapshot(ctx) l.Info("previous-sync spare found; source-cache replay enabled for this sync", zap.String("spare_path", c.previousSyncSparePath), + zap.String("open_path", openPath), zap.String("spare_sync_id", c.openedSpareSyncID)) - syncOpts = append(syncOpts, sdkSync.WithOptionalPreviousSyncC1ZPath(c.previousSyncSparePath)) + syncOpts = append(syncOpts, sdkSync.WithOptionalPreviousSyncC1ZPath(openPath)) } else { l.Info("no previous-sync spare on disk; source-cache replay inactive for this sync (expected on the first sync after enabling, or after a failed rotation)", zap.String("spare_path", c.previousSyncSparePath)) @@ -366,14 +449,7 @@ func (c *fullSyncTaskHandler) sync(ctx context.Context, c1zPath string) error { if rmErr := os.Remove(c1zPath); rmErr != nil && !os.IsNotExist(rmErr) { return errors.Join(err, rmErr) } - if c.previousSyncSparePath != "" { - // Under the spare lock, and only the exact artifact this sync - // opened — a concurrent task's freshly promoted spare is not - // implicated and must survive. - c.withSpareLock(func() { - retireSpareIfStillImplicated(ctx, c.previousSyncSparePath, c.openedSpareSyncID) - }) - } + c.retireImplicatedSpare(ctx) err = runSync(append(slices.Clone(syncOpts), sdkSync.WithoutPreviousSync())) } if err != nil { diff --git a/pkg/tasks/c1api/full_sync_spare_race_test.go b/pkg/tasks/c1api/full_sync_spare_race_test.go index 79dcf6f02..0a4c4a4ca 100644 --- a/pkg/tasks/c1api/full_sync_spare_race_test.go +++ b/pkg/tasks/c1api/full_sync_spare_race_test.go @@ -96,6 +96,78 @@ func TestRetireSpareIfStillImplicated(t *testing.T) { }) } +// TestRetireImplicatedSpare_RequiresOpenedSpare pins the retirement +// gate: only a sync that actually opened the spare may retire it. An +// operator-supplied --previous-sync-c1z wins selection over the spare, +// so a replay-integrity failure on the operator's file must leave the +// never-opened spare on disk — before the gate, any warm failure with a +// spare path configured deleted the spare unconditionally (and, with no +// opened identity recorded, would also delete a concurrent promotion). +func TestRetireImplicatedSpare_RequiresOpenedSpare(t *testing.T) { + ctx, err := logging.Init(context.Background()) + require.NoError(t, err) + tmpDir := t.TempDir() + sparePath := filepath.Join(tmpDir, "baton-previous-sync-test.c1z") + writeSpareC1Z(ctx, t, sparePath, tmpDir) + + h := &fullSyncTaskHandler{ + task: fullSyncTask(t, "pebble"), + previousSyncSparePath: sparePath, + previousSyncC1ZPath: filepath.Join(tmpDir, "operator.c1z"), // operator path configured too + // openedSpare deliberately false: the operator path won selection. + } + h.retireImplicatedSpare(ctx) + require.FileExists(t, sparePath, + "a warm failure on the operator's previous file must not retire the SDK spare it never opened") + + // The same handler after actually opening the spare retires it. + h2 := &fullSyncTaskHandler{task: fullSyncTask(t, "pebble"), previousSyncSparePath: sparePath} + openPath := h2.openSpareForReplay(ctx) + require.NotEmpty(t, openPath) + h2.retireImplicatedSpare(ctx) + h2.releaseSpareSnapshot(ctx) + require.NoFileExists(t, sparePath, "an opened, unrotated spare is implicated and must be retired") +} + +// TestOpenSpareForReplay_PinSurvivesConcurrentPromotion pins the +// selection/open race fix: the task-private hardlink freezes the exact +// inode the sync replays from, so a promotion landing between selection +// and open can neither swap the file under the sync nor desync the +// recorded identity — retirement then correctly preserves the fresh +// promotion while the sync read the implicated artifact throughout. +func TestOpenSpareForReplay_PinSurvivesConcurrentPromotion(t *testing.T) { + ctx, err := logging.Init(context.Background()) + require.NoError(t, err) + tmpDir := t.TempDir() + sparePath := filepath.Join(tmpDir, "baton-previous-sync-test.c1z") + implicatedID := writeSpareC1Z(ctx, t, sparePath, tmpDir) + + h := &fullSyncTaskHandler{task: fullSyncTask(t, "pebble"), previousSyncSparePath: sparePath} + openPath := h.openSpareForReplay(ctx) + require.NotEmpty(t, openPath) + require.NotEqual(t, sparePath, openPath, "selection must pin a task-private link, not the shared slot") + require.Equal(t, implicatedID, h.openedSpareSyncID) + + // Concurrent task promotes a FRESH spare onto the shared slot after + // selection but before/during this sync's reads. + freshPath := filepath.Join(tmpDir, "fresh.c1z") + freshID := writeSpareC1Z(ctx, t, freshPath, tmpDir) + require.NoError(t, os.Rename(freshPath, sparePath)) + + // The pinned inode still serves the artifact this sync selected. + require.Equal(t, implicatedID, c1zSyncIDBestEffort(openPath), + "the pinned link must keep serving the selected artifact across a promotion") + + // Warm failure: retirement preserves the fresh promotion (identity + // mismatch) — and the implicated artifact dies with the pin release. + h.retireImplicatedSpare(ctx) + require.FileExists(t, sparePath) + require.Equal(t, freshID, c1zSyncIDBestEffort(sparePath), + "the concurrently promoted spare is not implicated and must survive") + h.releaseSpareSnapshot(ctx) + require.NoFileExists(t, openPath, "the task-private pin must be cleaned up") +} + // TestSpareRetentionEligibility pins the engine gate on spare rotation: // only Pebble artifacts can serve source-cache replay (SQLite replay is // an explicit non-goal), so a keep-previous-sync-c1z opt-in running on diff --git a/proto/c1/connector/v2/annotation_source_cache.proto b/proto/c1/connector/v2/annotation_source_cache.proto index 02fbada11..a262a2169 100644 --- a/proto/c1/connector/v2/annotation_source_cache.proto +++ b/proto/c1/connector/v2/annotation_source_cache.proto @@ -65,6 +65,35 @@ message SourceCacheCapability { MODE_READ_WRITE = 2; } Mode mode = 1; + + // cache_generation is the connector's replay-compatibility generation: + // bump it whenever the connector changes how it computes scopes, + // validators, or rows in a way that makes a PRIOR artifact's cached + // rows unsafe to replay (schema of emitted rows, scope partitioning, + // id construction). The SDK never interprets the value — replay is + // permitted only when the previous artifact recorded a byte-identical + // generation (any mismatch runs the sync cold). Never derive this from + // a semver: compatibility is an explicit declaration, not an + // inference. Empty is a valid (constant) generation. + string cache_generation = 2 [(validate.rules).string = { + ignore_empty: true + max_bytes: 256 + }]; + + // config_fingerprint is an opaque connector-computed digest of every + // configuration and permission input that changes what upstream data + // the connector CAN see (credentials scope, tenant/org selection, + // API-permission grants, include/exclude config). Replay is permitted + // only when the previous artifact recorded a byte-identical + // fingerprint: a config change means cached scopes may cover a + // different universe, and replaying them would resurrect rows the new + // configuration can no longer observe (or hide rows it now can). + // Empty means "connector declares no config sensitivity" and matches + // only empty. + string config_fingerprint = 3 [(validate.rules).string = { + ignore_empty: true + max_bytes: 256 + }]; } // SourceCacheRecord is attached to a list-response page whose rows were diff --git a/proto/c1/connector/v2/annotation_type_scoped_entitlements.proto b/proto/c1/connector/v2/annotation_type_scoped_entitlements.proto index 5f028c3eb..cd2ddb416 100644 --- a/proto/c1/connector/v2/annotation_type_scoped_entitlements.proto +++ b/proto/c1/connector/v2/annotation_type_scoped_entitlements.proto @@ -52,9 +52,10 @@ option go_package = "github.com/conductorone/baton-sdk/pb/c1/connector/v2"; // leaves type-scoped entitlement enumeration to a full entitlements // phase. // -// ROUTING MARKER ONLY: TypeScopedEntitlements is not a stream-coupled -// side-effect annotation. It is absent from the ingestion-invariant -// side-effect coverage map for the same reason as TypeScopedGrants — -// it routes the ListEntitlements call; it does not imply post-ingest -// repair or validation beyond ordinary entitlement handling. +// ROUTING MARKER: TypeScopedEntitlements routes the ListEntitlements +// call; it implies no repair machinery of its own. It IS listed in the +// ingestion-invariant side-effect coverage map, covered by invariant I7 +// (entitlement→resource referential integrity): type-granularity scopes +// can carry entitlement rows for resources that vanished between +// enumerations, and the post-collection check is what catches that. message TypeScopedEntitlements {} diff --git a/proto/c1/connector/v2/annotation_type_scoped_grants.proto b/proto/c1/connector/v2/annotation_type_scoped_grants.proto index ef7707817..3b8a19c74 100644 --- a/proto/c1/connector/v2/annotation_type_scoped_grants.proto +++ b/proto/c1/connector/v2/annotation_type_scoped_grants.proto @@ -48,9 +48,12 @@ option go_package = "github.com/conductorone/baton-sdk/pb/c1/connector/v2"; // SyncGrantsOp for a type-scoped type. Type-scoped grant enumeration is // left to a full grants phase. // -// ROUTING MARKER ONLY: TypeScopedGrants is absent from the ingestion- -// invariant side-effect coverage map — it routes ListGrants; it does not -// imply post-ingest repair beyond ordinary grant handling. +// ROUTING MARKER: TypeScopedGrants routes ListGrants; it implies no +// repair machinery of its own. It IS listed in the ingestion-invariant +// side-effect coverage map, covered by invariant I8 (grant→entitlement +// referential integrity): independently-fresh type scopes can strand +// grant references against a refreshed entitlement set, and the +// post-collection check is what catches that. message TypeScopedGrants {} // EnqueuePageTokens is attached to a ListGrants or ListEntitlements response to diff --git a/proto/c1/storage/v3/records.proto b/proto/c1/storage/v3/records.proto index 292d35f65..0f10bf5ee 100644 --- a/proto/c1/storage/v3/records.proto +++ b/proto/c1/storage/v3/records.proto @@ -327,4 +327,51 @@ message SourceCacheEntryRecord { string cache_validator = 3; google.protobuf.Timestamp discovered_at = 4; + + // Set when the dangling-reference drops (ingestion invariants I7/I8/I9, + // pkg/sync/ingest_invariants.go) deleted rows stamped with this scope: + // the artifact no longer holds the full row set the validator vouches + // for, so a future sync must not 304-replay it. Lookups treat an + // invalidated entry as a miss (the scope re-fetches cold and converges + // with a cold sync); the entry itself is kept so the scope's surviving + // stamped rows do not read as an I6 orphan (lost manifest write). + bool invalidated = 5; +} + +// SourceCacheCompatRecord is the sync's replay-compatibility key: the +// exact conditions under which this artifact's source-cache manifest was +// recorded. A future sync may replay from this artifact ONLY when its +// own key matches byte-exactly on every component; an absent, unreadable, +// or mismatched record degrades that sync to cold (no-op lookup). +// Compatibility is never inferred from versions — each component is an +// explicit declaration by its owner. +// +// One record per sync, written when the write side of the source cache +// enables (before any manifest entry), cleared by compaction folds with +// the manifest. +message SourceCacheCompatRecord { + option (table) = { + name: "source_cache_compat" + primary_key: ["id"] + }; + + // Fixed singleton id ("compat"); present so the record satisfies the + // table shape. + string id = 1; + + // Connector-declared components (SourceCacheCapability annotations): + // the connector's cache generation and configuration/permission + // fingerprint. + string connector_cache_generation = 2; + string connector_config_fingerprint = 3; + + // SDK materialization-policy generation: bumped when the SDK changes + // how response rows or their side effects materialize into the store + // in a replay-visible way (sourcecache.MaterializationPolicyGeneration). + string sdk_materialization_generation = 4; + + // SDK sync-selection fingerprint: digest of the sync-shaping inputs + // (enabled resource types, skip flags) — a selection change means the + // prior artifact's scopes cover a different row universe. + string sync_selection_fingerprint = 5; } From e6bf4b5e251599e46f491fd69280abe8148aef32 Mon Sep 17 00:00:00 2001 From: Matt Kaniaris Date: Thu, 16 Jul 2026 10:33:08 -0600 Subject: [PATCH 14/17] expanded harness coverage --- .../v2/annotation_source_cache.pb.go | 16 ++ .../annotation_source_cache_protoopaque.pb.go | 8 + pkg/dotc1z/engine/pebble/ingest_repair.go | 141 ++++++++++- .../pebble/ingest_repair_orphan_index_test.go | 97 ++++++++ .../pebble/obligations_on_failure_test.go | 225 ++++++++++++++++++ pkg/sync/ingest_invariants.go | 54 ++++- pkg/sync/ingest_invariants_probe_test.go | 45 ++++ .../source_cache_equivalence_dirty_test.go | 33 ++- .../v2/annotation_source_cache.proto | 8 + 9 files changed, 613 insertions(+), 14 deletions(-) create mode 100644 pkg/dotc1z/engine/pebble/ingest_repair_orphan_index_test.go create mode 100644 pkg/dotc1z/engine/pebble/obligations_on_failure_test.go diff --git a/pb/c1/connector/v2/annotation_source_cache.pb.go b/pb/c1/connector/v2/annotation_source_cache.pb.go index f11ddfe0b..93e5092a5 100644 --- a/pb/c1/connector/v2/annotation_source_cache.pb.go +++ b/pb/c1/connector/v2/annotation_source_cache.pb.go @@ -393,6 +393,14 @@ type SourceCacheReplay struct { // SourceCacheRecord with the same scope_key) contains changed rows to // upsert on top of the replayed base. When false the response must // contain no rows for this scope. + // + // TRANSITIONAL: the SDK currently tolerates rows on a non-overlay + // replay page — it warns and upserts them with overlay semantics — + // while the model is validated against real providers. Do not rely on + // this: the tolerance will become a hard error, because a connector + // that fetched fresh rows and still attached a replay annotation keeps + // resurrecting the replayed base every sync (upstream deletions never + // propagate). Overlay bool `protobuf:"varint,3,opt,name=overlay,proto3" json:"overlay,omitempty"` // Public canonical IDs (grant/entitlement IDs, or resource BIDs for // RowKindResources) to delete from the current sync after the replay @@ -501,6 +509,14 @@ type SourceCacheReplay_builder struct { // SourceCacheRecord with the same scope_key) contains changed rows to // upsert on top of the replayed base. When false the response must // contain no rows for this scope. + // + // TRANSITIONAL: the SDK currently tolerates rows on a non-overlay + // replay page — it warns and upserts them with overlay semantics — + // while the model is validated against real providers. Do not rely on + // this: the tolerance will become a hard error, because a connector + // that fetched fresh rows and still attached a replay annotation keeps + // resurrecting the replayed base every sync (upstream deletions never + // propagate). Overlay bool // Public canonical IDs (grant/entitlement IDs, or resource BIDs for // RowKindResources) to delete from the current sync after the replay diff --git a/pb/c1/connector/v2/annotation_source_cache_protoopaque.pb.go b/pb/c1/connector/v2/annotation_source_cache_protoopaque.pb.go index 25a3a06ab..74c055264 100644 --- a/pb/c1/connector/v2/annotation_source_cache_protoopaque.pb.go +++ b/pb/c1/connector/v2/annotation_source_cache_protoopaque.pb.go @@ -428,6 +428,14 @@ type SourceCacheReplay_builder struct { // SourceCacheRecord with the same scope_key) contains changed rows to // upsert on top of the replayed base. When false the response must // contain no rows for this scope. + // + // TRANSITIONAL: the SDK currently tolerates rows on a non-overlay + // replay page — it warns and upserts them with overlay semantics — + // while the model is validated against real providers. Do not rely on + // this: the tolerance will become a hard error, because a connector + // that fetched fresh rows and still attached a replay annotation keeps + // resurrecting the replayed base every sync (upstream deletions never + // propagate). Overlay bool // Public canonical IDs (grant/entitlement IDs, or resource BIDs for // RowKindResources) to delete from the current sync after the replay diff --git a/pkg/dotc1z/engine/pebble/ingest_repair.go b/pkg/dotc1z/engine/pebble/ingest_repair.go index ef32fafc0..6378c9226 100644 --- a/pkg/dotc1z/engine/pebble/ingest_repair.go +++ b/pkg/dotc1z/engine/pebble/ingest_repair.go @@ -21,6 +21,8 @@ import ( "fmt" "github.com/cockroachdb/pebble/v2" + "github.com/grpc-ecosystem/go-grpc-middleware/logging/zap/ctxzap" + "go.uber.org/zap" v3 "github.com/conductorone/baton-sdk/pb/c1/storage/v3" "github.com/conductorone/baton-sdk/pkg/dotc1z/engine/pebble/codec" @@ -63,6 +65,11 @@ func (e *Engine) EnsureGrantIndexes(ctx context.Context) error { // (the mixed-principal path reports its own per-grant skip count). // Value reads happen only for dangling principals, never on the // healthy path. +// +// A principal whose index entries are ALL orphans (no primary rows) is +// never visited: it has no grants to judge, so it is healed in place — +// the orphan index keys are deleted — instead of being vacuously +// classified as match-annotated-only. func (e *Engine) ForEachDanglingGrantPrincipal(ctx context.Context, visit func(principalRT, principalID string, matchAnnotatedOnly bool, carrierGrants int64) error) error { prefix := []byte{versionV3, typeIndex, idxGrantByPrincipal} prefix = codec.AppendTupleSeparator(prefix) @@ -75,6 +82,14 @@ func (e *Engine) ForEachDanglingGrantPrincipal(ctx context.Context, visit func(p } defer iter.Close() + // Healed-orphan aggregation: one Warn per sweep (the house shape — + // never per-principal logs; a systematic writer bug could strand + // entries for thousands of principals). Orphans are always a bug + // signal, so the aggregate must be visible at default level. + var healedEntries int64 + healedPrincipals := 0 + var healedExamples []string + for valid := iter.First(); valid; { if err := ctx.Err(); err != nil { return err @@ -99,14 +114,101 @@ func (e *Engine) ForEachDanglingGrantPrincipal(ctx context.Context, visit func(p if err != nil { return err } - if err := visit(rt, rid, matchOnly, carrierGrants); err != nil { - return err + switch { + case matchOnly && carrierGrants == 0: + // Every index entry under this principal is an orphan + // (no primary row): there are no grants to judge, so + // neither the match-carrier exemption nor the fail/drop + // arms apply — the "all match-annotated" verdict was + // vacuous. Heal the index garbage instead, mirroring the + // replay pre-clear's treatment of orphan scope-index + // entries, so a writer bug that strands by_principal + // entries is scrubbed rather than vacuously exempted on + // every future sweep. + healed, err := e.healOrphanPrincipalIndexEntries(ctx, rt, rid) + if err != nil { + return err + } + if healed > 0 { + healedEntries += healed + healedPrincipals++ + if len(healedExamples) < maxHealedOrphanExamples { + healedExamples = append(healedExamples, rt+"/"+rid) + } + } + default: + if err := visit(rt, rid, matchOnly, carrierGrants); err != nil { + return err + } } } // Skip every remaining grant of this principal. valid = iter.SeekGE(upperBoundOf(encodeGrantByPrincipalPrefix(rt, rid))) } - return iter.Error() + if err := iter.Error(); err != nil { + return err + } + if healedEntries > 0 { + ctxzap.Extract(ctx).Warn("HEALED orphan by_principal index entries (index keys with no grant row — evidence of a writer bug, not connector data)", + zap.Int64("healed_entries", healedEntries), + zap.Int("principals", healedPrincipals), + zap.Strings("principal_examples", healedExamples), + ) + } + return nil +} + +// maxHealedOrphanExamples caps the principal examples on the aggregated +// healed-orphan warning, matching the dangling-reference warnings' shape. +const maxHealedOrphanExamples = 25 + +// healOrphanPrincipalIndexEntries deletes by_principal index entries +// whose primary grant row does not exist, for one principal. Identities +// are re-collected and re-probed under the write lock, so an entry whose +// row appeared since the caller's read scan is kept. Returns the number +// of entries healed. +func (e *Engine) healOrphanPrincipalIndexEntries(ctx context.Context, principalRT, principalID string) (int64, error) { + var healed int64 + err := e.withWrite(func() error { + ids, err := e.grantIdentitiesForPrincipal(ctx, principalRT, principalID) + if err != nil { + return err + } + batch := e.db.NewBatch() + defer batch.Close() + for _, id := range ids { + if err := ctx.Err(); err != nil { + return err + } + _, closer, getErr := e.db.Get(encodeGrantIdentityKey(id)) + switch { + case errors.Is(getErr, pebble.ErrNotFound): + if err := batch.Delete(encodeGrantByPrincipalIdentityIndexKey(id), nil); err != nil { + return err + } + healed++ + case getErr != nil: + return getErr + default: + closer.Close() + } + } + if healed == 0 { + return nil + } + return batch.Commit(e.dropWriteOpts()) + }) + if err != nil { + return 0, err + } + if healed > 0 { + ctxzap.Extract(ctx).Debug("healed orphan by_principal index entries", + zap.String("principal_resource_type_id", principalRT), + zap.String("principal_resource_id", principalID), + zap.Int64("healed", healed), + ) + } + return healed, nil } // grantsForPrincipalAllMatchAnnotated reports whether every grant under @@ -415,6 +517,15 @@ func (e *Engine) DeleteGrantsForPrincipal(ctx context.Context, principalRT, prin b := e.newDropBatcher(ctx) defer b.close() + // Keyed on commits that LANDED, not on success: committed drop + // chunks make any stashed deferred-grant stats stale even when a + // later chunk errors, so the sidecar must recount rather than + // report rows that were durably dropped. + defer func() { + if b.committedRows > 0 { + e.invalidateDeferredGrantStats() + } + }() for iter.First(); iter.Valid(); iter.Next() { if err := ctx.Err(); err != nil { @@ -448,7 +559,16 @@ func (e *Engine) DeleteGrantsForPrincipal(ctx context.Context, principalRT, prin val, closer, getErr := e.db.Get(priKey) if getErr != nil { if errors.Is(getErr, pebble.ErrNotFound) { - continue // index entry without a row; tolerated + // Orphan index entry (no row): heal it in the same + // batch pass, mirroring the replay pre-clear, so the + // drop doesn't leave index garbage behind. + if err := b.batch.Delete(iter.Key(), nil); err != nil { + return err + } + if err := b.rotate(); err != nil { + return err + } + continue } return getErr } @@ -477,9 +597,6 @@ func (e *Engine) DeleteGrantsForPrincipal(ctx context.Context, principalRT, prin if err != nil { return 0, 0, err } - if deleted > 0 { - e.invalidateDeferredGrantStats() - } return deleted, skipped, nil } @@ -514,6 +631,13 @@ func (e *Engine) DeleteGrantsForEntitlement(ctx context.Context, entitlementID, b := e.newDropBatcher(ctx) defer b.close() + // Keyed on commits that LANDED, not on success — see + // DeleteGrantsForPrincipal's twin defer. + defer func() { + if b.committedRows > 0 { + e.invalidateDeferredGrantStats() + } + }() for iter.First(); iter.Valid(); iter.Next() { if err := ctx.Err(); err != nil { @@ -547,9 +671,6 @@ func (e *Engine) DeleteGrantsForEntitlement(ctx context.Context, entitlementID, if err != nil { return 0, 0, err } - if deleted > 0 { - e.invalidateDeferredGrantStats() - } return deleted, skipped, nil } diff --git a/pkg/dotc1z/engine/pebble/ingest_repair_orphan_index_test.go b/pkg/dotc1z/engine/pebble/ingest_repair_orphan_index_test.go new file mode 100644 index 000000000..106a03487 --- /dev/null +++ b/pkg/dotc1z/engine/pebble/ingest_repair_orphan_index_test.go @@ -0,0 +1,97 @@ +package pebble + +// Pins the orphan by_principal handling in the dangling-principal sweep +// (I9's engine side): an index entry with no primary grant row must be +// HEALED (deleted), never vacuously classified. Before the fix, a +// principal whose entries were all orphans came back as "all +// match-annotated" with zero carriers — skipping the fail/drop arms, +// emitting no signal, and leaving the garbage for every future sweep — +// and the drop path kept orphan keys behind the rows it deleted. + +import ( + "context" + "fmt" + "testing" + + "github.com/cockroachdb/pebble/v2" + "github.com/stretchr/testify/require" + + "github.com/conductorone/baton-sdk/pkg/connectorstore" +) + +// plantOrphanPrincipalIndexEntry writes ONLY a by_principal index key — +// no primary row — simulating a writer bug that stranded the entry. +func plantOrphanPrincipalIndexEntry(t *testing.T, e *Engine, principalID, entTail string) grantIdentity { + t.Helper() + id := grantIdentity{ + entitlement: entitlementIdentityFromParts("group", "g1", "group:g1:"+entTail), + principalTypeID: "user", + principalID: principalID, + } + require.NoError(t, e.db.Set(encodeGrantByPrincipalIdentityIndexKey(id), nil, nil)) + return id +} + +func countPrincipalIndexEntries(t *testing.T, e *Engine, principalRT, principalID string) int { + t.Helper() + ids, err := e.grantIdentitiesForPrincipal(context.Background(), principalRT, principalID) + require.NoError(t, err) + return len(ids) +} + +func TestDanglingPrincipalSweepHealsAllOrphanIndexEntries(t *testing.T) { + ctx := context.Background() + a := newAdapter(t) + _, err := a.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + e := a.PebbleEngine() + + // "ghost" has index entries but no grant rows and no resource row: + // the all-orphan shape. Before the fix this visited as + // (matchAnnotatedOnly=true, carrierGrants=0) and nothing was cleaned. + for i := 0; i < 3; i++ { + plantOrphanPrincipalIndexEntry(t, e, "ghost", fmt.Sprintf("member%d", i)) + } + + visited := 0 + require.NoError(t, e.ForEachDanglingGrantPrincipal(ctx, func(rt, rid string, matchOnly bool, carriers int64) error { + visited++ + return nil + })) + require.Zero(t, visited, "an all-orphan principal has no grants to judge and must not be visited") + require.Zero(t, countPrincipalIndexEntries(t, e, "user", "ghost"), + "orphan index entries must be healed, not tolerated forever") + + // Idempotent: a re-run sees a clean index. + require.NoError(t, e.ForEachDanglingGrantPrincipal(ctx, func(rt, rid string, matchOnly bool, carriers int64) error { + visited++ + return nil + })) + require.Zero(t, visited) +} + +func TestDeleteGrantsForPrincipalHealsOrphanEntries(t *testing.T) { + ctx := context.Background() + a := newAdapter(t) + _, err := a.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + e := a.PebbleEngine() + + // Mixed population: one REAL dangling grant row (drops) plus one + // orphan index entry (must be healed by the same pass, not left + // behind the dropped rows). + g := mkV2Grant("g-real", "group:g1:member", "user", "mixed") + require.NoError(t, a.PutGrants(ctx, g)) + plantOrphanPrincipalIndexEntry(t, e, "mixed", "owner") + require.Equal(t, 2, countPrincipalIndexEntries(t, e, "user", "mixed")) + + deleted, skipped, err := e.DeleteGrantsForPrincipal(ctx, "user", "mixed") + require.NoError(t, err) + require.Equal(t, int64(1), deleted, "the real dangling row drops") + require.Zero(t, skipped) + require.Zero(t, countPrincipalIndexEntries(t, e, "user", "mixed"), + "the orphan entry must be healed in the same pass") + + _, err = e.GetGrantRecord(ctx, g.GetId()) + require.ErrorIs(t, err, pebble.ErrNotFound) +} diff --git a/pkg/dotc1z/engine/pebble/obligations_on_failure_test.go b/pkg/dotc1z/engine/pebble/obligations_on_failure_test.go new file mode 100644 index 000000000..94af9f734 --- /dev/null +++ b/pkg/dotc1z/engine/pebble/obligations_on_failure_test.go @@ -0,0 +1,225 @@ +package pebble + +// Table-driven harness for the recurring bug class of this branch: +// POST-MUTATION OBLIGATIONS SKIPPED ON THE ERROR PATH. Every mutation +// path that commits intermediate batches owes side effects the moment +// any commit lands — empty-keyspace fast-path disarm, bare-id lookup +// invalidation, manifest invalidation, stats-stash invalidation — and +// each has independently shipped a bug where the obligation only fired +// on full success (the replay pre-clear, the partial replay copy, the +// deferred-grant-stats stash). +// +// One table row per mutation path: force a failure AFTER the first +// landed chunk (via the replay batch var or the drop chunk hook) and +// assert every declared obligation fired anyway. A new mutation path +// gets a table row, not a bespoke test. + +import ( + "context" + "errors" + "fmt" + "testing" + + "github.com/stretchr/testify/require" + + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" + "github.com/conductorone/baton-sdk/pkg/connectorstore" + "github.com/conductorone/baton-sdk/pkg/sourcecache" +) + +// obligationsCase drives one mutation path to a mid-flight failure. +// run must return the injected error (asserted non-nil); obligations +// assert the engine state the committed prefix demands. +type obligationsCase struct { + name string + run func(t *testing.T, ctx context.Context, cur, prev *Adapter, syncID string) error + obligations func(t *testing.T, e *Engine, syncID string) +} + +// plantDanglingGrants puts enough grants under one missing-referent +// shape to span at least one full drop chunk, all stamped with scopeA +// under a live manifest entry, so the chunk hook can fire mid-drop. +// varyEnt=true varies the entitlement per grant (one shared principal — +// the by_principal drop shape); varyEnt=false keeps one entitlement +// (the by_entitlement drop shape). +func plantDanglingGrants(t *testing.T, ctx context.Context, a *Adapter, varyEnt bool) { + t.Helper() + const total = dropBatchRows + 500 + scoped := sourcecache.WithScope(ctx, scopeA) + const putChunk = 1000 + for start := 0; start < total; start += putChunk { + end := min(start+putChunk, total) + page := make([]*v2.Grant, 0, end-start) + for i := start; i < end; i++ { + if varyEnt { + page = append(page, scGrant(fmt.Sprintf("member%05d", i), "dangler", false)) + } else { + page = append(page, scGrant("member", fmt.Sprintf("u%05d", i), false)) + } + } + require.NoError(t, a.PutGrants(scoped, page...)) + } + require.NoError(t, a.PebbleEngine().PutSourceCacheEntry(ctx, "grants", scopeA, "etag-v1")) +} + +func requireStashInvalidated(t *testing.T, e *Engine, syncID string) { + t.Helper() + require.Nil(t, e.takeDeferredGrantStats(syncID), + "committed drop chunks make the stashed grant counts stale; the stash must be invalidated even when the drop fails") +} + +func requireScopeAInvalidated(t *testing.T, e *Engine) { + t.Helper() + rec, err := e.GetSourceCacheEntry(context.Background(), "grants", scopeA) + require.NoError(t, err) + require.True(t, rec.GetInvalidated(), + "the scope losing rows must be invalidated with the first committed chunk") +} + +func TestFailedMutationPathsFireObligations(t *testing.T) { + injected := errors.New("injected mid-mutation failure") + + cases := []obligationsCase{ + { + // Replay pre-clear: crashed-attempt residue deleted, then the + // copy is cancelled. The empty-keyspace fast path must be + // disarmed by the landed deletes alone. + name: "replay-grants-preclear", + run: func(t *testing.T, ctx context.Context, cur, prev *Adapter, _ string) error { + e := cur.PebbleEngine() + for i := 0; i < 3; i++ { + plantScopedGrant(t, e, scopeA, fmt.Sprintf("u%d", i)) + } + cctx := &failAfterNChecks{Context: ctx, remaining: 1} + _, err := e.ReplaySourceCacheGrants(cctx, prev.PebbleEngine(), scopeA) + return err + }, + obligations: func(t *testing.T, e *Engine, _ string) { + require.False(t, e.takeFreshGrantsEmpty(), + "landed pre-clear deletes must disarm the grants fast path") + }, + }, + { + name: "replay-entitlements-preclear", + run: func(t *testing.T, ctx context.Context, cur, prev *Adapter, _ string) error { + e := cur.PebbleEngine() + for i := 0; i < 3; i++ { + plantScopedEntitlement(t, e, scopeA, fmt.Sprintf("g%d", i)) + } + cctx := &failAfterNChecks{Context: ctx, remaining: 1} + _, err := e.ReplaySourceCacheEntitlements(cctx, prev.PebbleEngine(), scopeA) + return err + }, + obligations: func(t *testing.T, e *Engine, _ string) { + require.False(t, e.takeFreshEntitlementsEmpty(), + "landed pre-clear deletes must disarm the entitlements fast path (and bump the bare-id generation)") + }, + }, + { + name: "replay-resources-preclear", + run: func(t *testing.T, ctx context.Context, cur, prev *Adapter, _ string) error { + e := cur.PebbleEngine() + for i := 0; i < 3; i++ { + plantScopedResource(t, e, scopeA, fmt.Sprintf("u%d", i)) + } + cctx := &failAfterNChecks{Context: ctx, remaining: 1} + _, err := e.ReplaySourceCacheResources(cctx, prev.PebbleEngine(), scopeA) + return err + }, + obligations: func(t *testing.T, e *Engine, _ string) { + require.False(t, e.takeFreshResourcesEmpty(), + "landed pre-clear deletes must disarm the resources fast path") + }, + }, + { + // I8's drop arm: one entitlement, > one chunk of grants. Fail + // after the first committed chunk. + name: "drop-grants-for-entitlement", + run: func(t *testing.T, ctx context.Context, cur, prev *Adapter, syncID string) error { + e := cur.PebbleEngine() + plantDanglingGrants(t, ctx, cur, false) + e.stashDeferredGrantStats(&deferredGrantStats{syncID: syncID, grants: 999_999}) + e.testDropChunkHook = func() error { return injected } + t.Cleanup(func() { e.testDropChunkHook = nil }) + _, _, err := e.DeleteGrantsForEntitlement(ctx, "group:g1:member", "group", "g1") + return err + }, + obligations: func(t *testing.T, e *Engine, syncID string) { + requireScopeAInvalidated(t, e) + requireStashInvalidated(t, e, syncID) + }, + }, + { + // I9's drop arm: one principal, > one chunk of grants. + name: "drop-grants-for-principal", + run: func(t *testing.T, ctx context.Context, cur, prev *Adapter, syncID string) error { + e := cur.PebbleEngine() + plantDanglingGrants(t, ctx, cur, true) + e.stashDeferredGrantStats(&deferredGrantStats{syncID: syncID, grants: 999_999}) + e.testDropChunkHook = func() error { return injected } + t.Cleanup(func() { e.testDropChunkHook = nil }) + _, _, err := e.DeleteGrantsForPrincipal(ctx, "user", "dangler") + return err + }, + obligations: func(t *testing.T, e *Engine, syncID string) { + requireScopeAInvalidated(t, e) + requireStashInvalidated(t, e, syncID) + }, + }, + { + // I7's drop arm: entitlements under one resource. Obligation: + // the bare-id keyspace note (observable via the fresh flag — + // the delete mutated the keyspace, so the flag must be gone). + name: "drop-entitlements-for-resource", + run: func(t *testing.T, ctx context.Context, cur, prev *Adapter, _ string) error { + e := cur.PebbleEngine() + const total = dropBatchRows + 500 + const putChunk = 1000 + for start := 0; start < total; start += putChunk { + end := min(start+putChunk, total) + page := make([]*v2.Entitlement, 0, end-start) + for i := start; i < end; i++ { + page = append(page, v2.Entitlement_builder{ + Id: fmt.Sprintf("group:g1:member%05d", i), + Resource: v2.Resource_builder{ + Id: v2.ResourceId_builder{ResourceType: "group", Resource: "g1"}.Build(), + }.Build(), + }.Build()) + } + require.NoError(t, cur.PutEntitlements(sourcecache.WithScope(ctx, scopeA), page...)) + } + require.NoError(t, e.PutSourceCacheEntry(ctx, "entitlements", scopeA, "etag-v1")) + e.testDropChunkHook = func() error { return injected } + t.Cleanup(func() { e.testDropChunkHook = nil }) + _, _, err := e.DeleteEntitlementsForResource(ctx, "group", "g1", 5) + return err + }, + obligations: func(t *testing.T, e *Engine, _ string) { + rec, err := e.GetSourceCacheEntry(context.Background(), "entitlements", scopeA) + require.NoError(t, err) + require.True(t, rec.GetInvalidated(), + "the scope losing entitlement rows must be invalidated with the first committed chunk") + }, + }, + } + + for _, tc := range cases { + t.Run(tc.name, func(t *testing.T) { + ctx := context.Background() + oldBatch := replayBatchRows + replayBatchRows = 1 + t.Cleanup(func() { replayBatchRows = oldBatch }) + + prev := newAdapter(t) + _, err := prev.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + cur := newAdapter(t) + syncID, err := cur.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + + err = tc.run(t, ctx, cur, prev, syncID) + require.Error(t, err, "the case must fail mid-mutation; a clean run proves nothing about error-path obligations") + tc.obligations(t, cur.PebbleEngine(), syncID) + }) + } +} diff --git a/pkg/sync/ingest_invariants.go b/pkg/sync/ingest_invariants.go index faad7d4ba..2d6539235 100644 --- a/pkg/sync/ingest_invariants.go +++ b/pkg/sync/ingest_invariants.go @@ -476,6 +476,13 @@ func (t *exclusionGroupTracker) record(ent *v2.Entitlement) error { // sole exclusion-group validation: it covers fresh, replayed, and // generated (static) entitlements uniformly, on every engine, regardless // of how rows arrived. +// +// Verdicts ride the warm/cold ladder like the other FAIL-class +// invariants (I3/I6/enabled I7–I9): replayed stale entitlement rows can +// manufacture warm-only conflicts (a stale group member beside a fresh +// one), so a warm failure wraps ErrReplayIntegrity for discard+cold +// while a cold failure is attributed plainly to the connector. Store IO +// errors stay plain — they are not replay-implicated. func (s *syncer) validateStoredExclusionGroups(ctx context.Context) error { tracker := &exclusionGroupTracker{} pageToken := "" @@ -489,7 +496,7 @@ func (s *syncer) validateStoredExclusionGroups(ctx context.Context) error { } for _, ent := range resp.GetList() { if err := tracker.record(ent); err != nil { - return err + return s.wrapWarmReplayIntegrity(fmt.Errorf("ingest invariant I5 violated: %w", err)) } } if pageToken = resp.GetNextPageToken(); pageToken == "" { @@ -557,7 +564,16 @@ func (p *danglingTypeProbe) typeExists(ctx context.Context, resourceTypeID strin // consistent with one aggregated warning, and their scopes' manifest // entries are invalidated so a later type-enable re-fetches them cold. func (s *syncer) unexpectedDanglingError(format string, args ...any) error { - err := fmt.Errorf(format, args...) + return s.wrapWarmReplayIntegrity(fmt.Errorf(format, args...)) +} + +// wrapWarmReplayIntegrity classifies a FAIL-class invariant verdict for +// the cold-retry ladder: on a WARM sync the evidence may be +// replay-carried staleness, so the error wraps ErrReplayIntegrity and +// the runners discard the output and retry cold; the same evidence on a +// COLD sync is terminal and attributed plainly. Only VERDICTS ride the +// ladder — store IO errors are not replay-implicated and stay plain. +func (s *syncer) wrapWarmReplayIntegrity(err error) error { if s.sourceCache.prev != nil { return fmt.Errorf("%w (warm sync: discard the output and retry cold): %w", err, ErrReplayIntegrity) } @@ -802,6 +818,14 @@ func (s *syncer) checkGrantPrincipalReferences(ctx context.Context) error { if err := facts.EnsureGrantIndexes(ctx); err != nil { return fmt.Errorf("ingest invariant I9: ensuring grant indexes: %w", err) } + if s.testSourceCacheHaltHook != nil { + // Seam between the deferred index build (marker durably cleared) + // and I9's scan/drops: a crash here must resume without EndSync + // repeating the O(grants) rebuild and with I9 re-scanning cleanly. + if err := s.testSourceCacheHaltHook("invariants-I9-indexes-ensured", ""); err != nil { + return err + } + } probe := &danglingTypeProbe{s: s} danglingPrincipals, unsyncedType, mergeManufactured := 0, 0, 0 var droppedRows, matchCarriersKept int64 @@ -932,23 +956,49 @@ func (s *syncer) checkSourceCacheScopeConsistency(ctx context.Context) error { // re-reaches this point re-evaluates with the same verdict (drops // already applied leave nothing dangling on re-run). func (s *syncer) runIngestionInvariants(ctx context.Context) error { + // halt fires the test-only seam hook between invariants, so the + // crash sweep (TestSourceCache_DirtyHaltSweepInsideInvariantPass) + // can pin resume convergence at every mid-pass stop point — e.g. + // "I7's drops committed, I8 never ran" must converge on re-run via + // the drop cascade. Nil-guarded no-op in production. + halt := func(stage string) error { + if s.testSourceCacheHaltHook == nil { + return nil + } + return s.testSourceCacheHaltHook(stage, "") + } if err := s.validateStoredExclusionGroups(ctx); err != nil { return err } + if err := halt("invariants-I5-complete"); err != nil { + return err + } if err := s.checkSourceCacheScopeConsistency(ctx); err != nil { return err } + if err := halt("invariants-I6-complete"); err != nil { + return err + } if err := s.checkChildScheduling(ctx); err != nil { return err } if err := s.checkEntitlementResourceReferences(ctx); err != nil { return err } + if err := halt("invariants-I7-complete"); err != nil { + return err + } if err := s.checkGrantResourceReferences(ctx); err != nil { return err } + if err := halt("invariants-I3-complete"); err != nil { + return err + } if err := s.checkGrantEntitlementReferences(ctx); err != nil { return err } + if err := halt("invariants-I8-complete"); err != nil { + return err + } return s.checkGrantPrincipalReferences(ctx) } diff --git a/pkg/sync/ingest_invariants_probe_test.go b/pkg/sync/ingest_invariants_probe_test.go index ba23e6b28..1f2aaf30e 100644 --- a/pkg/sync/ingest_invariants_probe_test.go +++ b/pkg/sync/ingest_invariants_probe_test.go @@ -14,7 +14,9 @@ import ( "github.com/stretchr/testify/require" + v2 "github.com/conductorone/baton-sdk/pb/c1/connector/v2" reader_v2 "github.com/conductorone/baton-sdk/pb/c1/reader/v2" + "github.com/conductorone/baton-sdk/pkg/annotations" "github.com/conductorone/baton-sdk/pkg/connectorstore" "github.com/conductorone/baton-sdk/pkg/dotc1z" "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" @@ -82,6 +84,49 @@ func TestDanglingTypeProbeErrorFailsInsteadOfDropping(t *testing.T) { require.NoError(t, cur.Close(ctx)) } +// TestExclusionGroupVerdictRidesWarmColdLadder pins I5's membership in +// the warm/cold ErrReplayIntegrity ladder: replayed stale entitlement +// rows can manufacture warm-only exclusion-group conflicts (a stale +// group member beside a fresh one), so a warm verdict must carry the +// sentinel (runners discard + retry cold) while a cold verdict is +// attributed plainly to the connector — the same classification as +// I3/I6/enabled I7–I9. +func TestExclusionGroupVerdictRidesWarmColdLadder(t *testing.T) { + ctx, err := logging.Init(t.Context()) + require.NoError(t, err) + tmpDir := t.TempDir() + + group := v2.EntitlementExclusionGroup_builder{ExclusionGroupId: "eg-1", IsDefault: true}.Build() + res := v2.Resource_builder{ + Id: v2.ResourceId_builder{ResourceType: "group", Resource: "g1"}.Build(), + }.Build() + // Two defaults in one group: an I5 violation however the rows arrived. + ent1 := v2.Entitlement_builder{Id: "group:g1:member", Resource: res, Annotations: annotations.New(group)}.Build() + ent2 := v2.Entitlement_builder{Id: "group:g1:owner", Resource: res, Annotations: annotations.New(group)}.Build() + + cur := newRepairTestStore(ctx, t, filepath.Join(tmpDir, "cur.c1z"), tmpDir) + _, err = cur.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + require.NoError(t, cur.PutEntitlements(ctx, ent1, ent2)) + + s := &syncer{store: cur, syncType: connectorstore.SyncTypeFull} + + // Cold: plain connector attribution, no sentinel. + err = s.validateStoredExclusionGroups(ctx) + require.Error(t, err) + require.Contains(t, err.Error(), "ingest invariant I5") + require.Contains(t, err.Error(), "multiple default entitlements") + require.NotErrorIs(t, err, ErrReplayIntegrity, "a cold verdict is terminal, not a replay fault") + + // Warm: the same evidence may be replay-carried staleness. + s.sourceCache.prev = struct{ dotc1z.SourceCacheStore }{} + err = s.validateStoredExclusionGroups(ctx) + require.Error(t, err) + require.ErrorIs(t, err, ErrReplayIntegrity, "a warm verdict must ride the discard-and-retry-cold ladder") + + require.NoError(t, cur.Close(ctx)) +} + // TestDanglingAttributionBuckets pins the drop-arm attribution wording: // the drop arm only ever sees config gaps and compaction-manufactured // gaps (enabled-type danglings in a normal sync FAIL before any drop), diff --git a/pkg/sync/source_cache_equivalence_dirty_test.go b/pkg/sync/source_cache_equivalence_dirty_test.go index 222c2e910..250824cec 100644 --- a/pkg/sync/source_cache_equivalence_dirty_test.go +++ b/pkg/sync/source_cache_equivalence_dirty_test.go @@ -328,6 +328,35 @@ const dirtyCrashRound = 2 // through round 1's dirty→clean replay, where a lost invalidation would // 304 the sanitized subsets. func TestSourceCache_DirtyHaltAtInvariantSeam(t *testing.T) { + runDirtyHaltAtStage(t, "ingest-invariants-complete") +} + +// TestSourceCache_DirtyHaltSweepInsideInvariantPass sweeps every seam +// INSIDE the invariant pass (the stage hooks in runIngestionInvariants): +// halt after each invariant's drops/repairs committed, resume the same +// file, and require the resumed output to equal a cold sync. This is +// what pins the stop-point matrix from the crash review — "I7's drops +// committed, I8 never ran" converges via the drop cascade on re-run; +// "deferred index built, marker cleared, I9 never scanned" re-scans +// without repeating the O(grants) build — instead of arguing those +// recoveries from code structure. +func TestSourceCache_DirtyHaltSweepInsideInvariantPass(t *testing.T) { + stages := []string{ + "invariants-I5-complete", + "invariants-I6-complete", + "invariants-I7-complete", + "invariants-I3-complete", + "invariants-I8-complete", + "invariants-I9-indexes-ensured", + } + for _, stage := range stages { + t.Run(stage, func(t *testing.T) { + runDirtyHaltAtStage(t, stage) + }) + } +} + +func runDirtyHaltAtStage(t *testing.T, stage string) { ctx, err := logging.Init(t.Context()) require.NoError(t, err) tmpDir := t.TempDir() @@ -338,7 +367,7 @@ func TestSourceCache_DirtyHaltAtInvariantSeam(t *testing.T) { var warmClient types.ConnectorClient = warmConn var coldClient types.ConnectorClient = coldConn - halt := &haltOnce{stage: "ingest-invariants-complete"} + halt := &haltOnce{stage: stage} hookOpt := func(s *syncer) { s.testSourceCacheHaltHook = halt.hook } // Round 0: the warm leg crashes at the seam, then resumes the SAME @@ -346,7 +375,7 @@ func TestSourceCache_DirtyHaltAtInvariantSeam(t *testing.T) { warm0 := filepath.Join(tmpDir, "warm-r0.c1z") haltErr := runEquivSyncMode(ctx, t, warmClient, warm0, "", tmpDir, false, hookOpt) require.Error(t, haltErr, "the armed halt must crash the dirty sync at the invariant seam") - require.Contains(t, haltErr.Error(), "injected halt at ingest-invariants-complete") + require.Contains(t, haltErr.Error(), "injected halt at "+stage) require.NoError(t, runEquivSyncMode(ctx, t, warmClient, warm0, "", tmpDir, false, hookOpt), "resume after the invariant-seam halt must complete (drops must be idempotent over already-dropped state)") diff --git a/proto/c1/connector/v2/annotation_source_cache.proto b/proto/c1/connector/v2/annotation_source_cache.proto index a262a2169..d4fcce1f7 100644 --- a/proto/c1/connector/v2/annotation_source_cache.proto +++ b/proto/c1/connector/v2/annotation_source_cache.proto @@ -170,6 +170,14 @@ message SourceCacheReplay { // SourceCacheRecord with the same scope_key) contains changed rows to // upsert on top of the replayed base. When false the response must // contain no rows for this scope. + // + // TRANSITIONAL: the SDK currently tolerates rows on a non-overlay + // replay page — it warns and upserts them with overlay semantics — + // while the model is validated against real providers. Do not rely on + // this: the tolerance will become a hard error, because a connector + // that fetched fresh rows and still attached a replay annotation keeps + // resurrecting the replayed base every sync (upstream deletions never + // propagate). bool overlay = 3; // Public canonical IDs (grant/entitlement IDs, or resource BIDs for From a4791bc55a1049b45896ee7ac6648a878a8360c4 Mon Sep 17 00:00:00 2001 From: Matt Kaniaris Date: Thu, 16 Jul 2026 11:26:33 -0600 Subject: [PATCH 15/17] more findings --- pkg/dotc1z/engine/pebble/adapter.go | 57 ++++++++---- pkg/dotc1z/engine/pebble/cleanup.go | 13 +++ .../pebble/end_sync_flush_failure_test.go | 60 +++++++++++++ pkg/dotc1z/engine/pebble/engine.go | 23 ++++- pkg/sync/ingest_invariants.go | 27 +++++- pkg/sync/ingest_invariants_probe_test.go | 87 +++++++++++++++++++ pkg/sync/syncer.go | 22 +++-- 7 files changed, 263 insertions(+), 26 deletions(-) create mode 100644 pkg/dotc1z/engine/pebble/end_sync_flush_failure_test.go diff --git a/pkg/dotc1z/engine/pebble/adapter.go b/pkg/dotc1z/engine/pebble/adapter.go index 3fb22f48d..daff3d4c6 100644 --- a/pkg/dotc1z/engine/pebble/adapter.go +++ b/pkg/dotc1z/engine/pebble/adapter.go @@ -327,7 +327,9 @@ func (a *Adapter) EndSync(ctx context.Context) error { } // endSyncFinalize runs the sealed tail of EndSync: the deferred index -// build, the ended_at stamp, the stats sidecar, and the durability flush. +// build, the stats sidecar, the durability flush, and — strictly last — +// the ended_at stamp (the stamp is the finished verdict; it must never +// precede the flush that makes the sync's data durable). // Runs with the engine SEALED (see EndSync) — every write below goes // through an AllowSealed path. Split out so EndSync can unseal on failure. func (e *Engine) endSyncFinalize(ctx context.Context, existing *v3.SyncRunRecord) error { @@ -365,22 +367,13 @@ func (e *Engine) endSyncFinalize(ctx context.Context, existing *v3.SyncRunRecord return fmt.Errorf("EndSync: repair grant digests: %w", err) } } - // Mutate the loaded record rather than rebuilding it field-by-field — - // a rebuild drops every field it doesn't name (supports_diff, - // linked_sync_id, compacted), which erased compaction provenance and - // diff markers when a sync re-ended after expansion or resume. - existing.SetEndedAt(timestamppb.Now()) - if err := e.PutSyncRunRecord(ctx, existing); err != nil { - return err - } // Populate the stats sidecar BEFORE the durability flush. Stats // is engine-meta keyspace; the EndFreshSync flush below covers - // the WAL fsync for both the sync_run record and the stats key. - // Failures here are non-fatal — Stats() falls back to legacy - // iteration on a missing sidecar, and the on-Open migration - // framework will backfill next time the file opens. We log a - // warning so the failure is visible in production telemetry but - // don't fail the sync end on stats-sidecar trouble. + // the WAL fsync for the stats key. Failures here are non-fatal — + // Stats() falls back to legacy iteration on a missing sidecar, and + // the on-Open migration framework will backfill next time the file + // opens. We log a warning so the failure is visible in production + // telemetry but don't fail the sync end on stats-sidecar trouble. if err := e.PersistSyncStats(ctx, existing.GetSyncId()); err != nil { ctxzap.Extract(ctx).Warn("pebble: persist sync stats sidecar failed; Stats() will fall back to O(N) iteration until the next Open backfills it", zap.String("sync_id", existing.GetSyncId()), @@ -389,8 +382,38 @@ func (e *Engine) endSyncFinalize(ctx context.Context, existing *v3.SyncRunRecord } // Single flush + WAL fsync at sync end. This is the durability // boundary — counterpart to MarkFreshSync at StartNewSync. After - // this returns, all writes from the sync are on disk. - return e.EndFreshSync(ctx) + // this returns, all writes from the sync are on disk. The sync stays + // BOUND across the flush (FlushForEndSync, not EndFreshSync) so a + // failed ended_at stamp below leaves the engine retryable in-process. + if e.testEndSyncFlushHook != nil { + if err := e.testEndSyncFlushHook(); err != nil { + return err + } + } + if err := e.FlushForEndSync(ctx); err != nil { + return err + } + // Stamp ended_at strictly AFTER the durability flush: the stamp is + // what makes the sync FINISHED (LatestUnfinishedSyncRecord skips + // stamped records), so it must never precede the flush that makes + // the sync's NoSync writes durable. A stamp committed ahead of a + // failed flush would orphan the unfinished work — resume starts a + // NEW sync instead of resuming — and, across a crash, could surface + // a partially-durable artifact as a FINISHED sync to replay/uplift + // consumers. The stamp's own commit is fsynced (PutSyncRunRecord + // writes with the engine's configured durability), so a crash after + // this line keeps the finished verdict and a crash before it keeps + // resumability. Mutate the loaded record rather than rebuilding it + // field-by-field — a rebuild drops every field it doesn't name + // (supports_diff, linked_sync_id, compacted), which erased compaction + // provenance and diff markers when a sync re-ended after expansion + // or resume. + existing.SetEndedAt(timestamppb.Now()) + if err := e.PutSyncRunRecord(ctx, existing); err != nil { + return err + } + e.clearCurrentSync() + return nil } // === writes === diff --git a/pkg/dotc1z/engine/pebble/cleanup.go b/pkg/dotc1z/engine/pebble/cleanup.go index ef8e84f29..13b374699 100644 --- a/pkg/dotc1z/engine/pebble/cleanup.go +++ b/pkg/dotc1z/engine/pebble/cleanup.go @@ -95,6 +95,19 @@ func (e *Engine) ResetForNewSync(ctx context.Context) error { // sync; the wipe is the first step of leaving the sealed state. The // engine stays sealed until MarkFreshSync unseals it right after. return e.withWriteAllowSealed(func() error { + // Flush first so the excises apply DIRECTLY to the manifest. + // Pebble downgrades an excise whose span overlaps unflushed + // memtable data to a flushable ingest — queued behind the + // memtable, taking effect only at the next flush — which defeats + // this method's immediate-reclaim contract (a short replacement + // sync could hard-link the prior sync's dead SSTs into the saved + // envelope). The sync-run record ({v3, typeSyncRun}) lives INSIDE + // the excised record span and EndSync stamps it AFTER its + // durability flush, so a just-ended sync always has exactly that + // key unflushed here. + if err := e.db.Flush(); err != nil { + return fmt.Errorf("ResetForNewSync: pre-excise flush: %w", err) + } for _, span := range spans { if err := e.db.Excise(ctx, span); err != nil { return fmt.Errorf("ResetForNewSync: excise [%x, %x): %w", span.Start, span.End, err) diff --git a/pkg/dotc1z/engine/pebble/end_sync_flush_failure_test.go b/pkg/dotc1z/engine/pebble/end_sync_flush_failure_test.go new file mode 100644 index 000000000..b47a41262 --- /dev/null +++ b/pkg/dotc1z/engine/pebble/end_sync_flush_failure_test.go @@ -0,0 +1,60 @@ +package pebble + +// Pins the seal-failure contract: EndSync's finalize stamps ended_at on +// the sync-run record, and the durability flush (EndFreshSync) turns the +// sync's NoSync writes into on-disk state. If the flush FAILS, the sync +// must remain visibly UNFINISHED — LatestUnfinishedSyncRecord skips any +// record with ended_at, so a premature stamp costs resumability (the +// next StartOrResumeSync starts a new sync instead of resuming) and, +// after a crash, can surface a partially-durable artifact as a FINISHED +// sync to replay/uplift consumers. + +import ( + "context" + "errors" + "testing" + + "github.com/stretchr/testify/require" + + v3 "github.com/conductorone/baton-sdk/pb/c1/storage/v3" + "github.com/conductorone/baton-sdk/pkg/connectorstore" +) + +func TestEndSyncFlushFailureKeepsSyncResumable(t *testing.T) { + ctx := context.Background() + a := newAdapter(t) + syncID, err := a.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + e := a.PebbleEngine() + require.NoError(t, a.PutGrants(ctx, mkV2Grant("g1", "group:g1:member", "user", "alice"))) + + injected := errors.New("injected flush failure at the durability boundary") + e.testEndSyncFlushHook = func() error { return injected } + err = a.EndSync(ctx) + require.ErrorIs(t, err, injected, "the failed flush must fail EndSync") + e.testEndSyncFlushHook = nil + + // THE CONTRACT: a sync whose durability flush failed is NOT finished. + // It must still be discoverable as unfinished (resumable), and must + // not carry ended_at (a stamped record with maybe-undurable data is + // a finished-looking lie). + unfinished, err := e.LatestUnfinishedSyncRecord(ctx, nil) + require.NoError(t, err) + require.NotNil(t, unfinished, + "a failed seal must leave the sync resumable; a premature ended_at stamp orphans the work") + require.Equal(t, syncID, unfinished.GetSyncId()) + + rec, err := e.GetSyncRunRecord(ctx, syncID) + require.NoError(t, err) + require.Nil(t, rec.GetEndedAt(), + "ended_at must only be stamped after the durability flush succeeds") + + // Retry converges: the second EndSync finishes the sync for real. + require.NoError(t, a.EndSync(ctx)) + rec, err = e.GetSyncRunRecord(ctx, syncID) + require.NoError(t, err) + require.NotNil(t, rec.GetEndedAt(), "the retried seal must stamp the sync finished") + unfinished, err = e.LatestUnfinishedSyncRecord(ctx, func(v3.SyncType) bool { return true }) + require.NoError(t, err) + require.Nil(t, unfinished, "no unfinished run may remain after a successful seal") +} diff --git a/pkg/dotc1z/engine/pebble/engine.go b/pkg/dotc1z/engine/pebble/engine.go index 4e3554b8f..013558b74 100644 --- a/pkg/dotc1z/engine/pebble/engine.go +++ b/pkg/dotc1z/engine/pebble/engine.go @@ -96,6 +96,14 @@ type Engine struct { // as one sorted SST at EndSync — see BuildDeferredGrantIndexes). deferredIdxPending atomic.Bool + // testEndSyncFlushHook, when non-nil, runs in endSyncFinalize + // immediately before the EndFreshSync durability flush — the + // in-process analog of the flush failing. Tests use it to pin the + // seal-failure contract: a sync whose finalize failed must remain + // UNFINISHED (no ended_at), or a resume would start a new sync and a + // crash could surface a partially-durable artifact as finished. + testEndSyncFlushHook func() error + // testDropChunkHook, when non-nil, runs after every chunked // dangling-reference drop batch commit (ingest_repair.go). Test-only: // deterministic crash injection at the exact seam between a committed @@ -532,6 +540,19 @@ func (e *Engine) takeFreshEntitlementsEmpty() bool { // closing check and writeWG: Close tears e.db down after writeWG.Wait, // and a bare-mutex EndFreshSync racing Close would flush a nil db. func (e *Engine) EndFreshSync(ctx context.Context) error { + if err := e.FlushForEndSync(ctx); err != nil { + return err + } + e.clearCurrentSync() + return nil +} + +// FlushForEndSync is EndFreshSync's durability flush WITHOUT the +// current-sync unbind: endSyncFinalize flushes first, stamps ended_at +// second, and unbinds last — so a failed stamp leaves the sync bound +// (and fresh) for an in-process retry, and a failed flush never follows +// a committed stamp (see the ordering contract in endSyncFinalize). +func (e *Engine) FlushForEndSync(ctx context.Context) error { // AllowSealed: this is the last step of EndSync's sealed finalize // window (see Adapter.EndSync). return e.withWriteAllowSealed(func() error { @@ -539,7 +560,6 @@ func (e *Engine) EndFreshSync(ctx context.Context) error { wasFresh := e.freshSync e.currentSyncMu.RUnlock() if !wasFresh { - e.clearCurrentSync() return nil } // Flush the memtable (turns NoSync-buffered writes into on-disk @@ -550,7 +570,6 @@ func (e *Engine) EndFreshSync(ctx context.Context) error { if err := e.db.LogData(nil, pebble.Sync); err != nil { return fmt.Errorf("EndFreshSync: fsync WAL: %w", err) } - e.clearCurrentSync() return nil }) } diff --git a/pkg/sync/ingest_invariants.go b/pkg/sync/ingest_invariants.go index 2d6539235..fa428ac96 100644 --- a/pkg/sync/ingest_invariants.go +++ b/pkg/sync/ingest_invariants.go @@ -309,7 +309,16 @@ func (s *syncer) checkGrantResourceReferences(ctx context.Context) error { rt, rid, ErrReplayIntegrity) } // Tolerated today: connectors emitting grants for unlisted - // resources without InsertResourceGrants. Visible, not fatal. + // resources without InsertResourceGrants. Visible, not fatal — + // except under fail-fast, whose contract is that every tolerated + // warn becomes a named failure (the harness must catch connector + // shapes production only logs). + if s.failFastInvariants { + return fmt.Errorf( + "ingest invariant I3 violated: grants reference resource %s/%s which was never synced "+ + "(no InsertResourceGrants annotation — tolerated with a warning in production, promoted under fail-fast)", + rt, rid) + } l.Warn("ingest invariant I3: grants reference a resource that was never synced", zap.String("resource_type_id", rt), zap.String("resource_id", rid), @@ -484,6 +493,14 @@ func (t *exclusionGroupTracker) record(ent *v2.Entitlement) error { // while a cold failure is attributed plainly to the connector. Store IO // errors stay plain — they are not replay-implicated. func (s *syncer) validateStoredExclusionGroups(ctx context.Context) error { + if s.syncType != connectorstore.SyncTypeFull { + // Store-derived verdicts are only evaluable over a COMPLETE + // keyspace; a partial sync writes a deliberate subset (the + // referential invariants I3/I7-I9 gate identically). Partial + // syncs keep the page-level streaming validation + // (validateEntitlementExclusionGroups). + return nil + } tracker := &exclusionGroupTracker{} pageToken := "" for { @@ -907,6 +924,14 @@ func (s *syncer) checkGrantPrincipalReferences(ctx context.Context) error { // lost manifest write or a stamp leak — and the damage is deferred: THIS // sync reads clean, the NEXT sync replays from the damaged state. func (s *syncer) checkSourceCacheScopeConsistency(ctx context.Context) error { + if s.syncType != connectorstore.SyncTypeFull { + // Partial syncs never replay and never enable the source-cache + // write side (configureSourceCache degrades on non-full syncs), + // so scope/manifest evidence in a partial store is inherited + // state this sync neither produced nor will serve replay from — + // same gate as the referential invariants. + return nil + } if s.onlyExpandGrants { // Expansion-only runs ingest no pages, so scope/manifest state // was settled (and I6-checked) by whoever produced the store. diff --git a/pkg/sync/ingest_invariants_probe_test.go b/pkg/sync/ingest_invariants_probe_test.go index 1f2aaf30e..76b02f94e 100644 --- a/pkg/sync/ingest_invariants_probe_test.go +++ b/pkg/sync/ingest_invariants_probe_test.go @@ -21,7 +21,9 @@ import ( "github.com/conductorone/baton-sdk/pkg/dotc1z" "github.com/conductorone/baton-sdk/pkg/dotc1z/c1zstore" "github.com/conductorone/baton-sdk/pkg/logging" + "github.com/conductorone/baton-sdk/pkg/sourcecache" et "github.com/conductorone/baton-sdk/pkg/types/entitlement" + "github.com/conductorone/baton-sdk/pkg/types/grant" rs "github.com/conductorone/baton-sdk/pkg/types/resource" ) @@ -127,6 +129,91 @@ func TestExclusionGroupVerdictRidesWarmColdLadder(t *testing.T) { require.NoError(t, cur.Close(ctx)) } +// TestFailFastPromotesI3UnannotatedWarn pins the fail-fast contract for +// I3's tolerated arm: production WARNS on grants referencing a +// never-synced entitlement resource without InsertResourceGrants +// (tolerated connector behavior), but fail-fast mode — the harness's +// whole mechanism for catching engineered violations — must promote +// that warn to a hard failure, or the harness silently passes connector +// shapes production would flag. +func TestFailFastPromotesI3UnannotatedWarn(t *testing.T) { + ctx, err := logging.Init(t.Context()) + require.NoError(t, err) + tmpDir := t.TempDir() + + repo, err := rs.NewResource("Repo r1", equivRepoRT, "r1") + require.NoError(t, err) + principal := v2.Resource_builder{ + Id: v2.ResourceId_builder{ResourceType: "user", Resource: "alice"}.Build(), + }.Build() + g := grant.NewGrant(repo, "admin", principal.GetId()) + + // The tolerated shape: a grant whose entitlement RESOURCE has no row + // and no InsertResourceGrants annotation. + cur := newRepairTestStore(ctx, t, filepath.Join(tmpDir, "cur.c1z"), tmpDir) + _, err = cur.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + require.NoError(t, cur.PutGrants(ctx, g)) + + // Default mode: warn only, no error (production tolerance). + s := &syncer{store: cur, syncType: connectorstore.SyncTypeFull} + require.NoError(t, s.checkGrantResourceReferences(ctx), + "default mode tolerates unannotated danglings with a warning") + + // Fail-fast: the warn must become a named failure. + s.failFastInvariants = true + err = s.checkGrantResourceReferences(ctx) + require.Error(t, err, "fail-fast must promote the tolerated warn to a hard failure") + require.Contains(t, err.Error(), "ingest invariant I3") + require.NotErrorIs(t, err, ErrReplayIntegrity, "fail-fast verdicts are plain, matching I7-I9") + + require.NoError(t, cur.Close(ctx)) +} + +// TestPartialSyncSkipsStoredInvariants pins the partial-sync gate for +// I5 and I6, matching I3/I7/I8/I9: a partial sync writes a deliberate +// subset, so store-derived verdicts computed over it are not +// evaluable — a partial must neither hard-fail on exclusion-group +// evidence nor on orphan-scope evidence while the referential arms are +// inert. +func TestPartialSyncSkipsStoredInvariants(t *testing.T) { + ctx, err := logging.Init(t.Context()) + require.NoError(t, err) + tmpDir := t.TempDir() + + group := v2.EntitlementExclusionGroup_builder{ExclusionGroupId: "eg-1", IsDefault: true}.Build() + res := v2.Resource_builder{ + Id: v2.ResourceId_builder{ResourceType: "group", Resource: "g1"}.Build(), + }.Build() + ent1 := v2.Entitlement_builder{Id: "group:g1:member", Resource: res, Annotations: annotations.New(group)}.Build() + ent2 := v2.Entitlement_builder{Id: "group:g1:owner", Resource: res, Annotations: annotations.New(group)}.Build() + + cur := newRepairTestStore(ctx, t, filepath.Join(tmpDir, "cur.c1z"), tmpDir) + _, err = cur.StartNewSync(ctx, connectorstore.SyncTypePartial, "") + require.NoError(t, err) + require.NoError(t, cur.PutEntitlements(ctx, ent1, ent2)) + // An orphan scope shape: stamped rows, no manifest entry. + repo, err := rs.NewResource("Repo r1", equivRepoRT, "r1") + require.NoError(t, err) + principal := v2.Resource_builder{ + Id: v2.ResourceId_builder{ResourceType: "user", Resource: "alice"}.Build(), + }.Build() + require.NoError(t, cur.PutGrants(sourcecache.WithScope(ctx, "scope-partial"), grant.NewGrant(repo, "admin", principal.GetId()))) + + s := &syncer{store: cur, syncType: connectorstore.SyncTypePartial} + require.NoError(t, s.validateStoredExclusionGroups(ctx), + "I5 must not run on a partial sync (subset store; referential arms are inert too)") + require.NoError(t, s.checkSourceCacheScopeConsistency(ctx), + "I6 must not run on a partial sync") + + // The same evidence on a FULL sync still fails both. + s.syncType = connectorstore.SyncTypeFull + require.Error(t, s.validateStoredExclusionGroups(ctx)) + require.Error(t, s.checkSourceCacheScopeConsistency(ctx)) + + require.NoError(t, cur.Close(ctx)) +} + // TestDanglingAttributionBuckets pins the drop-arm attribution wording: // the drop arm only ever sees config gaps and compaction-manufactured // gaps (enabled-type danglings in a normal sync FAIL before any drop), diff --git a/pkg/sync/syncer.go b/pkg/sync/syncer.go index 4eb84ef4f..2d5530832 100644 --- a/pkg/sync/syncer.go +++ b/pkg/sync/syncer.go @@ -147,8 +147,13 @@ type syncer struct { // childSchedule is the monotone record backing invariant I4; // resourcesPhaseRanHere gates I4 to processes that actually ran the // resources phase; failFastInvariants promotes every invariant - // verdict from its default handling (warn, drop, or repair) to hard - // sync failure (tests and the equivalence harness set it). + // verdict to a hard, plainly-attributed sync failure — tolerated + // warns fail, drop arms fail before dropping, repair arms are + // skipped so the underlying bug is named — and enables I4 (skipped + // entirely in default mode). Tests and the equivalence harness set + // it; production default follows the per-invariant policy in + // ingest_invariants.go (drop/warn/repair/fail with the warm-cold + // ErrReplayIntegrity ladder on FAIL-class verdicts). childSchedule childScheduleSet resourcesPhaseRanHere bool failFastInvariants bool @@ -3454,10 +3459,15 @@ func WithStorageEngine(engine c1zstore.Engine) SyncOpt { } } -// WithFailFastInvariants promotes check-only ingestion invariants -// (see ingest_invariants.go) from warn-with-log to hard sync failure. -// Tests and the replay-equivalence harness enable it; production default -// stays warn-and-drop until the predicates have a clean release behind them. +// WithFailFastInvariants promotes every ingestion-invariant verdict +// (see ingest_invariants.go) to a hard, plainly-attributed sync +// failure: tolerated warns fail, drop arms fail before dropping, repair +// arms are skipped so the underlying bug is named, and I4 (skipped in +// default mode) runs. Tests and the replay-equivalence harness enable +// it; production default follows the per-invariant policy — drops with +// aggregated warnings for disabled-type danglings, warm repair for I3, +// hard failure (with the warm/cold ErrReplayIntegrity ladder) for +// enabled-type danglings, I5, and I6. func WithFailFastInvariants() SyncOpt { return func(s *syncer) { s.failFastInvariants = true From b297f4bee3f234e08abaf40650164d5eb3a71143 Mon Sep 17 00:00:00 2001 From: Matt Kaniaris Date: Thu, 16 Jul 2026 15:34:42 -0600 Subject: [PATCH 16/17] pin endsync stamp-commit failure contract (H1 residual: verified in-memory stamp unreachable) Co-authored-by: Cursor --- pkg/dotc1z/engine/pebble/adapter.go | 9 ++++ .../pebble/end_sync_flush_failure_test.go | 41 +++++++++++++++++++ pkg/dotc1z/engine/pebble/engine.go | 12 ++++++ .../pebble/obligations_on_failure_test.go | 28 +++++++++++++ 4 files changed, 90 insertions(+) diff --git a/pkg/dotc1z/engine/pebble/adapter.go b/pkg/dotc1z/engine/pebble/adapter.go index daff3d4c6..a42581cf5 100644 --- a/pkg/dotc1z/engine/pebble/adapter.go +++ b/pkg/dotc1z/engine/pebble/adapter.go @@ -409,7 +409,16 @@ func (e *Engine) endSyncFinalize(ctx context.Context, existing *v3.SyncRunRecord // provenance and diff markers when a sync re-ended after expansion // or resume. existing.SetEndedAt(timestamppb.Now()) + if e.testEndSyncStampHook != nil { + if err := e.testEndSyncStampHook(); err != nil { + return err + } + } if err := e.PutSyncRunRecord(ctx, existing); err != nil { + // The in-memory stamp above needs no rollback: `existing` is a + // local copy unmarshalled by this EndSync call, unreachable once + // this error returns; a retried EndSync reloads the (unstamped) + // stored record. Pinned by the endsync-stamp-commit harness row. return err } e.clearCurrentSync() diff --git a/pkg/dotc1z/engine/pebble/end_sync_flush_failure_test.go b/pkg/dotc1z/engine/pebble/end_sync_flush_failure_test.go index b47a41262..1eadae948 100644 --- a/pkg/dotc1z/engine/pebble/end_sync_flush_failure_test.go +++ b/pkg/dotc1z/engine/pebble/end_sync_flush_failure_test.go @@ -58,3 +58,44 @@ func TestEndSyncFlushFailureKeepsSyncResumable(t *testing.T) { require.NoError(t, err) require.Nil(t, unfinished, "no unfinished run may remain after a successful seal") } + +// The stamp-commit analog of the flush test above (H1 residual): the +// durability flush SUCCEEDS, then the ended_at stamp's commit fails. +// endSyncFinalize has already mutated the in-memory record +// (existing.SetEndedAt) by then — that stamp must leak nowhere, and a +// retried EndSync must converge. State assertions on the engine live in +// the obligations harness row (endsync-stamp-commit-failure); this test +// owns the adapter-level retry contract. +func TestEndSyncStampFailureKeepsSyncResumable(t *testing.T) { + ctx := context.Background() + a := newAdapter(t) + syncID, err := a.StartNewSync(ctx, connectorstore.SyncTypeFull, "") + require.NoError(t, err) + e := a.PebbleEngine() + require.NoError(t, a.PutGrants(ctx, mkV2Grant("g1", "group:g1:member", "user", "alice"))) + + injected := errors.New("injected stamp-commit failure after the durability flush") + e.testEndSyncStampHook = func() error { return injected } + err = a.EndSync(ctx) + require.ErrorIs(t, err, injected, "the failed stamp commit must fail EndSync") + e.testEndSyncStampHook = nil + + rec, err := e.GetSyncRunRecord(ctx, syncID) + require.NoError(t, err) + require.Nil(t, rec.GetEndedAt(), + "the in-memory stamp must not reach the store when its commit fails") + unfinished, err := e.LatestUnfinishedSyncRecord(ctx, nil) + require.NoError(t, err) + require.NotNil(t, unfinished, "a failed stamp commit must leave the sync resumable") + require.Equal(t, syncID, unfinished.GetSyncId()) + + // Retry converges: the second EndSync reloads the (unstamped) stored + // record and finishes the sync for real. + require.NoError(t, a.EndSync(ctx)) + rec, err = e.GetSyncRunRecord(ctx, syncID) + require.NoError(t, err) + require.NotNil(t, rec.GetEndedAt(), "the retried seal must stamp the sync finished") + unfinished, err = e.LatestUnfinishedSyncRecord(ctx, func(v3.SyncType) bool { return true }) + require.NoError(t, err) + require.Nil(t, unfinished, "no unfinished run may remain after a successful seal") +} diff --git a/pkg/dotc1z/engine/pebble/engine.go b/pkg/dotc1z/engine/pebble/engine.go index 013558b74..a0997a25d 100644 --- a/pkg/dotc1z/engine/pebble/engine.go +++ b/pkg/dotc1z/engine/pebble/engine.go @@ -104,6 +104,18 @@ type Engine struct { // crash could surface a partially-durable artifact as finished. testEndSyncFlushHook func() error + // testEndSyncStampHook, when non-nil, runs in endSyncFinalize after + // the in-memory ended_at stamp but before PutSyncRunRecord commits + // it — the in-process analog of the stamp's commit failing (H1 + // residual). Tests use it to pin that a failed stamp commit leaks + // nothing: the stored record stays unstamped, the sync stays + // discoverable as unfinished, and a retried EndSync converges. The + // stamped in-memory record is a local, freshly-unmarshalled copy + // owned by the EndSync call (GetSyncRunRecord never returns a shared + // or cached record), so the mutation is unreachable once the error + // returns; the hook exists to keep that true against future caching. + testEndSyncStampHook func() error + // testDropChunkHook, when non-nil, runs after every chunked // dangling-reference drop batch commit (ingest_repair.go). Test-only: // deterministic crash injection at the exact seam between a committed diff --git a/pkg/dotc1z/engine/pebble/obligations_on_failure_test.go b/pkg/dotc1z/engine/pebble/obligations_on_failure_test.go index 94af9f734..036704cc2 100644 --- a/pkg/dotc1z/engine/pebble/obligations_on_failure_test.go +++ b/pkg/dotc1z/engine/pebble/obligations_on_failure_test.go @@ -166,6 +166,34 @@ func TestFailedMutationPathsFireObligations(t *testing.T) { requireStashInvalidated(t, e, syncID) }, }, + { + // H1 residual: endSyncFinalize stamps the in-memory record + // (existing.SetEndedAt) BEFORE PutSyncRunRecord commits it. If + // that commit fails, the stamp must leak nowhere — the stored + // record stays unstamped and the sync stays discoverable as + // unfinished. Safe today because `existing` is a local copy + // unmarshalled per EndSync call; this row keeps it safe against + // a future record cache. + name: "endsync-stamp-commit-failure", + run: func(t *testing.T, ctx context.Context, cur, prev *Adapter, _ string) error { + e := cur.PebbleEngine() + require.NoError(t, cur.PutGrants(ctx, mkV2Grant("g1", "group:g1:member", "user", "alice"))) + e.testEndSyncStampHook = func() error { return injected } + t.Cleanup(func() { e.testEndSyncStampHook = nil }) + return cur.EndSync(ctx) + }, + obligations: func(t *testing.T, e *Engine, syncID string) { + rec, err := e.GetSyncRunRecord(context.Background(), syncID) + require.NoError(t, err) + require.Nil(t, rec.GetEndedAt(), + "a failed stamp commit must not surface ended_at; the in-memory stamp on the local record must be unreachable") + unfinished, err := e.LatestUnfinishedSyncRecord(context.Background(), nil) + require.NoError(t, err) + require.NotNil(t, unfinished, + "the sync must stay discoverable as unfinished after a failed stamp commit") + require.Equal(t, syncID, unfinished.GetSyncId()) + }, + }, { // I7's drop arm: entitlements under one resource. Obligation: // the bare-id keyspace note (observable via the fresh flag — From 32d44120bee748cdc23c2a74beb92ff42b42ff42 Mon Sep 17 00:00:00 2001 From: Matt Kaniaris Date: Thu, 16 Jul 2026 16:17:09 -0600 Subject: [PATCH 17/17] extend raceEnabled guard to TestPoolGrowsFromDecoder sync.Pool randomly discards Puts under -race, same as the other two pool-reuse assertions already guarded in this file. Co-authored-by: Cursor --- pkg/dotc1z/pool_test.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pkg/dotc1z/pool_test.go b/pkg/dotc1z/pool_test.go index 092685335..432588660 100644 --- a/pkg/dotc1z/pool_test.go +++ b/pkg/dotc1z/pool_test.go @@ -341,7 +341,12 @@ func TestPoolGrowsFromDecoder(t *testing.T) { // Now the decoder pool should have a decoder. dec2, fromPool2 := getDecoder() - require.True(t, fromPool2, "NewDecoder.Close should have returned decoder to pool") + if !raceEnabled { + // See TestEncoderPool/TestDecoderPool: sync.Pool randomly + // discards Puts under -race; reuse is only deterministic + // without it. + require.True(t, fromPool2, "NewDecoder.Close should have returned decoder to pool") + } putDecoder(dec2) }