From 3fa9a277ff162fa8a42ba1c90968499fa0d57b55 Mon Sep 17 00:00:00 2001 From: Anpoo Sivanadi Date: Tue, 20 Oct 2026 10:29:43 -0700 Subject: [PATCH] controlplane: log the served_by and audience refusals at /sts/token The served_by check and the resolveEgressAudience check in HandleSTSToken return 403 before the caller's Biscuit is verified, and neither wrote a Border Crossing line, so an audit join on the deny lines could not see a node probing destinations it does not serve. Both refusals now write a deny line in the same shape as the existing /sts/token one: direction, endpoint, decision, actor_node (proven by the PoP challenge), destination, the requested audience and the reason. client_peer_id is empty because nothing in the Biscuit has been verified at that point. Refusals before the challenge verifies stay as they are. TestSTSTokenBorderCrossingDeniesBeforeBiscuit covers both refusals and checks for exactly one deny line each. Refs #742 Signed-off-by: Anpoo Sivanadi --- internal/controlplane/sts.go | 23 ++- internal/controlplane/sts_test.go | 173 ++++++++++++++++++ .../contributing/security-architecture.md | 2 +- 3 files changed, 196 insertions(+), 2 deletions(-) diff --git a/internal/controlplane/sts.go b/internal/controlplane/sts.go index 35fe26d3..ad01cab8 100644 --- a/internal/controlplane/sts.go +++ b/internal/controlplane/sts.go @@ -1022,7 +1022,9 @@ func (s *Server) HandleSTSToken(w http.ResponseWriter, r *http.Request) { if len(d.GetServedBy()) > 0 { callingRoles := s.nodeRoles(r.Context(), nodeRecord) if !api.EgressServedBy(d, callingRoles, nodeRecord.Labels) { - http.Error(w, fmt.Sprintf("calling node %s is not in served_by for egress://%s", nodeRecord.PeerID, destHost), http.StatusForbidden) + reason := fmt.Sprintf("calling node %s is not in served_by for egress://%s", nodeRecord.PeerID, destHost) + logSTSDenyBeforeBiscuit(nodeRecord.PeerID, &req, reason) + http.Error(w, reason, http.StatusForbidden) return } } @@ -1033,6 +1035,7 @@ func (s *Server) HandleSTSToken(w http.ResponseWriter, r *http.Request) { audience, audErr := s.resolveEgressAudience(r.Context(), req.Destination, req.Audience) if audErr != nil { + logSTSDenyBeforeBiscuit(nodeRecord.PeerID, &req, audErr.Error()) http.Error(w, audErr.Error(), http.StatusForbidden) return } @@ -1129,6 +1132,24 @@ func (s *Server) HandleSTSToken(w http.ResponseWriter, r *http.Request) { _, _ = w.Write(respData) } +// logSTSDenyBeforeBiscuit writes the Border Crossing deny line for a /sts/token +// refusal that happens after the calling node's PoP challenge has verified but +// before the caller's Biscuit is checked (served_by and audience). actor_node is +// proven at that point; client_peer_id is left empty because nothing in the +// Biscuit has been verified yet, and audience is the one the caller requested. +func logSTSDenyBeforeBiscuit(actorNode string, req *api.STSTokenRequest, reason string) { + logger.Infow("Border Crossing", + "direction", "outbound", + "endpoint", "/sts/token", + "decision", "deny", + "actor_node", actorNode, + "client_peer_id", "", + "audience", req.GetAudience(), + "destination", req.GetDestination(), + "reason", reason, + ) +} + func (s *Server) resolveEgressAudience(ctx context.Context, destination, reqAudience string) (string, error) { destHost := api.NormalizeMeshHost(strings.TrimPrefix(strings.TrimSpace(destination), api.EgressServicePrefix)) reqAud := strings.TrimSpace(reqAudience) diff --git a/internal/controlplane/sts_test.go b/internal/controlplane/sts_test.go index 783f449f..476f737b 100644 --- a/internal/controlplane/sts_test.go +++ b/internal/controlplane/sts_test.go @@ -1128,3 +1128,176 @@ func TestSTSTokenBorderCrossingNamesTheOriginPeer(t *testing.T) { t.Fatalf("got Border Crossing lines for decisions %v, want allow and deny", seen) } } + +// TestSTSTokenBorderCrossingDeniesBeforeBiscuit checks that the two /sts/token +// refusals that happen before the caller's Biscuit is verified (served_by and +// audience) still write a Border Crossing deny line naming the calling node, +// the destination, the requested audience and the reason, with client_peer_id +// left empty because nothing in the Biscuit has been verified yet. +func TestSTSTokenBorderCrossingDeniesBeforeBiscuit(t *testing.T) { + issuer, mintOIDC := startCustomMockOIDC(t) + srv, store, baseURL := setupTestServer(t, issuer) + defer func() { _ = srv.Close() }() + + const ( + servedByDest = "s3.amazonaws.com" + audienceDest = "bigquery.googleapis.com" + policyAud = "//iam.googleapis.com/projects/123/locations/global/workloadIdentityPools/agentmesh/providers/cp" + ) + roles := []*api.PolicyRole{ + {Name: api.RoleNode, AllowedServices: []string{"egress://" + servedByDest, "egress://" + audienceDest}, AllowedTargets: []string{"*"}}, + {Name: "egress-gateway", AllowedServices: []string{"egress://" + servedByDest}, AllowedTargets: []string{"*"}}, + } + bindings := []*api.PolicyBinding{ + {Role: api.RoleNode, Members: []string{api.SystemAuthenticated}}, + } + egress := []*api.EgressDestination{ + { + Name: servedByDest, + ServedBy: []string{"egress-gateway"}, + Broker: &api.CredentialBroker{ + Kind: &api.CredentialBroker_AwsAssumeRole{ + AwsAssumeRole: &api.AWSAssumeRole{RoleArn: "arn:aws:iam::123456789012:role/agentmesh-s3"}, + }, + }, + }, + { + Name: audienceDest, + Broker: &api.CredentialBroker{ + Kind: &api.CredentialBroker_OidcFederation{ + OidcFederation: &api.OIDCFederation{ + TokenEndpoint: "https://sts.googleapis.com/v1/token", + Audience: policyAud, + }, + }, + }, + }, + } + if err := store.SavePolicyDocument(context.Background(), roles, bindings, egress); err != nil { + t.Fatalf("SavePolicyDocument: %v", err) + } + + _, _, originBiscuit, _ := enrollTestNodeForSTS(t, baseURL, mintOIDC) + egressPriv, egressPeerID, egressBiscuit, _ := enrollTestNodeForSTS(t, baseURL, mintOIDC) + // A Task Biscuit bound to the origin peer: if either refusal were logged + // only after the Biscuit verified, the line would carry that peer. + taskBiscuit, err := identity.AttenuateBiscuit(originBiscuit, &api.TaskAuthorizationRule{ + Name: "export", + Rules: []*api.TaskRule{{AllowedServices: []string{"egress://" + servedByDest, "egress://" + audienceDest}}}, + }) + if err != nil { + t.Fatalf("attenuate the Member Biscuit: %v", err) + } + + const subsystem = "agentmesh-control-plane" + prevLevel, err := golog.SubsystemLevelName(subsystem) + if err != nil { + t.Fatalf("read the %s log level: %v", subsystem, err) + } + if err := golog.SetLogLevel(subsystem, "info"); err != nil { + t.Fatalf("set the %s log level: %v", subsystem, err) + } + t.Cleanup(func() { _ = golog.SetLogLevel(subsystem, prevLevel) }) + + // The pipe sees every logger, including other tests running in parallel, + // so the reader keeps only this egress node's /sts/token lines. + pipe := golog.NewPipeReader() + t.Cleanup(func() { _ = pipe.Close() }) + crossings := make(chan map[string]any, 4) + go func() { + scanner := bufio.NewScanner(pipe) + scanner.Buffer(make([]byte, 0, 64*1024), 1<<20) + for scanner.Scan() { + var entry map[string]any + if json.Unmarshal(scanner.Bytes(), &entry) != nil { + continue + } + if entry["msg"] == "Border Crossing" && entry["endpoint"] == "/sts/token" && entry["actor_node"] == egressPeerID.String() { + crossings <- entry + } + } + }() + + postSTS := func(destination, audience string) int { + t.Helper() + ts := time.Now().UnixMilli() + sig, err := egressPriv.Sign(api.STSTokenChallenge(egressPeerID.String(), ts)) + if err != nil { + t.Fatalf("sign the STS challenge: %v", err) + } + reqBytes, err := proto.Marshal(&api.STSTokenRequest{ + Biscuit: taskBiscuit, + Destination: destination, + Audience: audience, + ChallengeUnixMs: ts, + ChallengeSignature: sig, + }) + if err != nil { + t.Fatalf("marshal STSTokenRequest: %v", err) + } + req, err := http.NewRequest(http.MethodPost, baseURL+"/sts/token", bytes.NewReader(reqBytes)) + if err != nil { + t.Fatalf("build POST /sts/token: %v", err) + } + req.Header.Set("Authorization", "Bearer "+base64.StdEncoding.EncodeToString(egressBiscuit)) + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("POST /sts/token for %s: %v", destination, err) + } + _ = resp.Body.Close() + return resp.StatusCode + } + + cases := []struct { + name string + destination string + audience string + reason string + }{ + { + name: "served_by", + destination: servedByDest, + audience: "sts.amazonaws.com", + reason: "calling node " + egressPeerID.String() + " is not in served_by for egress://" + servedByDest, + }, + { + name: "audience", + destination: audienceDest, + audience: "https://evil.example.com", + reason: `requested audience "https://evil.example.com" does not match policy audience for egress://` + audienceDest, + }, + } + for _, tc := range cases { + if status := postSTS(tc.destination, tc.audience); status != http.StatusForbidden { + t.Fatalf("%s: POST /sts/token for %s returned %d, want 403", tc.name, tc.destination, status) + } + select { + case entry := <-crossings: + if entry["decision"] != "deny" { + t.Fatalf("%s: Border Crossing decision = %v, want deny", tc.name, entry["decision"]) + } + if entry["direction"] != "outbound" { + t.Errorf("%s: Border Crossing direction = %v, want outbound", tc.name, entry["direction"]) + } + if entry["destination"] != tc.destination { + t.Errorf("%s: Border Crossing destination = %v, want %s", tc.name, entry["destination"], tc.destination) + } + if entry["audience"] != tc.audience { + t.Errorf("%s: Border Crossing audience = %v, want the requested %s", tc.name, entry["audience"], tc.audience) + } + if entry["reason"] != tc.reason { + t.Errorf("%s: Border Crossing reason = %v, want %q", tc.name, entry["reason"], tc.reason) + } + if cp, ok := entry["client_peer_id"]; ok && cp != "" { + t.Errorf("%s: Border Crossing client_peer_id = %v, want empty", tc.name, cp) + } + case <-time.After(5 * time.Second): + t.Fatalf("%s: no Border Crossing line for the refused /sts/token request", tc.name) + } + } + select { + case entry := <-crossings: + t.Fatalf("got an extra Border Crossing line %v, want exactly one per refusal", entry) + case <-time.After(200 * time.Millisecond): + } +} diff --git a/site/content/docs/contributing/security-architecture.md b/site/content/docs/contributing/security-architecture.md index 1b5fe9d5..76c500af 100644 --- a/site/content/docs/contributing/security-architecture.md +++ b/site/content/docs/contributing/security-architecture.md @@ -344,7 +344,7 @@ as a standard OIDC issuer: | :--- | :--- | | **Signing key** | **ES256** (P-256 ECDSA), rotated with overlap and `kid` (`internal/controlplane/sts.go`). Google Cloud and AWS STS accept RS256 and ES256, not Ed25519; Ed25519 remains the Biscuit authority key. | | **Discovery & JWKS** | `GET /.well-known/openid-configuration` and `GET /jwks` on the control plane. For air-gapped or private control planes, the JWKS can be uploaded directly to the cloud workload identity pool. | -| **Mint endpoint** | `POST /sts/token` (protobuf over HTTP): accepts the caller's Biscuit and `destination` name, verifies the Biscuit and the full `tar_block` chain against `egress://`, writes a `Border Crossing` audit log, and returns a short-lived ES256 JWT. In the `/sts/token` lines, `actor_node` is the egress node that called it and `client_peer_id` is the origin peer the caller's Biscuit is bound to, on allow and on deny once the Biscuit has verified; in the `/token/exchange` lines, `actor_node` is the exchanging node, which becomes that Biscuit's `client_peer_id`. | +| **Mint endpoint** | `POST /sts/token` (protobuf over HTTP): accepts the caller's Biscuit and `destination` name, verifies the Biscuit and the full `tar_block` chain against `egress://`, writes a `Border Crossing` audit log, and returns a short-lived ES256 JWT. In the `/sts/token` lines, `actor_node` is the egress node that called it and `client_peer_id` is the origin peer the caller's Biscuit is bound to, on allow and on deny once the Biscuit has verified. The `served_by` and audience refusals, which happen after the node's PoP challenge but before the Biscuit is checked, also write a deny line, with the requested `audience` and `client_peer_id` empty; in the `/token/exchange` lines, `actor_node` is the exchanging node, which becomes that Biscuit's `client_peer_id`. | | **JWT claims** | `iss` = control plane URL; `sub` = caller principal (`email` or `issuer#subject`); `act` = `{"sub": ""}`; `aud` = destination audience from `EgressDestination` (one audience per destination); `mesh_roles` = authority-block roles (mapped to `google.groups` for `principalSet` bindings); `mesh_task` = innermost `TaskAuthorizationRule.name` (audit-only). | | **TAR narrows, never selects** | Nothing in a holder-appended `tar_block` can select a broker, role ARN, service account, or audience; every claim that cloud IAM binds on (`sub`, `mesh_roles`, `act.sub`, `aud`) comes from Block 0 or the control plane's `EgressDestination` policy. | | **Caching** | The egress node caches minted border JWTs and upstream cloud tokens by Biscuit SHA-256 digest, keeping the control plane off the per-request hot path. |