diff --git a/internal/controlplane/sts.go b/internal/controlplane/sts.go index 35fe26d3..ad01cab8 100644 --- a/internal/controlplane/sts.go +++ b/internal/controlplane/sts.go @@ -1022,7 +1022,9 @@ func (s *Server) HandleSTSToken(w http.ResponseWriter, r *http.Request) { if len(d.GetServedBy()) > 0 { callingRoles := s.nodeRoles(r.Context(), nodeRecord) if !api.EgressServedBy(d, callingRoles, nodeRecord.Labels) { - http.Error(w, fmt.Sprintf("calling node %s is not in served_by for egress://%s", nodeRecord.PeerID, destHost), http.StatusForbidden) + reason := fmt.Sprintf("calling node %s is not in served_by for egress://%s", nodeRecord.PeerID, destHost) + logSTSDenyBeforeBiscuit(nodeRecord.PeerID, &req, reason) + http.Error(w, reason, http.StatusForbidden) return } } @@ -1033,6 +1035,7 @@ func (s *Server) HandleSTSToken(w http.ResponseWriter, r *http.Request) { audience, audErr := s.resolveEgressAudience(r.Context(), req.Destination, req.Audience) if audErr != nil { + logSTSDenyBeforeBiscuit(nodeRecord.PeerID, &req, audErr.Error()) http.Error(w, audErr.Error(), http.StatusForbidden) return } @@ -1129,6 +1132,24 @@ func (s *Server) HandleSTSToken(w http.ResponseWriter, r *http.Request) { _, _ = w.Write(respData) } +// logSTSDenyBeforeBiscuit writes the Border Crossing deny line for a /sts/token +// refusal that happens after the calling node's PoP challenge has verified but +// before the caller's Biscuit is checked (served_by and audience). actor_node is +// proven at that point; client_peer_id is left empty because nothing in the +// Biscuit has been verified yet, and audience is the one the caller requested. +func logSTSDenyBeforeBiscuit(actorNode string, req *api.STSTokenRequest, reason string) { + logger.Infow("Border Crossing", + "direction", "outbound", + "endpoint", "/sts/token", + "decision", "deny", + "actor_node", actorNode, + "client_peer_id", "", + "audience", req.GetAudience(), + "destination", req.GetDestination(), + "reason", reason, + ) +} + func (s *Server) resolveEgressAudience(ctx context.Context, destination, reqAudience string) (string, error) { destHost := api.NormalizeMeshHost(strings.TrimPrefix(strings.TrimSpace(destination), api.EgressServicePrefix)) reqAud := strings.TrimSpace(reqAudience) diff --git a/internal/controlplane/sts_test.go b/internal/controlplane/sts_test.go index 783f449f..476f737b 100644 --- a/internal/controlplane/sts_test.go +++ b/internal/controlplane/sts_test.go @@ -1128,3 +1128,176 @@ func TestSTSTokenBorderCrossingNamesTheOriginPeer(t *testing.T) { t.Fatalf("got Border Crossing lines for decisions %v, want allow and deny", seen) } } + +// TestSTSTokenBorderCrossingDeniesBeforeBiscuit checks that the two /sts/token +// refusals that happen before the caller's Biscuit is verified (served_by and +// audience) still write a Border Crossing deny line naming the calling node, +// the destination, the requested audience and the reason, with client_peer_id +// left empty because nothing in the Biscuit has been verified yet. +func TestSTSTokenBorderCrossingDeniesBeforeBiscuit(t *testing.T) { + issuer, mintOIDC := startCustomMockOIDC(t) + srv, store, baseURL := setupTestServer(t, issuer) + defer func() { _ = srv.Close() }() + + const ( + servedByDest = "s3.amazonaws.com" + audienceDest = "bigquery.googleapis.com" + policyAud = "//iam.googleapis.com/projects/123/locations/global/workloadIdentityPools/agentmesh/providers/cp" + ) + roles := []*api.PolicyRole{ + {Name: api.RoleNode, AllowedServices: []string{"egress://" + servedByDest, "egress://" + audienceDest}, AllowedTargets: []string{"*"}}, + {Name: "egress-gateway", AllowedServices: []string{"egress://" + servedByDest}, AllowedTargets: []string{"*"}}, + } + bindings := []*api.PolicyBinding{ + {Role: api.RoleNode, Members: []string{api.SystemAuthenticated}}, + } + egress := []*api.EgressDestination{ + { + Name: servedByDest, + ServedBy: []string{"egress-gateway"}, + Broker: &api.CredentialBroker{ + Kind: &api.CredentialBroker_AwsAssumeRole{ + AwsAssumeRole: &api.AWSAssumeRole{RoleArn: "arn:aws:iam::123456789012:role/agentmesh-s3"}, + }, + }, + }, + { + Name: audienceDest, + Broker: &api.CredentialBroker{ + Kind: &api.CredentialBroker_OidcFederation{ + OidcFederation: &api.OIDCFederation{ + TokenEndpoint: "https://sts.googleapis.com/v1/token", + Audience: policyAud, + }, + }, + }, + }, + } + if err := store.SavePolicyDocument(context.Background(), roles, bindings, egress); err != nil { + t.Fatalf("SavePolicyDocument: %v", err) + } + + _, _, originBiscuit, _ := enrollTestNodeForSTS(t, baseURL, mintOIDC) + egressPriv, egressPeerID, egressBiscuit, _ := enrollTestNodeForSTS(t, baseURL, mintOIDC) + // A Task Biscuit bound to the origin peer: if either refusal were logged + // only after the Biscuit verified, the line would carry that peer. + taskBiscuit, err := identity.AttenuateBiscuit(originBiscuit, &api.TaskAuthorizationRule{ + Name: "export", + Rules: []*api.TaskRule{{AllowedServices: []string{"egress://" + servedByDest, "egress://" + audienceDest}}}, + }) + if err != nil { + t.Fatalf("attenuate the Member Biscuit: %v", err) + } + + const subsystem = "agentmesh-control-plane" + prevLevel, err := golog.SubsystemLevelName(subsystem) + if err != nil { + t.Fatalf("read the %s log level: %v", subsystem, err) + } + if err := golog.SetLogLevel(subsystem, "info"); err != nil { + t.Fatalf("set the %s log level: %v", subsystem, err) + } + t.Cleanup(func() { _ = golog.SetLogLevel(subsystem, prevLevel) }) + + // The pipe sees every logger, including other tests running in parallel, + // so the reader keeps only this egress node's /sts/token lines. + pipe := golog.NewPipeReader() + t.Cleanup(func() { _ = pipe.Close() }) + crossings := make(chan map[string]any, 4) + go func() { + scanner := bufio.NewScanner(pipe) + scanner.Buffer(make([]byte, 0, 64*1024), 1<<20) + for scanner.Scan() { + var entry map[string]any + if json.Unmarshal(scanner.Bytes(), &entry) != nil { + continue + } + if entry["msg"] == "Border Crossing" && entry["endpoint"] == "/sts/token" && entry["actor_node"] == egressPeerID.String() { + crossings <- entry + } + } + }() + + postSTS := func(destination, audience string) int { + t.Helper() + ts := time.Now().UnixMilli() + sig, err := egressPriv.Sign(api.STSTokenChallenge(egressPeerID.String(), ts)) + if err != nil { + t.Fatalf("sign the STS challenge: %v", err) + } + reqBytes, err := proto.Marshal(&api.STSTokenRequest{ + Biscuit: taskBiscuit, + Destination: destination, + Audience: audience, + ChallengeUnixMs: ts, + ChallengeSignature: sig, + }) + if err != nil { + t.Fatalf("marshal STSTokenRequest: %v", err) + } + req, err := http.NewRequest(http.MethodPost, baseURL+"/sts/token", bytes.NewReader(reqBytes)) + if err != nil { + t.Fatalf("build POST /sts/token: %v", err) + } + req.Header.Set("Authorization", "Bearer "+base64.StdEncoding.EncodeToString(egressBiscuit)) + resp, err := http.DefaultClient.Do(req) + if err != nil { + t.Fatalf("POST /sts/token for %s: %v", destination, err) + } + _ = resp.Body.Close() + return resp.StatusCode + } + + cases := []struct { + name string + destination string + audience string + reason string + }{ + { + name: "served_by", + destination: servedByDest, + audience: "sts.amazonaws.com", + reason: "calling node " + egressPeerID.String() + " is not in served_by for egress://" + servedByDest, + }, + { + name: "audience", + destination: audienceDest, + audience: "https://evil.example.com", + reason: `requested audience "https://evil.example.com" does not match policy audience for egress://` + audienceDest, + }, + } + for _, tc := range cases { + if status := postSTS(tc.destination, tc.audience); status != http.StatusForbidden { + t.Fatalf("%s: POST /sts/token for %s returned %d, want 403", tc.name, tc.destination, status) + } + select { + case entry := <-crossings: + if entry["decision"] != "deny" { + t.Fatalf("%s: Border Crossing decision = %v, want deny", tc.name, entry["decision"]) + } + if entry["direction"] != "outbound" { + t.Errorf("%s: Border Crossing direction = %v, want outbound", tc.name, entry["direction"]) + } + if entry["destination"] != tc.destination { + t.Errorf("%s: Border Crossing destination = %v, want %s", tc.name, entry["destination"], tc.destination) + } + if entry["audience"] != tc.audience { + t.Errorf("%s: Border Crossing audience = %v, want the requested %s", tc.name, entry["audience"], tc.audience) + } + if entry["reason"] != tc.reason { + t.Errorf("%s: Border Crossing reason = %v, want %q", tc.name, entry["reason"], tc.reason) + } + if cp, ok := entry["client_peer_id"]; ok && cp != "" { + t.Errorf("%s: Border Crossing client_peer_id = %v, want empty", tc.name, cp) + } + case <-time.After(5 * time.Second): + t.Fatalf("%s: no Border Crossing line for the refused /sts/token request", tc.name) + } + } + select { + case entry := <-crossings: + t.Fatalf("got an extra Border Crossing line %v, want exactly one per refusal", entry) + case <-time.After(200 * time.Millisecond): + } +} diff --git a/site/content/docs/contributing/security-architecture.md b/site/content/docs/contributing/security-architecture.md index 1b5fe9d5..76c500af 100644 --- a/site/content/docs/contributing/security-architecture.md +++ b/site/content/docs/contributing/security-architecture.md @@ -344,7 +344,7 @@ as a standard OIDC issuer: | :--- | :--- | | **Signing key** | **ES256** (P-256 ECDSA), rotated with overlap and `kid` (`internal/controlplane/sts.go`). Google Cloud and AWS STS accept RS256 and ES256, not Ed25519; Ed25519 remains the Biscuit authority key. | | **Discovery & JWKS** | `GET /.well-known/openid-configuration` and `GET /jwks` on the control plane. For air-gapped or private control planes, the JWKS can be uploaded directly to the cloud workload identity pool. | -| **Mint endpoint** | `POST /sts/token` (protobuf over HTTP): accepts the caller's Biscuit and `destination` name, verifies the Biscuit and the full `tar_block` chain against `egress://`, writes a `Border Crossing` audit log, and returns a short-lived ES256 JWT. In the `/sts/token` lines, `actor_node` is the egress node that called it and `client_peer_id` is the origin peer the caller's Biscuit is bound to, on allow and on deny once the Biscuit has verified; in the `/token/exchange` lines, `actor_node` is the exchanging node, which becomes that Biscuit's `client_peer_id`. | +| **Mint endpoint** | `POST /sts/token` (protobuf over HTTP): accepts the caller's Biscuit and `destination` name, verifies the Biscuit and the full `tar_block` chain against `egress://`, writes a `Border Crossing` audit log, and returns a short-lived ES256 JWT. In the `/sts/token` lines, `actor_node` is the egress node that called it and `client_peer_id` is the origin peer the caller's Biscuit is bound to, on allow and on deny once the Biscuit has verified. The `served_by` and audience refusals, which happen after the node's PoP challenge but before the Biscuit is checked, also write a deny line, with the requested `audience` and `client_peer_id` empty; in the `/token/exchange` lines, `actor_node` is the exchanging node, which becomes that Biscuit's `client_peer_id`. | | **JWT claims** | `iss` = control plane URL; `sub` = caller principal (`email` or `issuer#subject`); `act` = `{"sub": ""}`; `aud` = destination audience from `EgressDestination` (one audience per destination); `mesh_roles` = authority-block roles (mapped to `google.groups` for `principalSet` bindings); `mesh_task` = innermost `TaskAuthorizationRule.name` (audit-only). | | **TAR narrows, never selects** | Nothing in a holder-appended `tar_block` can select a broker, role ARN, service account, or audience; every claim that cloud IAM binds on (`sub`, `mesh_roles`, `act.sub`, `aud`) comes from Block 0 or the control plane's `EgressDestination` policy. | | **Caching** | The egress node caches minted border JWTs and upstream cloud tokens by Biscuit SHA-256 digest, keeping the control plane off the per-request hot path. |