feat(events): attribute explicit sandbox kills with requester IP and user agent (rebase of #3465)

packages/api/internal/handlers/sandbox_kill.go

@@ -62,8 +62,10 @@ func (a *APIStore) DeleteSandboxesSandboxID(
 	killedOrRemoved := false
 
 	err = a.orchestrator.RemoveSandbox(ctx, teamID, sandboxID, sandbox.RemoveOpts{
-		Action: sandbox.StateActionKill,
-		Reason: sandbox.KillReasonRequest,
+		Action:      sandbox.StateActionKill,
+		Reason:      sandbox.KillReasonRequest,
+		RequesterIP: c.ClientIP(),
+		UserAgent:   c.Request.UserAgent(),
 	})
 	switch {
 	case err == nil:

packages/api/internal/orchestrator/create_instance.go

@@ -607,6 +607,7 @@ func (o *Orchestrator) CreateSandbox(
 			sandbox.StateActionKill,
 			sandbox.KillReasonUnknown,
 			false, // kill: no snapshot
+			"", "",
 		)
 		if killErr != nil {
 			logger.L().Error(ctx, "Error removing memory-restored sandbox after unhonored filesystem-boot demand",
@@ -637,6 +638,7 @@ func (o *Orchestrator) CreateSandbox(
 				sandbox.StateActionKill,
 				sandbox.KillReasonUnknown,
 				false, // kill: no snapshot
+				"", "",
 			)
 			if killErr != nil {
 				logger.L().Error(ctx, "Error removing sandbox",

packages/api/internal/orchestrator/delete_instance.go

@@ -149,7 +149,7 @@ func (o *Orchestrator) RemoveSandbox(ctx context.Context, teamID uuid.UUID, sand
 		o.sandboxStore.Remove(context.WithoutCancel(ctx), teamID, sandboxID)
 		go o.analyticsRemove(context.WithoutCancel(ctx), sbx, opts.Action)
 	}()
-	err = o.removeSandboxFromNode(ctx, sbx, opts.Action, opts.Reason, opts.FilesystemOnly)
+	err = o.removeSandboxFromNode(ctx, sbx, opts.Action, opts.Reason, opts.FilesystemOnly, opts.RequesterIP, opts.UserAgent)
 	if err != nil {
 		if errors.Is(err, PauseQueueExhaustedError{}) {
 			if restoreOnRefusal {
@@ -231,7 +231,7 @@ func (o *Orchestrator) killRefusedSandbox(ctx context.Context, sbx sandbox.Sandb
 		return
 	}
 
-	if err := o.killSandboxOnNode(ctx, node, sbx.ToNodeSandbox(), sandbox.KillReasonOrphaned); err != nil {
+	if err := o.killSandboxOnNode(ctx, node, sbx.ToNodeSandbox(), sandbox.KillReasonOrphaned, "", ""); err != nil {
 		logger.L().Error(ctx, "failed to kill a refused sandbox after a failed restore", zap.Error(err), logger.WithSandboxID(sbx.SandboxID))
 	}
 }
@@ -276,6 +276,8 @@ func (o *Orchestrator) removeSandboxFromNode(
 	stateAction sandbox.StateAction,
 	reason sandbox.KillReason,
 	filesystemOnly bool,
+	requesterIP string,
+	userAgent string,
 ) error {
 	ctx, span := tracer.Start(ctx, "remove-sandbox-from-node")
 	defer span.End()
@@ -304,7 +306,7 @@ func (o *Orchestrator) removeSandboxFromNode(
 		err := o.pauseSandbox(ctx, node, sbx, filesystemOnly)
 		if err != nil {
 			if dberrors.IsForeignKeyViolation(err) {
-				killErr := o.killSandboxOnNode(ctx, node, sbx.ToNodeSandbox(), sandbox.KillReasonBaseTemplateMissing)
+				killErr := o.killSandboxOnNode(ctx, node, sbx.ToNodeSandbox(), sandbox.KillReasonBaseTemplateMissing, "", "")
 				logger.L().Error(ctx, "Pause failed due to missing base template, killed sandbox as fallback",
 					logger.WithSandboxID(sbx.SandboxID),
 					zap.String("base_template_id", sbx.BaseTemplateID),
@@ -321,7 +323,7 @@ func (o *Orchestrator) removeSandboxFromNode(
 
 		return nil
 	case sandbox.StateActionKill:
-		return o.killSandboxOnNode(ctx, node, sbx.ToNodeSandbox(), reason)
+		return o.killSandboxOnNode(ctx, node, sbx.ToNodeSandbox(), reason, requesterIP, userAgent)
 	}
 
 	return nil
@@ -339,7 +341,7 @@ func (o *Orchestrator) killOrphanSandbox(ctx context.Context, sbx sandbox.NodeSa
 		return
 	}
 
-	err := o.killSandboxOnNode(ctx, node, sbx, sandbox.KillReasonOrphaned)
+	err := o.killSandboxOnNode(ctx, node, sbx, sandbox.KillReasonOrphaned, "", "")
 	if err != nil {
 		logger.L().Error(ctx, "Failed to kill orphan sandbox on node",
 			zap.Error(err),
@@ -355,12 +357,20 @@ func (o *Orchestrator) killSandboxOnNode(
 	node *nodemanager.Node,
 	sbx sandbox.NodeSandbox,
 	reason sandbox.KillReason,
+	requesterIP string,
+	userAgent string,
 ) error {
 	killReason := reason.String()
 	req := &orchestrator.SandboxDeleteRequest{
 		SandboxId:  sbx.SandboxID,
 		KillReason: &killReason,
 	}
+	if requesterIP != "" {
+		req.RequesterIp = &requesterIP
+	}
+	if userAgent != "" {
+		req.RequesterUserAgent = &userAgent
+	}
 
 	client, ctx := node.GetSandboxClient(ctx, sbx.TeamID.String())
 	_, err := client.Sandbox.Delete(ctx, req)

packages/api/internal/sandbox/sandboxtypes/states.go

@@ -84,13 +84,15 @@ type RemoveOpts struct {
 	// When set, StartRemoving fails with ErrExecutionMismatch if the stored
 	// record carries a different execution ID.
 	//
-	// For callers that decide from a snapshot taken earlier — a background
-	// scan, a queued batch — the sandbox ID alone is not a stable identity:
-	// the record can be removed and the ID reused by a resume before the
-	// removal runs, and the transition would then land on a live sandbox that
-	// was never in scope. Empty means "remove whatever is stored", which is
-	// correct for callers acting on a fresh read or on user intent.
 	ExpectExecutionID string
+
+	// RequesterIP and UserAgent attribute an explicit API kill (KillReasonRequest)
+	// to the client that issued DELETE /sandboxes/{id}, so the sandbox.lifecycle.killed
+	// event can record who initiated it. Both are empty for timeout, eviction and
+	// orchestrator-internal kills. They ride the event's existing per-team TTL, so
+	// no new unbounded store of client data is introduced.
+	RequesterIP string
+	UserAgent   string
 }
 
 var AllowedTransitions = map[State]map[State]bool{

packages/orchestrator/orchestrator.proto

@@ -210,6 +210,8 @@ message SandboxUpdateRequest {
 message SandboxDeleteRequest {
   string sandbox_id = 1;
   optional string kill_reason = 2;
+  optional string requester_ip = 3;
+  optional string requester_user_agent = 4;
 }
 
 message SandboxPauseRequest {

packages/orchestrator/pkg/server/requester_attribution_test.go

@@ -0,0 +1,44 @@
+package server
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/e2b-dev/infra/packages/shared/pkg/grpc/orchestrator"
+)
+
+// TestRequesterAttribution pins that an explicit API kill records the client IP
+// and User-Agent on the killed event, while a kill without them (timeout,
+// eviction, internal) adds nothing.
+func TestRequesterAttribution(t *testing.T) {
+	t.Parallel()
+
+	ip := "203.0.113.7"
+	ua := "e2b-python-sdk/1.2.3"
+
+	t.Run("request kill records requester fields", func(t *testing.T) {
+		t.Parallel()
+		in := &orchestrator.SandboxDeleteRequest{
+			SandboxId:          "isbx",
+			RequesterIp:        &ip,
+			RequesterUserAgent: &ua,
+		}
+		data := map[string]any{}
+		requesterAttribution(in)(data)
+		assert.Equal(t, ip, data["requester_ip"])
+		assert.Equal(t, ua, data["requester_user_agent"])
+	})
+
+	t.Run("kill without requester adds nothing", func(t *testing.T) {
+		t.Parallel()
+		in := &orchestrator.SandboxDeleteRequest{SandboxId: "isbx"}
+		data := map[string]any{}
+		requesterAttribution(in)(data)
+		_, hasIP := data["requester_ip"]
+		_, hasUA := data["requester_user_agent"]
+		assert.False(t, hasIP, "requester_ip must be absent")
+		assert.False(t, hasUA, "requester_user_agent must be absent")
+		assert.Empty(t, data)
+	})
+}

packages/orchestrator/pkg/server/sandboxes.go

@@ -863,7 +863,7 @@ func (s *Server) Delete(ctxConn context.Context, in *orchestrator.SandboxDeleteR
 		}
 	}()
 
-	s.emitSandboxKilled(ctx, sbx, killReason)
+	s.emitSandboxKilled(ctx, sbx, killReason, requesterAttribution(in))
 
 	return &emptypb.Empty{}, nil
 }
@@ -940,10 +940,13 @@ func (s *Server) reportCheckpointStepError(ctx context.Context, msg string, err
 // are the Delete RPC and the paths that destroy a live-registered sandbox
 // in-request without passing through it: the pause-path seal-failure kill and
 // the in-place resume-failure teardown.
-func (s *Server) emitSandboxKilled(ctx context.Context, sbx *sandbox.Sandbox, killReason string) {
+func (s *Server) emitSandboxKilled(ctx context.Context, sbx *sandbox.Sandbox, killReason string, decorate ...func(map[string]any)) {
 	teamID, buildId, eventsTTLDays, eventData := s.prepareSandboxEventData(ctx, sbx)
 	eventData[executionEventDataKey] = s.getSandboxExecutionData(sbx)
 	addKillReason(eventData, killReason)
+	for _, d := range decorate {
+		d(eventData)
+	}
 	recordSandboxKill(ctx, s.sandboxKilledCounter, sbx.Runtime.TeamID, killReason)
 	s.publishEventAsync(
 		ctx,
@@ -967,6 +970,23 @@ func (s *Server) emitSandboxKilled(ctx context.Context, sbx *sandbox.Sandbox, ki
 
 // addKillReason records the kill reason on killed events. Empty input is
 // normalized to "unknown" so killed events always carry a kill_reason key.
+// requesterAttribution returns an event-data decorator that records who issued
+// an explicit API kill. The API populates requester_ip / requester_user_agent
+// only on a user-issued DELETE (KillReasonRequest); for timeout, eviction and
+// orchestrator-internal kills both are empty and the decorator adds nothing.
+// The fields ride the killed event's existing per-team TTL, so no new unbounded
+// store of client data is created.
+func requesterAttribution(in *orchestrator.SandboxDeleteRequest) func(map[string]any) {
+	return func(eventData map[string]any) {
+		if ip := in.GetRequesterIp(); ip != "" {
+			eventData["requester_ip"] = ip
+		}
+		if ua := in.GetRequesterUserAgent(); ua != "" {
+			eventData["requester_user_agent"] = ua
+		}
+	}
+}
+
 func addKillReason(eventData map[string]any, killReason string) {
 	if killReason == "" {
 		killReason = killReasonUnknown