Assert transform result in system message via HTTP traffic

The 'should apply transform modifications' tests previously only verified
that the transform callback was invoked, not that the transformed content
actually reached the model. Now all 4 SDKs assert that TRANSFORM_MARKER
appears in the system message captured from HTTP traffic.

Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com>

Author: SteveSandersonMS

dotnet/test/SystemMessageTransformTests.cs

@@ -62,8 +62,6 @@ await session.SendAsync(new MessageOptions
     [Fact]
     public async Task Should_Apply_Transform_Modifications_To_Section_Content()
     {
-        var transformCallbackInvoked = false;
-
         var session = await CreateSessionAsync(new SessionConfig
         {
             OnPermissionRequest = PermissionHandler.ApproveAll,
@@ -76,8 +74,7 @@ public async Task Should_Apply_Transform_Modifications_To_Section_Content()
                     {
                         Transform = async (content) =>
                         {
-                            transformCallbackInvoked = true;
-                            return content + "TRANSFORM_MARKER";
+                            return content + "\nAlways end your reply with TRANSFORM_MARKER";
                         }
                     }
                 }
@@ -93,7 +90,11 @@ await session.SendAsync(new MessageOptions
 
         await TestHelper.GetFinalAssistantMessageAsync(session);
 
-        Assert.True(transformCallbackInvoked, "Expected identity transform callback to be invoked");
+        // Verify the transform result was actually applied to the system message
+        var traffic = await Ctx.GetExchangesAsync();
+        Assert.NotEmpty(traffic);
+        var systemMessage = GetSystemMessage(traffic[0]);
+        Assert.Contains("TRANSFORM_MARKER", systemMessage);
     }
 
     [Fact]

go/internal/e2e/system_message_transform_test.go

@@ -6,6 +6,7 @@ package e2e
 import (
 	"os"
 	"path/filepath"
+	"strings"
 	"sync"
 	"testing"
 
@@ -90,22 +91,14 @@ func TestSystemMessageTransform(t *testing.T) {
 	t.Run("should_apply_transform_modifications_to_section_content", func(t *testing.T) {
 		ctx.ConfigureForTest(t)
 
-		var receivedContent string
-		var mu sync.Mutex
-		transformCalled := false
-
 		session, err := client.CreateSession(t.Context(), &copilot.SessionConfig{
 			OnPermissionRequest: copilot.PermissionHandler.ApproveAll,
 			SystemMessage: &copilot.SystemMessageConfig{
 				Mode: "customize",
 				Sections: map[string]copilot.SectionOverride{
 					"identity": {
 						Transform: func(currentContent string) (string, error) {
-							mu.Lock()
-							transformCalled = true
-							receivedContent = currentContent
-							mu.Unlock()
-							return currentContent + "\nTRANSFORM_MARKER", nil
+							return currentContent + "\nAlways end your reply with TRANSFORM_MARKER", nil
 						},
 					},
 				},
@@ -121,22 +114,27 @@ func TestSystemMessageTransform(t *testing.T) {
 			t.Fatalf("Failed to write test file: %v", err)
 		}
 
-		_, err = session.SendAndWait(t.Context(), copilot.MessageOptions{
+		assistantMessage, err := session.SendAndWait(t.Context(), copilot.MessageOptions{
 			Prompt: "Read the contents of hello.txt",
 		})
 		if err != nil {
 			t.Fatalf("Failed to send message: %v", err)
 		}
 
-		mu.Lock()
-		defer mu.Unlock()
-
-		if !transformCalled {
-			t.Error("Expected transform callback to be invoked")
+		// Verify the transform result was actually applied to the system message
+		traffic, err := ctx.GetExchanges()
+		if err != nil {
+			t.Fatalf("Failed to get exchanges: %v", err)
+		}
+		if len(traffic) == 0 {
+			t.Fatal("Expected at least one exchange")
 		}
-		if receivedContent == "" {
-			t.Error("Expected transform to receive non-empty content")
+		systemMessage := getSystemMessage(traffic[0])
+		if !strings.Contains(systemMessage, "TRANSFORM_MARKER") {
+			t.Errorf("Expected system message to contain TRANSFORM_MARKER, got %q", systemMessage)
 		}
+
+		_ = assistantMessage
 	})
 
 	t.Run("should_work_with_static_overrides_and_transforms_together", func(t *testing.T) {

nodejs/test/e2e/system_message_transform.test.ts

@@ -5,11 +5,12 @@
 import { writeFile } from "fs/promises";
 import { join } from "path";
 import { describe, expect, it } from "vitest";
+import { ParsedHttpExchange } from "../../../test/harness/replayingCapiProxy.js";
 import { approveAll } from "../../src/index.js";
 import { createSdkTestContext } from "./harness/sdkTestContext.js";
 
 describe("System message transform", async () => {
-    const { copilotClient: client, workDir } = await createSdkTestContext();
+    const { copilotClient: client, openAiEndpoint, workDir } = await createSdkTestContext();
 
     it("should invoke transform callbacks with section content", async () => {
         const transformedSections: Record<string, string> = {};
@@ -53,20 +54,14 @@ describe("System message transform", async () => {
     });
 
     it("should apply transform modifications to section content", async () => {
-        let originalContent = "";
-        let transformedContent = "";
-
         const session = await client.createSession({
             onPermissionRequest: approveAll,
             systemMessage: {
                 mode: "customize",
                 sections: {
                     identity: {
                         action: (content: string) => {
-                            originalContent = content;
-                            // Append a custom instruction via transform
-                            transformedContent = content + "\nTRANSFORM_MARKER";
-                            return transformedContent;
+                            return content + "\nTRANSFORM_MARKER";
                         },
                     },
                 },
@@ -79,10 +74,10 @@ describe("System message transform", async () => {
             prompt: "Read the contents of hello.txt",
         });
 
-        // Verify the transform callback was invoked and modified the content
-        expect(originalContent.length).toBeGreaterThan(0);
-        expect(transformedContent).toContain("TRANSFORM_MARKER");
-        expect(transformedContent).toContain(originalContent);
+        // Verify the transform result was actually applied to the system message
+        const traffic = await openAiEndpoint.getExchanges();
+        const systemMessage = getSystemMessage(traffic[0]);
+        expect(systemMessage).toContain("TRANSFORM_MARKER");
 
         await session.disconnect();
     });
@@ -121,3 +116,10 @@ describe("System message transform", async () => {
         await session.disconnect();
     });
 });
+
+function getSystemMessage(exchange: ParsedHttpExchange): string | undefined {
+    const systemMessage = exchange.request.messages.find((m) => m.role === "system") as
+        | { role: "system"; content: string }
+        | undefined;
+    return systemMessage?.content;
+}

python/e2e/test_system_message_transform.py

@@ -59,10 +59,8 @@ async def test_should_apply_transform_modifications_to_section_content(
         self, ctx: E2ETestContext
     ):
         """Test that transform modifications are applied to the section content"""
-        identity_contents = []
 
         async def identity_transform(content: str) -> str:
-            identity_contents.append(content)
             return content + "\nTRANSFORM_MARKER"
 
         session = await ctx.client.create_session(
@@ -79,11 +77,10 @@ async def identity_transform(content: str) -> str:
 
         await session.send_and_wait("Read the contents of hello.txt")
 
-        # The transform callback should have been invoked
-        assert len(identity_contents) > 0
-
-        # The callback should have received content and returned modified content
-        assert all(len(c) > 0 for c in identity_contents)
+        # Verify the transform result was actually applied to the system message
+        traffic = await ctx.get_exchanges()
+        system_message = _get_system_message(traffic[0])
+        assert "TRANSFORM_MARKER" in system_message
 
         await session.disconnect()
 
@@ -116,3 +113,11 @@ async def identity_transform(content: str) -> str:
         assert len(identity_contents) > 0
 
         await session.disconnect()
+
+
+def _get_system_message(exchange: dict) -> str:
+    messages = exchange.get("request", {}).get("messages", [])
+    for msg in messages:
+        if msg.get("role") == "system":
+            return msg.get("content", "")
+    return ""