feat(attributes): add gen ai data migrations

javascript/sentry-conventions/src/attributes.ts

@@ -19613,6 +19613,9 @@ export const ATTRIBUTE_METADATA: Record<AttributeName, AttributeMetadata> = {
     visibility: 'public',
     example:
       '[{"role": "user", "parts": [{"type": "text", "content": "Weather in Paris?"}]}, {"role": "assistant", "parts": [{"type": "tool_call", "id": "call_VSPygqKTWdrhaFErNvMV18Yl", "name": "get_weather", "arguments": {"location": "Paris"}}]}, {"role": "tool", "parts": [{"type": "tool_call_response", "id": "call_VSPygqKTWdrhaFErNvMV18Yl", "result": "rainy, 57°F"}]}]',
+    migration: {
+      targetOf: ['gen_ai_request_messages_to_input_messages'],
+    },
     aliases: [AI_TEXTS],
     changelog: [
       { version: '0.5.0', prs: [264] },
@@ -19660,6 +19663,9 @@ export const ATTRIBUTE_METADATA: Record<AttributeName, AttributeMetadata> = {
     visibility: 'public',
     example:
       '[{"role": "assistant", "parts": [{"type": "text", "content": "The weather in Paris is currently rainy with a temperature of 57°F."}], "finish_reason": "stop"}]',
+    migration: {
+      targetOf: ['gen_ai_response_to_output_messages'],
+    },
     changelog: [{ version: '0.4.0', prs: [221] }],
   },
   [GEN_AI_PIPELINE_NAME]: {
@@ -19773,6 +19779,9 @@ export const ATTRIBUTE_METADATA: Record<AttributeName, AttributeMetadata> = {
     visibility: 'public',
     example:
       '[{"role": "system", "content": "Generate a random number."}, {"role": "user", "content": [{"text": "Generate a random number between 0 and 10.", "type": "text"}]}, {"role": "tool", "content": {"toolCallId": "1", "toolName": "Weather", "output": "rainy"}}]',
+    migration: {
+      sourceFor: ['gen_ai_request_messages_to_input_messages'],
+    },
     deprecation: {
       replacement: 'gen_ai.input.messages',
       status: 'normalize',
@@ -19940,6 +19949,9 @@ export const ATTRIBUTE_METADATA: Record<AttributeName, AttributeMetadata> = {
     visibility: 'public',
     example:
       '["The weather in Paris is rainy and overcast, with temperatures around 57°F", "The weather in London is sunny and warm, with temperatures around 65°F"]',
+    migration: {
+      sourceFor: ['gen_ai_response_to_output_messages'],
+    },
     deprecation: {
       replacement: 'gen_ai.output.messages',
       status: 'normalize',
@@ -20003,6 +20015,9 @@ export const ATTRIBUTE_METADATA: Record<AttributeName, AttributeMetadata> = {
     isInOtel: false,
     visibility: 'public',
     example: '[{"name": "get_weather", "arguments": {"location": "Paris"}}]',
+    migration: {
+      sourceFor: ['gen_ai_response_to_output_messages'],
+    },
     deprecation: {
       replacement: 'gen_ai.output.messages',
       status: 'normalize',

javascript/sentry-conventions/src/migrations.ts

@@ -1,7 +1,11 @@
-import { ATTRIBUTE_METADATA, type AttributeValue } from './attributes';
+import { ATTRIBUTE_METADATA, GEN_AI_INPUT_MESSAGES, GEN_AI_OUTPUT_MESSAGES, type AttributeValue } from './attributes';
 
 export type AttributeMigrationId = string;
 
+const GEN_AI_REQUEST_MESSAGES = 'gen_ai.request.messages';
+const GEN_AI_RESPONSE_TEXT = 'gen_ai.response.text';
+const GEN_AI_RESPONSE_TOOL_CALLS = 'gen_ai.response.tool_calls';
+
 type AttributeMap = Record<string, AttributeValue | undefined>;
 type AttributeMigrationFn = (attributes: AttributeMap) => AttributeValue | undefined;
 
@@ -26,8 +30,175 @@ function defineAttributeMigration(
   return migrate;
 }
 
+// Utilities
+
+function parseJson(value: AttributeValue): unknown | undefined {
+  if (typeof value !== 'string') {
+    return undefined;
+  }
+
+  try {
+    return JSON.parse(value);
+  } catch {
+    return undefined;
+  }
+}
+
+function parseJsonArray(value: AttributeValue): unknown[] | undefined {
+  const parsed = parseJson(value);
+  return Array.isArray(parsed) ? parsed : undefined;
+}
+
+function textPart(content: string): Record<string, unknown> {
+  return { type: 'text', content };
+}
+
+function normalizeMessageContent(content: unknown): unknown[] {
+  if (typeof content === 'string') {
+    return [textPart(content)];
+  }
+
+  if (Array.isArray(content)) {
+    return content.map((part) => {
+      if (typeof part === 'string') {
+        return textPart(part);
+      }
+
+      if (part && typeof part === 'object' && 'text' in part && !('content' in part)) {
+        return { ...part, content: (part as { text: unknown }).text };
+      }
+
+      return part;
+    });
+  }
+
+  return [content];
+}
+
+function migrateGenAiRequestMessagesValue(value: AttributeValue): AttributeValue {
+  if (typeof value === 'string' && parseJson(value) === undefined) {
+    return JSON.stringify([{ role: 'user', parts: [textPart(value)] }]);
+  }
+
+  const messages = parseJsonArray(value);
+  if (!messages) {
+    return value;
+  }
+
+  if (messages.every((message) => typeof message === 'string')) {
+    return JSON.stringify(messages.map((message) => ({ role: 'user', parts: [textPart(message as string)] })));
+  }
+
+  if (messages.every((message) => message && typeof message === 'object')) {
+    return JSON.stringify(
+      messages.map((message) => {
+        const input = message as Record<string, unknown>;
+        if ('parts' in input) {
+          return input;
+        }
+        if ('content' in input) {
+          const { content, ...rest } = input;
+          return { ...rest, parts: normalizeMessageContent(content) };
+        }
+        return input;
+      }),
+    );
+  }
+
+  return value;
+}
+
+function extractResponseTextParts(value: AttributeValue | undefined): Record<string, unknown>[] {
+  if (value === undefined) {
+    return [];
+  }
+
+  const parsed = parseJson(value);
+  if (typeof value === 'string' && parsed === undefined) {
+    return [textPart(value)];
+  }
+
+  if (typeof parsed === 'string') {
+    return [textPart(parsed)];
+  }
+
+  if (Array.isArray(parsed)) {
+    return parsed.flatMap((message) => {
+      if (typeof message === 'string') {
+        return [textPart(message)];
+      }
+      if (message && typeof message === 'object') {
+        const input = message as Record<string, unknown>;
+        if (typeof input.content === 'string') {
+          return [textPart(input.content)];
+        }
+        if (Array.isArray(input.parts)) {
+          return input.parts as Record<string, unknown>[];
+        }
+      }
+      return [];
+    });
+  }
+
+  if (parsed && typeof parsed === 'object') {
+    const input = parsed as Record<string, unknown>;
+    if (typeof input.content === 'string') {
+      return [textPart(input.content)];
+    }
+    if (Array.isArray(input.parts)) {
+      return input.parts as Record<string, unknown>[];
+    }
+  }
+
+  return [];
+}
+
+// Migrations
+
+defineAttributeMigration(
+  {
+    id: 'gen_ai_request_messages_to_input_messages',
+    sources: [GEN_AI_REQUEST_MESSAGES],
+    replacement: GEN_AI_INPUT_MESSAGES,
+  },
+  (attributes) => {
+    const value = attributes[GEN_AI_REQUEST_MESSAGES];
+    return value === undefined ? undefined : migrateGenAiRequestMessagesValue(value);
+  },
+);
+
+defineAttributeMigration(
+  {
+    id: 'gen_ai_response_to_output_messages',
+    sources: [GEN_AI_RESPONSE_TEXT, GEN_AI_RESPONSE_TOOL_CALLS],
+    replacement: GEN_AI_OUTPUT_MESSAGES,
+  },
+  (attributes) => {
+    const textValue = attributes[GEN_AI_RESPONSE_TEXT];
+    const toolCallsValue = attributes[GEN_AI_RESPONSE_TOOL_CALLS];
+    const parts: Record<string, unknown>[] = extractResponseTextParts(textValue);
+
+    const toolCalls = toolCallsValue === undefined ? undefined : parseJsonArray(toolCallsValue);
+    if (toolCalls) {
+      for (const toolCall of toolCalls) {
+        if (toolCall && typeof toolCall === 'object') {
+          parts.push({ ...(toolCall as Record<string, unknown>), type: 'tool_call' });
+        }
+      }
+    }
+
+    if (parts.length > 0) {
+      return JSON.stringify([{ role: 'assistant', parts }]);
+    }
+
+    return textValue ?? toolCallsValue;
+  },
+);
+
 export const ATTRIBUTE_MIGRATIONS = attributeMigrations as readonly AttributeMigration[];
 
+// Public API
+
 /**
  * Migrates a single deprecated attribute value when the migration only depends on that attribute.
  *

model/attributes/gen_ai/gen_ai__input__messages.json

@@ -8,6 +8,9 @@
   "is_in_otel": true,
   "example": "[{\"role\": \"user\", \"parts\": [{\"type\": \"text\", \"content\": \"Weather in Paris?\"}]}, {\"role\": \"assistant\", \"parts\": [{\"type\": \"tool_call\", \"id\": \"call_VSPygqKTWdrhaFErNvMV18Yl\", \"name\": \"get_weather\", \"arguments\": {\"location\": \"Paris\"}}]}, {\"role\": \"tool\", \"parts\": [{\"type\": \"tool_call_response\", \"id\": \"call_VSPygqKTWdrhaFErNvMV18Yl\", \"result\": \"rainy, 57°F\"}]}]",
   "alias": ["ai.texts"],
+  "migration": {
+    "target_of": ["gen_ai_request_messages_to_input_messages"]
+  },
   "visibility": "public",
   "changelog": [
     {

model/attributes/gen_ai/gen_ai__output__messages.json

@@ -7,6 +7,9 @@
   },
   "is_in_otel": true,
   "example": "[{\"role\": \"assistant\", \"parts\": [{\"type\": \"text\", \"content\": \"The weather in Paris is currently rainy with a temperature of 57°F.\"}], \"finish_reason\": \"stop\"}]",
+  "migration": {
+    "target_of": ["gen_ai_response_to_output_messages"]
+  },
   "visibility": "public",
   "changelog": [
     {

model/attributes/gen_ai/gen_ai__request__messages.json

@@ -8,6 +8,9 @@
   "is_in_otel": false,
   "example": "[{\"role\": \"system\", \"content\": \"Generate a random number.\"}, {\"role\": \"user\", \"content\": [{\"text\": \"Generate a random number between 0 and 10.\", \"type\": \"text\"}]}, {\"role\": \"tool\", \"content\": {\"toolCallId\": \"1\", \"toolName\": \"Weather\", \"output\": \"rainy\"}}]",
   "alias": ["ai.input_messages"],
+  "migration": {
+    "source_for": ["gen_ai_request_messages_to_input_messages"]
+  },
   "deprecation": {
     "_status": "normalize",
     "replacement": "gen_ai.input.messages"

model/attributes/gen_ai/gen_ai__response__text.json

@@ -8,6 +8,9 @@
   "is_in_otel": false,
   "example": "[\"The weather in Paris is rainy and overcast, with temperatures around 57°F\", \"The weather in London is sunny and warm, with temperatures around 65°F\"]",
   "alias": [],
+  "migration": {
+    "source_for": ["gen_ai_response_to_output_messages"]
+  },
   "deprecation": {
     "_status": "normalize",
     "replacement": "gen_ai.output.messages"

model/attributes/gen_ai/gen_ai__response__tool_calls.json

@@ -8,6 +8,9 @@
   "is_in_otel": false,
   "example": "[{\"name\": \"get_weather\", \"arguments\": {\"location\": \"Paris\"}}]",
   "alias": [],
+  "migration": {
+    "source_for": ["gen_ai_response_to_output_messages"]
+  },
   "deprecation": {
     "_status": "normalize",
     "replacement": "gen_ai.output.messages"

python/src/sentry_conventions/attributes.py

@@ -12332,6 +12332,9 @@ class ATTRIBUTE_NAMES(metaclass=_AttributeNamesMeta):
         is_in_otel=True,
         visibility=Visibility.PUBLIC,
         example='[{"role": "user", "parts": [{"type": "text", "content": "Weather in Paris?"}]}, {"role": "assistant", "parts": [{"type": "tool_call", "id": "call_VSPygqKTWdrhaFErNvMV18Yl", "name": "get_weather", "arguments": {"location": "Paris"}}]}, {"role": "tool", "parts": [{"type": "tool_call_response", "id": "call_VSPygqKTWdrhaFErNvMV18Yl", "result": "rainy, 57°F"}]}]',
+        migration=AttributeMigrationInfo(
+            target_of=["gen_ai_request_messages_to_input_messages"]
+        ),
         aliases=["ai.texts"],
         changelog=[
             ChangelogEntry(version="0.5.0", prs=[264]),
@@ -12369,6 +12372,9 @@ class ATTRIBUTE_NAMES(metaclass=_AttributeNamesMeta):
         is_in_otel=True,
         visibility=Visibility.PUBLIC,
         example='[{"role": "assistant", "parts": [{"type": "text", "content": "The weather in Paris is currently rainy with a temperature of 57°F."}], "finish_reason": "stop"}]',
+        migration=AttributeMigrationInfo(
+            target_of=["gen_ai_response_to_output_messages"]
+        ),
         changelog=[
             ChangelogEntry(version="0.4.0", prs=[221]),
         ],
@@ -12478,6 +12484,9 @@ class ATTRIBUTE_NAMES(metaclass=_AttributeNamesMeta):
         is_in_otel=False,
         visibility=Visibility.PUBLIC,
         example='[{"role": "system", "content": "Generate a random number."}, {"role": "user", "content": [{"text": "Generate a random number between 0 and 10.", "type": "text"}]}, {"role": "tool", "content": {"toolCallId": "1", "toolName": "Weather", "output": "rainy"}}]',
+        migration=AttributeMigrationInfo(
+            source_for=["gen_ai_request_messages_to_input_messages"]
+        ),
         deprecation=DeprecationInfo(
             replacement="gen_ai.input.messages", status=DeprecationStatus.NORMALIZE
         ),
@@ -12633,6 +12642,9 @@ class ATTRIBUTE_NAMES(metaclass=_AttributeNamesMeta):
         is_in_otel=False,
         visibility=Visibility.PUBLIC,
         example='["The weather in Paris is rainy and overcast, with temperatures around 57°F", "The weather in London is sunny and warm, with temperatures around 65°F"]',
+        migration=AttributeMigrationInfo(
+            source_for=["gen_ai_response_to_output_messages"]
+        ),
         deprecation=DeprecationInfo(
             replacement="gen_ai.output.messages", status=DeprecationStatus.NORMALIZE
         ),
@@ -12697,6 +12709,9 @@ class ATTRIBUTE_NAMES(metaclass=_AttributeNamesMeta):
         is_in_otel=False,
         visibility=Visibility.PUBLIC,
         example='[{"name": "get_weather", "arguments": {"location": "Paris"}}]',
+        migration=AttributeMigrationInfo(
+            source_for=["gen_ai_response_to_output_messages"]
+        ),
         deprecation=DeprecationInfo(
             replacement="gen_ai.output.messages", status=DeprecationStatus.NORMALIZE
         ),