Skip to content

Commit 6e42aa7

Browse files
copybara-githubcopybara-github
committed
Merge pull request google#706 from prasadskarmarkar:fix/spring-ai-media-support
PiperOrigin-RevId: 889231825
2 parents 82baba1 + 8ab7f07 commit 6e42aa7

2 files changed

Lines changed: 183 additions & 10 deletions

File tree

‎contrib/spring-ai/src/main/java/com/google/adk/models/springai/MessageConverter.java‎

Lines changed: 3 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -221,8 +221,7 @@ private List<Message> handleUserContent(Content content) {
221221
} catch (Exception e) {
222222
// Log warning but continue processing other parts
223223
// In production, consider proper logging framework
224-
System.err.println(
225-
"Warning: Failed to parse media mime type: " + blob.mimeType().get());
224+
System.err.println("Warning: Failed to process media part: " + e.getMessage());
226225
}
227226
}
228227
} else if (part.fileData().isPresent()) {
@@ -235,19 +234,14 @@ private List<Message> handleUserContent(Content content) {
235234
URI uri = URI.create(fileData.fileUri().get());
236235
mediaList.add(new Media(mimeType, uri));
237236
} catch (Exception e) {
238-
System.err.println(
239-
"Warning: Failed to parse media mime type: " + fileData.mimeType().get());
237+
System.err.println("Warning: Failed to process media part: " + e.getMessage());
240238
}
241239
}
242240
}
243241
}
244242

245243
List<Message> messages = new ArrayList<>();
246-
// Create UserMessage with text
247-
// TODO: Media attachments support - UserMessage constructors with media are private in Spring
248-
// AI 1.1.0
249-
// For now, only text content is supported
250-
messages.add(new UserMessage(textBuilder.toString()));
244+
messages.add(UserMessage.builder().text(textBuilder.toString()).media(mediaList).build());
251245
messages.addAll(toolResponseMessages);
252246

253247
return messages;

‎contrib/spring-ai/src/test/java/com/google/adk/models/springai/MessageConverterTest.java‎

Lines changed: 180 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,9 @@ void testToLlmPromptWithUserMessage() {
6060
assertThat(prompt.getInstructions()).hasSize(1);
6161
Message message = prompt.getInstructions().get(0);
6262
assertThat(message).isInstanceOf(UserMessage.class);
63-
assertThat(((UserMessage) message).getText()).isEqualTo("Hello, how are you?");
63+
UserMessage userMessage = (UserMessage) message;
64+
assertThat(userMessage.getText()).isEqualTo("Hello, how are you?");
65+
assertThat(userMessage.getMedia()).isEmpty();
6466
}
6567

6668
@Test
@@ -444,4 +446,181 @@ void testCombineMultipleSystemMessagesForGeminiCompatibility() {
444446
assertThat(secondMessage).isInstanceOf(UserMessage.class);
445447
assertThat(((UserMessage) secondMessage).getText()).isEqualTo("Hello world");
446448
}
449+
450+
@Test
451+
void testUserMessageWithInlineMediaData() {
452+
// Test conversion of ADK Content with inline media (image bytes) to Spring AI UserMessage
453+
byte[] imageData = "fake-image-data".getBytes();
454+
String mimeType = "image/png";
455+
456+
Content userContent =
457+
Content.builder()
458+
.role("user")
459+
.parts(
460+
List.of(
461+
Part.fromText("What's in this image?"),
462+
Part.builder()
463+
.inlineData(
464+
com.google.genai.types.Blob.builder()
465+
.mimeType(mimeType)
466+
.data(imageData)
467+
.build())
468+
.build()))
469+
.build();
470+
471+
LlmRequest request = LlmRequest.builder().contents(List.of(userContent)).build();
472+
473+
Prompt prompt = messageConverter.toLlmPrompt(request);
474+
475+
assertThat(prompt.getInstructions()).hasSize(1);
476+
Message message = prompt.getInstructions().get(0);
477+
assertThat(message).isInstanceOf(UserMessage.class);
478+
479+
UserMessage userMessage = (UserMessage) message;
480+
assertThat(userMessage.getText()).isEqualTo("What's in this image?");
481+
assertThat(userMessage.getMedia()).hasSize(1);
482+
org.springframework.ai.content.Media media = userMessage.getMedia().get(0);
483+
assertThat(media.getMimeType().toString()).isEqualTo(mimeType);
484+
assertThat(media.getData()).isInstanceOf(byte[].class);
485+
byte[] actualData = (byte[]) media.getData();
486+
assertThat(actualData).isEqualTo(imageData);
487+
}
488+
489+
@Test
490+
void testUserMessageWithFileMediaData() {
491+
// Test conversion of ADK Content with file-based media (URI) to Spring AI UserMessage
492+
String fileUri = "gs://bucket/image.jpg";
493+
String mimeType = "image/jpeg";
494+
495+
Content userContent =
496+
Content.builder()
497+
.role("user")
498+
.parts(
499+
List.of(
500+
Part.fromText("Analyze this image"),
501+
Part.builder()
502+
.fileData(
503+
com.google.genai.types.FileData.builder()
504+
.mimeType(mimeType)
505+
.fileUri(fileUri)
506+
.build())
507+
.build()))
508+
.build();
509+
510+
LlmRequest request = LlmRequest.builder().contents(List.of(userContent)).build();
511+
512+
Prompt prompt = messageConverter.toLlmPrompt(request);
513+
514+
assertThat(prompt.getInstructions()).hasSize(1);
515+
Message message = prompt.getInstructions().get(0);
516+
assertThat(message).isInstanceOf(UserMessage.class);
517+
518+
UserMessage userMessage = (UserMessage) message;
519+
assertThat(userMessage.getText()).isEqualTo("Analyze this image");
520+
assertThat(userMessage.getMedia()).hasSize(1);
521+
org.springframework.ai.content.Media media = userMessage.getMedia().get(0);
522+
assertThat(media.getMimeType().toString()).isEqualTo(mimeType);
523+
assertThat(media.getData()).isInstanceOf(String.class);
524+
String actualUri = (String) media.getData();
525+
assertThat(actualUri).isEqualTo(fileUri);
526+
}
527+
528+
@Test
529+
void testUserMessageWithMultipleMediaAttachments() {
530+
// Test conversion with multiple media attachments
531+
byte[] image1 = "image1-data".getBytes();
532+
byte[] image2 = "image2-data".getBytes();
533+
534+
Content userContent =
535+
Content.builder()
536+
.role("user")
537+
.parts(
538+
List.of(
539+
Part.fromText("Compare these images"),
540+
Part.builder()
541+
.inlineData(
542+
com.google.genai.types.Blob.builder()
543+
.mimeType("image/png")
544+
.data(image1)
545+
.build())
546+
.build(),
547+
Part.builder()
548+
.inlineData(
549+
com.google.genai.types.Blob.builder()
550+
.mimeType("image/jpeg")
551+
.data(image2)
552+
.build())
553+
.build()))
554+
.build();
555+
556+
LlmRequest request = LlmRequest.builder().contents(List.of(userContent)).build();
557+
558+
Prompt prompt = messageConverter.toLlmPrompt(request);
559+
560+
assertThat(prompt.getInstructions()).hasSize(1);
561+
UserMessage userMessage = (UserMessage) prompt.getInstructions().get(0);
562+
assertThat(userMessage.getText()).isEqualTo("Compare these images");
563+
assertThat(userMessage.getMedia()).hasSize(2);
564+
}
565+
566+
@Test
567+
void testUserMessageWithInvalidMimeTypeGracefullySkipsMediaPart() {
568+
// Test that an invalid MIME type string causes the media part to be skipped gracefully
569+
byte[] imageData = "fake-image-data".getBytes();
570+
571+
Content userContent =
572+
Content.builder()
573+
.role("user")
574+
.parts(
575+
List.of(
576+
Part.fromText("What's in this image?"),
577+
Part.builder()
578+
.inlineData(
579+
com.google.genai.types.Blob.builder()
580+
.mimeType("invalid/mime/type!!!") // invalid MIME type
581+
.data(imageData)
582+
.build())
583+
.build()))
584+
.build();
585+
586+
LlmRequest request = LlmRequest.builder().contents(List.of(userContent)).build();
587+
588+
// Should not throw — invalid MIME type is silently skipped
589+
Prompt prompt = messageConverter.toLlmPrompt(request);
590+
591+
assertThat(prompt.getInstructions()).hasSize(1);
592+
UserMessage userMessage = (UserMessage) prompt.getInstructions().get(0);
593+
assertThat(userMessage.getText()).isEqualTo("What's in this image?");
594+
// Media part is skipped due to invalid MIME type
595+
assertThat(userMessage.getMedia()).isEmpty();
596+
}
597+
598+
@Test
599+
void testUserMessageWithMediaOnly() {
600+
// Test conversion with media but no text
601+
byte[] imageData = "image-only".getBytes();
602+
603+
Content userContent =
604+
Content.builder()
605+
.role("user")
606+
.parts(
607+
List.of(
608+
Part.builder()
609+
.inlineData(
610+
com.google.genai.types.Blob.builder()
611+
.mimeType("image/png")
612+
.data(imageData)
613+
.build())
614+
.build()))
615+
.build();
616+
617+
LlmRequest request = LlmRequest.builder().contents(List.of(userContent)).build();
618+
619+
Prompt prompt = messageConverter.toLlmPrompt(request);
620+
621+
assertThat(prompt.getInstructions()).hasSize(1);
622+
UserMessage userMessage = (UserMessage) prompt.getInstructions().get(0);
623+
assertThat(userMessage.getText()).isEmpty();
624+
assertThat(userMessage.getMedia()).hasSize(1);
625+
}
447626
}

0 commit comments

Comments
 (0)