Merge pull request #1889 from rumpl/vision

👀 Vision 👀

Author: dgageot

e2e/testdata/cassettes/TestExec_Anthropic_ToolCall.yaml

@@ -60,6 +60,7 @@ require (
 	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.40.0
 	go.opentelemetry.io/otel/sdk v1.40.0
 	go.opentelemetry.io/otel/trace v1.40.0
+	golang.org/x/image v0.36.0
 	golang.org/x/net v0.51.0
 	golang.org/x/oauth2 v0.35.0
 	golang.org/x/sync v0.19.0

e2e/testdata/cassettes/TestExec_Gemini_ToolCall.yaml

@@ -529,6 +529,8 @@ golang.org/x/crypto v0.48.0 h1:/VRzVqiRSggnhY7gNRxPauEQ5Drw9haKdM0jqfcCFts=
 golang.org/x/crypto v0.48.0/go.mod h1:r0kV5h3qnFPlQnBSrULhlsRfryS2pmewsg+XfMgkVos=
 golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546 h1:mgKeJMpvi0yx/sU5GsxQ7p6s2wtOnGAHZWCHUM4KGzY=
 golang.org/x/exp v0.0.0-20251023183803-a4bb9ffd2546/go.mod h1:j/pmGrbnkbPtQfxEe5D0VQhZC6qKbfKifgD0oM7sR70=
+golang.org/x/image v0.36.0 h1:Iknbfm1afbgtwPTmHnS2gTM/6PPZfH+z2EFuOkSbqwc=
+golang.org/x/image v0.36.0/go.mod h1:YsWD2TyyGKiIX1kZlu9QfKIsQ4nAAK9bdgdrIsE7xy4=
 golang.org/x/mod v0.33.0 h1:tHFzIWbBifEmbwtGz65eaWyGiGZatSrT9prnU8DbVL8=
 golang.org/x/mod v0.33.0/go.mod h1:swjeQEj+6r7fODbD2cqrnje9PnziFuw4bmLbBZFrQ5w=
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=

e2e/testdata/cassettes/TestExec_Mistral_ToolCall.yaml

@@ -2,6 +2,7 @@ package app
 
 import (
 	"context"
+	"encoding/base64"
 	"fmt"
 	"log/slog"
 	"os"
@@ -396,13 +397,43 @@ func (a *App) processFileAttachment(ctx context.Context, att messages.Attachment
 		textBuilder.WriteString(content)
 
 	case chat.IsSupportedMimeType(mimeType):
-		*binaryParts = append(*binaryParts, chat.MessagePart{
-			Type: chat.MessagePartTypeFile,
-			File: &chat.MessageFile{
-				Path:     absPath,
-				MimeType: mimeType,
-			},
-		})
+		if chat.IsImageMimeType(mimeType) {
+			// Read, resize if needed, and inline as base64 data URL.
+			// This works across all providers (not just Anthropic's File API).
+			imgData, readErr := os.ReadFile(absPath)
+			if readErr != nil {
+				slog.Warn("skipping attachment: failed to read image", "path", absPath, "error", readErr)
+				a.sendEvent(ctx, runtime.Warning(fmt.Sprintf("Skipped attachment %s: failed to read image", att.Name), ""))
+				return
+			}
+			resized, resizeErr := chat.ResizeImage(imgData, mimeType)
+			if resizeErr != nil {
+				// Don't bypass security checks - reject the file if resize failed
+				slog.Warn("skipping attachment: image resize failed", "path", absPath, "error", resizeErr)
+				a.sendEvent(ctx, runtime.Warning(fmt.Sprintf("Skipped attachment %s: %s", att.Name, resizeErr), ""))
+				return
+			}
+			dataURL := fmt.Sprintf("data:%s;base64,%s", resized.MimeType, base64.StdEncoding.EncodeToString(resized.Data))
+			*binaryParts = append(*binaryParts, chat.MessagePart{
+				Type: chat.MessagePartTypeImageURL,
+				ImageURL: &chat.MessageImageURL{
+					URL:    dataURL,
+					Detail: chat.ImageURLDetailAuto,
+				},
+			})
+			if note := chat.FormatDimensionNote(resized); note != "" {
+				textBuilder.WriteString("\n" + note)
+			}
+		} else {
+			// Non-image supported types (e.g. PDF) use the file upload path.
+			*binaryParts = append(*binaryParts, chat.MessagePart{
+				Type: chat.MessagePartTypeFile,
+				File: &chat.MessageFile{
+					Path:     absPath,
+					MimeType: mimeType,
+				},
+			})
+		}
 
 	default:
 		slog.Warn("skipping attachment: unsupported file type", "path", absPath, "mime_type", mimeType)

e2e/testdata/cassettes/TestExec_OpenAI_HideToolCalls.yaml

@@ -173,32 +173,56 @@ type MessageStream interface {
 	Close()
 }
 
-// DetectMimeType returns the MIME type for a file based on its extension.
-// This is the canonical implementation used across all packages for consistency.
-// For binary file types (images, PDF), returns the specific MIME type.
-// For text-based files, returns "text/plain".
-// Unrecognized extensions return "application/octet-stream".
+// DetectMimeType returns the MIME type for a file by reading its first 512
+// bytes and inspecting the content (magic bytes). For text-based files that
+// http.DetectContentType cannot distinguish (e.g. source code vs plain text),
+// it falls back to extension matching. This is the canonical implementation
+// used across all packages for consistency.
 func DetectMimeType(filePath string) string {
-	ext := strings.ToLower(filepath.Ext(filePath))
-	switch ext {
-	// Images
-	case ".jpg", ".jpeg":
-		return "image/jpeg"
-	case ".png":
-		return "image/png"
-	case ".gif":
-		return "image/gif"
-	case ".webp":
-		return "image/webp"
-	// PDF
-	case ".pdf":
-		return "application/pdf"
-	default:
-		if isTextExtension(ext) {
-			return "text/plain"
-		}
+	// Content sniffing — reliably detects images, PDF, etc.
+	if ct := detectMimeTypeFromFile(filePath); ct != "application/octet-stream" {
+		return ct
+	}
+
+	// http.DetectContentType returns "application/octet-stream" for text
+	// files it can't classify, so fall back to extension for those.
+	if isTextExtension(strings.ToLower(filepath.Ext(filePath))) {
+		return "text/plain"
+	}
+	return "application/octet-stream"
+}
+
+// detectMimeTypeFromFile reads the first 512 bytes of a file and uses
+// content-based detection (magic bytes) to determine the MIME type.
+func detectMimeTypeFromFile(filePath string) string {
+	f, err := os.Open(filePath)
+	if err != nil {
+		return "application/octet-stream"
+	}
+	defer f.Close()
+
+	buf := make([]byte, 512)
+	n, _ := f.Read(buf)
+	if n == 0 {
 		return "application/octet-stream"
 	}
+	return DetectMimeTypeByContent(buf[:n])
+}
+
+// IsImageMimeType returns true if the MIME type is a supported image type.
+func IsImageMimeType(mimeType string) bool {
+	switch mimeType {
+	case "image/jpeg", "image/png", "image/gif", "image/webp":
+		return true
+	default:
+		return false
+	}
+}
+
+// IsImageFile returns true if the file at the given path is a supported image
+// based on its extension. Supported formats: JPEG, PNG, GIF, WebP.
+func IsImageFile(filePath string) bool {
+	return IsImageMimeType(DetectMimeType(filePath))
 }
 
 // IsSupportedMimeType returns true if the MIME type is supported for file attachments.

e2e/testdata/cassettes/TestExec_OpenAI_ToolCall.yaml

@@ -15,15 +15,7 @@ func TestDetectMimeType(t *testing.T) {
 		path     string
 		expected string
 	}{
-		// Images
-		{"photo.jpg", "image/jpeg"},
-		{"photo.jpeg", "image/jpeg"},
-		{"photo.png", "image/png"},
-		{"photo.gif", "image/gif"},
-		{"photo.webp", "image/webp"},
-		// PDF
-		{"document.pdf", "application/pdf"},
-		// Text files - all map to text/plain
+		// Text files (detected by extension fallback) - all map to text/plain
 		{"readme.txt", "text/plain"},
 		{"readme.md", "text/plain"},
 		{"readme.markdown", "text/plain"},
@@ -46,7 +38,7 @@ func TestDetectMimeType(t *testing.T) {
 		{"query.graphql", "text/plain"},
 		{"icon.svg", "text/plain"},
 		{"changes.diff", "text/plain"},
-		// Unknown binary
+		// Unknown binary (no file to sniff, unknown extension)
 		{"archive.tar.gz", "application/octet-stream"},
 		{"program.exe", "application/octet-stream"},
 		{"movie.mp4", "application/octet-stream"},
@@ -169,3 +161,50 @@ func TestReadFileForInline_NotFound(t *testing.T) {
 	_, err := ReadFileForInline("/nonexistent/file.txt")
 	assert.Error(t, err)
 }
+
+func TestDetectMimeType_ContentSniffing(t *testing.T) {
+	t.Parallel()
+
+	tests := []struct {
+		name     string
+		filename string
+		content  []byte
+		want     string
+	}{
+		{
+			name:     "png with unknown extension",
+			filename: "image.bin",
+			content:  []byte{0x89, 'P', 'N', 'G', 0x0D, 0x0A, 0x1A, 0x0A, 0, 0, 0, 0},
+			want:     "image/png",
+		},
+		{
+			name:     "jpeg with unknown extension",
+			filename: "photo.dat",
+			content:  []byte{0xFF, 0xD8, 0xFF, 0xE0, 0, 0, 0, 0},
+			want:     "image/jpeg",
+		},
+		{
+			name:     "webp with unknown extension",
+			filename: "anim.unknown",
+			content:  append([]byte("RIFF"), append([]byte{0, 0, 0, 0}, []byte("WEBP")...)...),
+			want:     "image/webp",
+		},
+		{
+			name:     "plain text with unknown extension",
+			filename: "data.xyz",
+			content:  []byte("just some text content here"),
+			want:     "text/plain; charset=utf-8",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			t.Parallel()
+			dir := t.TempDir()
+			path := filepath.Join(dir, tt.filename)
+			require.NoError(t, os.WriteFile(path, tt.content, 0o644))
+			got := DetectMimeType(path)
+			assert.Equal(t, tt.want, got)
+		})
+	}
+}