media: send and receive GIFs like the LINE Chrome Extension

README.md

@@ -163,6 +163,7 @@ pacman -Syu mingw-w64-x86_64-gcc cmake
 ```
 
 - [Go](https://go.dev/dl/)
+- Optional: [FFmpeg](https://ffmpeg.org/) (`ffmpeg` and `ffprobe` on `PATH`) to send GIFs from GIF pickers, which send videos. Without it, the bridge sends only GIF files as GIFs.
 - [olm](https://gitlab.matrix.org/matrix-org/olm) library:
 
 ```bash
@@ -241,6 +242,7 @@ cd data
 
 - [Go](https://go.dev/dl/) and build tools (`gcc`, `make`)
 - [olm](https://gitlab.matrix.org/matrix-org/olm) library (install via your package manager or build from source)
+- Optional: [FFmpeg](https://ffmpeg.org/) (`ffmpeg` and `ffprobe` on `PATH`) to send GIFs from GIF pickers, which send videos. Without it, the bridge sends only GIF files as GIFs.
 - If using Beeper, compile [bbctl](https://github.com/beeper/bridge-manager) from source
 
 ### Linux / macOS build

pkg/connector/gif.go

@@ -0,0 +1,172 @@
+package connector
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"image"
+	"math"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strconv"
+	"strings"
+	"time"
+
+	"go.mau.fi/util/ffmpeg"
+
+	"maunium.net/go/mautrix/event"
+)
+
+// maxGIFSize is the largest GIF LINE sends as an image. The LINE Chrome
+// Extension sends bigger ones as files ("Photos larger than 20 MB can't be sent
+// in their original resolution"), so the bridge refuses them as too large.
+const maxGIFSize = 20 * 1000 * 1000
+
+// imageContentInfo is MEDIA_CONTENT_INFO as the LINE Chrome Extension sends it
+// for images: the original file's size and pixel dimensions, and for GIFs
+// whether they have more than one frame.
+type imageContentInfo struct {
+	Category  string `json:"category"`
+	FileSize  int    `json:"fileSize"`
+	Extension string `json:"extension"`
+	Animated  *bool  `json:"animated,omitempty"`
+	Width     int    `json:"width,omitempty"`
+	Height    int    `json:"height,omitempty"`
+}
+
+func originalImageInfo(data []byte, extension string) imageContentInfo {
+	info := imageContentInfo{
+		Category:  "original",
+		FileSize:  len(data),
+		Extension: extension,
+	}
+	if extension == "gif" {
+		animated := isAnimatedGif(data)
+		info.Animated = &animated
+	}
+	if config, _, err := image.DecodeConfig(bytes.NewReader(data)); err == nil {
+		info.Width, info.Height = config.Width, config.Height
+	}
+	return info
+}
+
+// imageMediaMetadata builds the content metadata the LINE Chrome Extension
+// sends with an image. Encrypted sends add OID, SID, FILE_SIZE and the
+// thumbnail info; plain sends carry MEDIA_CONTENT_INFO only. Neither sends
+// FILE_NAME or a contentType key.
+func imageMediaMetadata(data []byte, extension string, plain bool) map[string]string {
+	metadata := map[string]string{}
+	if infoJSON, err := json.Marshal(originalImageInfo(data, extension)); err == nil {
+		metadata["MEDIA_CONTENT_INFO"] = string(infoJSON)
+	}
+	if !plain {
+		metadata["FILE_SIZE"] = strconv.Itoa(len(data))
+	}
+	return metadata
+}
+
+// canConvertVideoToGIF reports whether the ffprobe and ffmpeg programs that
+// convertVideoToGIF runs are installed.
+func canConvertVideoToGIF() bool {
+	return ffmpeg.Supported() && ffmpeg.ProbeSupported()
+}
+
+// gifMimeTypes lists the fi.mau.gif types the bridge can send. Picker videos
+// are offered only when they can be converted to GIF.
+func gifMimeTypes() map[string]event.CapabilitySupportLevel {
+	mimeTypes := map[string]event.CapabilitySupportLevel{
+		"image/gif": event.CapLevelFullySupported,
+	}
+	if canConvertVideoToGIF() {
+		mimeTypes["video/mp4"] = event.CapLevelPartialSupport
+		mimeTypes["video/webm"] = event.CapLevelPartialSupport
+	}
+	return mimeTypes
+}
+
+const maxConvertedGIFSize = 100 * 1024 * 1024
+
+var gifConversionSlot = make(chan struct{}, 1)
+
+type gifOutputBuffer struct {
+	buffer bytes.Buffer
+	limit  int
+}
+
+func (b *gifOutputBuffer) Write(p []byte) (int, error) {
+	if len(p) > b.limit-b.buffer.Len() {
+		return 0, fmt.Errorf("converted GIF exceeds %d byte limit", b.limit)
+	}
+	return b.buffer.Write(p)
+}
+
+// convertVideoToGIF turns the MP4 or WebM that Matrix GIF pickers send
+// (fi.mau.gif) into a real GIF, because LINE only receives GIFs as images.
+func convertVideoToGIF(ctx context.Context, data []byte) ([]byte, error) {
+	ctx, cancel := context.WithTimeout(ctx, 2*time.Minute)
+	defer cancel()
+	if err := ctx.Err(); err != nil {
+		return nil, err
+	}
+	if len(data) > maxConvertedGIFSize {
+		return nil, fmt.Errorf("GIF video exceeds 100 MiB input limit")
+	}
+	select {
+	case gifConversionSlot <- struct{}{}:
+		defer func() { <-gifConversionSlot }()
+	case <-ctx.Done():
+		return nil, ctx.Err()
+	}
+	dir, err := os.MkdirTemp("", "line-gif-*")
+	if err != nil {
+		return nil, err
+	}
+	defer os.RemoveAll(dir)
+	input := filepath.Join(dir, "input")
+	palette := filepath.Join(dir, "palette.png")
+	if err = os.WriteFile(input, data, 0600); err != nil {
+		return nil, err
+	}
+	probe := exec.CommandContext(ctx, "ffprobe", "-v", "error", "-show_entries", "format=duration", "-of", "default=noprint_wrappers=1:nokey=1", input)
+	durationData, err := probe.Output()
+	if err != nil {
+		if ctx.Err() != nil {
+			return nil, ctx.Err()
+		}
+		return nil, fmt.Errorf("failed to inspect GIF video (ffprobe must be installed): %w", err)
+	}
+	duration, err := strconv.ParseFloat(strings.TrimSpace(string(durationData)), 64)
+	if err != nil || math.IsNaN(duration) || math.IsInf(duration, 0) || duration <= 0 || duration > 60 {
+		return nil, fmt.Errorf("GIF video must have a known duration of at most 60 seconds")
+	}
+	filter := "fps=15,scale=w='min(640,iw)':h='min(640,ih)':force_original_aspect_ratio=decrease:flags=lanczos"
+
+	makePalette := exec.CommandContext(ctx, "ffmpeg", "-hide_banner", "-loglevel", "error", "-nostdin",
+		"-threads", "1", "-i", input, "-an", "-vf", filter+",palettegen",
+		"-frames:v", "1", "-threads", "1", "-update", "1", palette)
+	if err = makePalette.Run(); err != nil {
+		if ctx.Err() != nil {
+			return nil, ctx.Err()
+		}
+		return nil, fmt.Errorf("ffmpeg GIF palette generation failed: %w", err)
+	}
+	output := gifOutputBuffer{limit: maxConvertedGIFSize}
+	cmd := exec.CommandContext(ctx, "ffmpeg", "-hide_banner", "-loglevel", "error", "-nostdin",
+		"-threads", "1", "-i", input, "-i", palette, "-an",
+		"-filter_complex", "[0:v]"+filter+"[v];[v][1:v]paletteuse",
+		"-loop", "0", "-f", "gif", "pipe:1")
+	cmd.Stdout = &output
+	if err = cmd.Run(); err != nil {
+		if ctx.Err() != nil {
+			return nil, ctx.Err()
+		}
+		return nil, fmt.Errorf("ffmpeg GIF conversion failed: %w", err)
+	}
+	converted := output.buffer.Bytes()
+	if _, format, err := image.DecodeConfig(bytes.NewReader(converted)); err != nil || format != "gif" {
+		return nil, fmt.Errorf("ffmpeg produced invalid GIF output")
+	}
+	return converted, nil
+}

pkg/connector/gif_test.go

@@ -0,0 +1,149 @@
+package connector
+
+import (
+	"bytes"
+	"encoding/json"
+	"image"
+	"image/color"
+	"image/gif"
+	"image/jpeg"
+	"testing"
+
+	"maunium.net/go/mautrix/event"
+)
+
+func testGIF(t *testing.T, width, height, frames int) []byte {
+	t.Helper()
+	palette := color.Palette{color.Transparent, color.RGBA{R: 6, G: 166, B: 74, A: 255}}
+	anim := &gif.GIF{}
+	for i := 0; i < frames; i++ {
+		frame := image.NewPaletted(image.Rect(0, 0, width, height), palette)
+		frame.SetColorIndex(i%width, 0, 1)
+		anim.Image = append(anim.Image, frame)
+		anim.Delay = append(anim.Delay, 10)
+	}
+	var buf bytes.Buffer
+	if err := gif.EncodeAll(&buf, anim); err != nil {
+		t.Fatalf("encode test GIF: %v", err)
+	}
+	return buf.Bytes()
+}
+
+func decodeContentInfo(t *testing.T, metadata map[string]string) map[string]any {
+	t.Helper()
+	var info map[string]any
+	if err := json.Unmarshal([]byte(metadata["MEDIA_CONTENT_INFO"]), &info); err != nil {
+		t.Fatalf("MEDIA_CONTENT_INFO is not JSON: %v", err)
+	}
+	return info
+}
+
+// The LINE Chrome Extension sends a GIF's real pixel size and original file
+// size in MEDIA_CONTENT_INFO (capture D2, D8, D9).
+func TestImageMediaMetadataEncryptedMatchesChrome(t *testing.T) {
+	data := testGIF(t, 960, 360, 3)
+	metadata := imageMediaMetadata(data, "gif", false)
+
+	info := decodeContentInfo(t, metadata)
+	if info["width"] != float64(960) || info["height"] != float64(360) {
+		t.Errorf("width/height = %v/%v, want 960/360", info["width"], info["height"])
+	}
+	if info["fileSize"] != float64(len(data)) {
+		t.Errorf("fileSize = %v, want the original size %d", info["fileSize"], len(data))
+	}
+	if info["animated"] != true || info["extension"] != "gif" || info["category"] != "original" {
+		t.Errorf("unexpected MEDIA_CONTENT_INFO %v", info)
+	}
+	if metadata["FILE_SIZE"] != itoa(len(data)) {
+		t.Errorf("FILE_SIZE = %q, want the original size %d", metadata["FILE_SIZE"], len(data))
+	}
+	for _, key := range []string{"FILE_NAME", "contentType"} {
+		if _, ok := metadata[key]; ok {
+			t.Errorf("metadata has %s, which the Chrome extension does not send", key)
+		}
+	}
+}
+
+// In plain chats the Chrome extension sends MEDIA_CONTENT_INFO only (capture D7).
+func TestImageMediaMetadataPlainIsContentInfoOnly(t *testing.T) {
+	metadata := imageMediaMetadata(testGIF(t, 480, 480, 2), "gif", true)
+	if len(metadata) != 1 || metadata["MEDIA_CONTENT_INFO"] == "" {
+		t.Fatalf("plain metadata = %v, want MEDIA_CONTENT_INFO only", metadata)
+	}
+}
+
+// A single-frame GIF is sent with animated: false (capture D3); non-GIF images
+// carry no animated flag.
+func TestOriginalImageInfoAnimatedFlag(t *testing.T) {
+	still := originalImageInfo(testGIF(t, 800, 800, 1), "gif")
+	if still.Animated == nil || *still.Animated {
+		t.Errorf("single-frame GIF animated = %v, want false", still.Animated)
+	}
+	moving := originalImageInfo(testGIF(t, 32, 32, 4), "gif")
+	if moving.Animated == nil || !*moving.Animated {
+		t.Errorf("multi-frame GIF animated = %v, want true", moving.Animated)
+	}
+	if moving.Width != 32 || moving.Height != 32 {
+		t.Errorf("tiny GIF size = %dx%d, want 32x32", moving.Width, moving.Height)
+	}
+	if jpg := originalImageInfo([]byte("not an image"), "jpg"); jpg.Animated != nil {
+		t.Errorf("JPEG has animated flag %v, want none", *jpg.Animated)
+	}
+}
+
+// Thumbnails keep the aspect ratio with the longest side at most 384 and are
+// never scaled up (capture D8: 384x144, D9: 144x384, E6: 32x32).
+func TestGenerateThumbnailMatchesChromeSizes(t *testing.T) {
+	cases := []struct{ w, h, wantW, wantH int }{
+		{960, 360, 384, 144},
+		{360, 960, 144, 384},
+		{480, 480, 384, 384},
+		{32, 32, 32, 32},
+	}
+	for _, c := range cases {
+		_, w, h, err := generateThumbnail(testGIF(t, c.w, c.h, 2))
+		if err != nil {
+			t.Fatalf("%dx%d: %v", c.w, c.h, err)
+		}
+		if w != c.wantW || h != c.wantH {
+			t.Errorf("%dx%d thumbnail = %dx%d, want %dx%d", c.w, c.h, w, h, c.wantW, c.wantH)
+		}
+	}
+}
+
+// Transparent areas become white, not black, in the JPEG thumbnail LINE shows
+// before a GIF plays.
+func TestGenerateThumbnailTransparentIsWhite(t *testing.T) {
+	thumb, _, _, err := generateThumbnail(testGIF(t, 64, 64, 2))
+	if err != nil {
+		t.Fatal(err)
+	}
+	img, err := jpeg.Decode(bytes.NewReader(thumb))
+	if err != nil {
+		t.Fatal(err)
+	}
+	r, g, b, _ := img.At(32, 40).RGBA()
+	if r>>8 < 240 || g>>8 < 240 || b>>8 < 240 {
+		t.Errorf("transparent pixel = (%d,%d,%d), want white", r>>8, g>>8, b>>8)
+	}
+}
+
+// GIF pickers send fi.mau.gif; LINE takes GIFs up to 20 MB (capture E1–E3).
+func TestCapabilitiesAdvertiseGIFs(t *testing.T) {
+	caps := (&LineClient{}).GetCapabilities(t.Context(), nil)
+	gifCaps, ok := caps.File[event.CapMsgGIF]
+	if !ok {
+		t.Fatal("no fi.mau.gif capability")
+	}
+	if gifCaps.MaxSize != maxGIFSize {
+		t.Errorf("GIF MaxSize = %d, want %d", gifCaps.MaxSize, maxGIFSize)
+	}
+	if gifCaps.MimeTypes["image/gif"] != event.CapLevelFullySupported {
+		t.Errorf("image/gif support = %v, want fully supported", gifCaps.MimeTypes["image/gif"])
+	}
+}
+
+func itoa(n int) string {
+	b, _ := json.Marshal(n)
+	return string(b)
+}