From 6241f354028d2625ba18f3b4ad41c319a5c73775 Mon Sep 17 00:00:00 2001 From: formeo14 Date: Sun, 27 Sep 2026 13:20:33 +0900 Subject: [PATCH 1/4] feat: support GIF sending with transparency --- pkg/connector/connector.go | 2 +- pkg/connector/gif.go | 337 ++++++++++++++++++++ pkg/connector/gif_capabilities_test.go | 24 ++ pkg/connector/gif_test.go | 167 ++++++++++ pkg/connector/handlers/directmedia.go | 16 +- pkg/connector/handlers/image.go | 33 +- pkg/connector/handlers/image_format_test.go | 89 ++++++ pkg/connector/media.go | 65 ++-- pkg/connector/png_stickers_test.go | 133 ++++++++ pkg/connector/reaction_test.go | 4 +- pkg/connector/send_message.go | 109 +++---- pkg/connector/transparency_test.go | 100 ++++++ pkg/connector/userinfo.go | 18 ++ pkg/line/client.go | 14 + pkg/line/obs_test.go | 47 +++ 15 files changed, 1056 insertions(+), 102 deletions(-) create mode 100644 pkg/connector/gif.go create mode 100644 pkg/connector/gif_capabilities_test.go create mode 100644 pkg/connector/gif_test.go create mode 100644 pkg/connector/handlers/image_format_test.go create mode 100644 pkg/connector/png_stickers_test.go create mode 100644 pkg/connector/transparency_test.go diff --git a/pkg/connector/connector.go b/pkg/connector/connector.go index af9b340a..6e7a8d70 100644 --- a/pkg/connector/connector.go +++ b/pkg/connector/connector.go @@ -52,7 +52,7 @@ func (lc *LineConnector) Start(ctx context.Context) error { } func (lc *LineConnector) GetBridgeInfoVersion() (info, capabilities int) { - return 1, 3 + return 1, 4 } func (lc *LineConnector) GetCapabilities() *bridgev2.NetworkGeneralCapabilities { diff --git a/pkg/connector/gif.go b/pkg/connector/gif.go new file mode 100644 index 00000000..a93292c2 --- /dev/null +++ b/pkg/connector/gif.go @@ -0,0 +1,337 @@ +package connector + +import ( + "bytes" + "context" + "encoding/binary" + "fmt" + "image" + "image/png" + "math" + "os" + "os/exec" + "path/filepath" + "slices" + "strconv" + "strings" + "time" +) + +type imageContentInfo struct { + Category string `json:"category"` + FileSize int `json:"fileSize"` + Extension string `json:"extension"` + Animated bool `json:"animated"` + Width int `json:"width,omitempty"` + Height int `json:"height,omitempty"` +} + +func originalImageInfo(data []byte, extension string) imageContentInfo { + info := imageContentInfo{ + Category: "original", + FileSize: len(data), + Extension: extension, + Animated: extension == "gif" && isAnimatedGif(data), + } + if config, _, err := image.DecodeConfig(bytes.NewReader(data)); err == nil { + info.Width, info.Height = config.Width, config.Height + } + return info +} + +const maxConvertedGIFSize = 100 * 1024 * 1024 + +var gifConversionSlot = make(chan struct{}, 1) + +type gifOutputBuffer struct { + buffer bytes.Buffer + limit int +} + +func (b *gifOutputBuffer) Write(p []byte) (int, error) { + if len(p) > b.limit-b.buffer.Len() { + return 0, fmt.Errorf("converted GIF exceeds %d byte limit", b.limit) + } + return b.buffer.Write(p) +} + +func convertVideoToGIF(ctx context.Context, data []byte) ([]byte, error) { + ctx, cancel := context.WithTimeout(ctx, 2*time.Minute) + defer cancel() + if err := ctx.Err(); err != nil { + return nil, err + } + if len(data) > maxConvertedGIFSize { + return nil, fmt.Errorf("GIF video exceeds 100 MiB input limit") + } + select { + case gifConversionSlot <- struct{}{}: + defer func() { <-gifConversionSlot }() + case <-ctx.Done(): + return nil, ctx.Err() + } + dir, err := os.MkdirTemp("", "line-gif-*") + if err != nil { + return nil, err + } + defer os.RemoveAll(dir) + input := filepath.Join(dir, "input") + palette := filepath.Join(dir, "palette.png") + if err = os.WriteFile(input, data, 0600); err != nil { + return nil, err + } + probe := exec.CommandContext(ctx, "ffprobe", "-v", "error", "-show_entries", "format=duration", "-of", "default=noprint_wrappers=1:nokey=1", input) + durationData, err := probe.Output() + if err != nil { + if ctx.Err() != nil { + return nil, ctx.Err() + } + return nil, fmt.Errorf("failed to inspect GIF video (ffprobe must be installed): %w", err) + } + duration, err := strconv.ParseFloat(strings.TrimSpace(string(durationData)), 64) + if err != nil || math.IsNaN(duration) || math.IsInf(duration, 0) || duration <= 0 || duration > 60 { + return nil, fmt.Errorf("GIF video must have a known duration of at most 60 seconds") + } + filter := "fps=15,scale=w='min(640,iw)':h='min(640,ih)':force_original_aspect_ratio=decrease:flags=lanczos" + + makePalette := exec.CommandContext(ctx, "ffmpeg", "-hide_banner", "-loglevel", "error", "-nostdin", + "-threads", "1", "-i", input, "-an", "-vf", filter+",palettegen", + "-frames:v", "1", "-threads", "1", "-update", "1", palette) + if err = makePalette.Run(); err != nil { + if ctx.Err() != nil { + return nil, ctx.Err() + } + return nil, fmt.Errorf("ffmpeg GIF palette generation failed: %w", err) + } + output := gifOutputBuffer{limit: maxConvertedGIFSize} + cmd := exec.CommandContext(ctx, "ffmpeg", "-hide_banner", "-loglevel", "error", "-nostdin", + "-threads", "1", "-i", input, "-i", palette, "-an", + "-filter_complex", "[0:v]"+filter+"[v];[v][1:v]paletteuse", + "-loop", "0", "-f", "gif", "pipe:1") + cmd.Stdout = &output + if err = cmd.Run(); err != nil { + if ctx.Err() != nil { + return nil, ctx.Err() + } + return nil, fmt.Errorf("ffmpeg GIF conversion failed: %w", err) + } + converted := output.buffer.Bytes() + if _, format, err := image.DecodeConfig(bytes.NewReader(converted)); err != nil || format != "gif" { + return nil, fmt.Errorf("ffmpeg produced invalid GIF output") + } + return converted, nil +} + +func isAnimatedGif(data []byte) bool { + if len(data) < 13 || (string(data[:6]) != "GIF87a" && string(data[:6]) != "GIF89a") { + return false + } + + pos := 13 + if data[10]&0x80 != 0 { + pos += 3 * (1 << ((data[10] & 7) + 1)) + } + skipBlocks := func() bool { + for pos < len(data) { + size := int(data[pos]) + pos++ + if size == 0 { + return true + } + pos += size + } + return false + } + frames := 0 + for pos < len(data) { + switch data[pos] { + case 0x21: + pos += 2 + if !skipBlocks() { + return false + } + case 0x2c: + if pos+10 > len(data) { + return false + } + packed := data[pos+9] + pos += 10 + if packed&0x80 != 0 { + pos += 3 * (1 << ((packed & 7) + 1)) + } + pos++ + if !skipBlocks() { + return false + } + frames++ + if frames > 1 { + return true + } + default: + return false + } + } + return false +} + +func pngAnimation(data []byte) (bool, error) { + if !bytes.HasPrefix(data, []byte("\x89PNG\r\n\x1a\n")) { + return false, nil + } + animated := false + frames := 0 + duration := 0.0 + for pos := 8; pos < len(data); { + if len(data)-pos < 12 { + return false, fmt.Errorf("truncated PNG chunk") + } + length := uint64(binary.BigEndian.Uint32(data[pos : pos+4])) + if length > uint64(len(data)-pos-12) { + return false, fmt.Errorf("invalid PNG chunk length") + } + chunk := data[pos+8 : pos+8+int(length)] + switch string(data[pos+4 : pos+8]) { + case "acTL": + if len(chunk) != 8 { + return false, fmt.Errorf("invalid APNG animation control") + } + animated = true + case "fcTL": + if len(chunk) != 26 { + return false, fmt.Errorf("invalid APNG frame control") + } + frames++ + den := binary.BigEndian.Uint16(chunk[22:24]) + if den == 0 { + den = 100 + } + delay := float64(binary.BigEndian.Uint16(chunk[20:22])) / float64(den) + if delay == 0 { + delay = 0.01 + } + duration += delay + if duration > 60 || frames > 1000 { + return false, fmt.Errorf("APNG exceeds 60 seconds or 1000 frames") + } + case "IEND": + if animated && frames == 0 { + return false, fmt.Errorf("APNG has no frame controls") + } + return animated, nil + } + pos += int(length) + 12 + } + return false, fmt.Errorf("PNG has no end chunk") +} + +func preparePNGSticker(ctx context.Context, data []byte, sticker bool) ([]byte, error) { + if !bytes.HasPrefix(data, []byte("\x89PNG\r\n\x1a\n")) { + return data, nil + } + animated, err := pngAnimation(data) + if err != nil { + return nil, err + } + if !animated && !sticker { + return data, nil + } + config, err := png.DecodeConfig(bytes.NewReader(data)) + if err != nil { + return nil, err + } + if config.Width > 8192 || config.Height > 8192 || int64(config.Width)*int64(config.Height) > 16_000_000 { + return nil, fmt.Errorf("sticker exceeds image dimension limit") + } + if !animated { + img, err := png.Decode(bytes.NewReader(data)) + if err != nil { + return nil, err + } + if opaque, ok := img.(interface{ Opaque() bool }); ok && opaque.Opaque() { + return data, nil + } + } + return convertPNGStickerToGIF(ctx, data, animated) +} + +func convertPNGStickerToGIF(ctx context.Context, data []byte, animated bool) ([]byte, error) { + if len(data) > maxConvertedGIFSize { + return nil, fmt.Errorf("sticker exceeds 100 MiB") + } + ctx, cancel := context.WithTimeout(ctx, 2*time.Minute) + defer cancel() + select { + case gifConversionSlot <- struct{}{}: + defer func() { <-gifConversionSlot }() + case <-ctx.Done(): + return nil, ctx.Err() + } + dir, err := os.MkdirTemp("", "line-png-sticker-*") + if err != nil { + return nil, err + } + defer os.RemoveAll(dir) + input := filepath.Join(dir, "sticker.png") + palette := filepath.Join(dir, "palette.png") + if err = os.WriteFile(input, data, 0600); err != nil { + return nil, err + } + args := []string{"-hide_banner", "-loglevel", "error", "-nostdin", "-threads", "1"} + duration := "60" + if animated { + args = append(args, "-ignore_loop", "1") + } else { + duration = "0.2" + args = append(args, "-loop", "1", "-framerate", "10", "-t", duration) + } + args = append(args, "-i", input) + filter := "scale=w='min(640,iw)':h='min(640,ih)':force_original_aspect_ratio=decrease:flags=lanczos" + paletteArgs := append(slices.Clone(args), + "-t", duration, "-vf", filter+",palettegen=reserve_transparent=1", + "-frames:v", "1", "-threads", "1", "-update", "1", palette) + if err = exec.CommandContext(ctx, "ffmpeg", paletteArgs...).Run(); err != nil { + return nil, fmt.Errorf("sticker palette conversion failed: %w", err) + } + outArgs := append(slices.Clone(args), + "-i", palette, "-t", "60", + "-filter_complex", "[0:v]"+filter+"[v];[v][1:v]paletteuse=alpha_threshold=128", + "-vsync", "0", "-enc_time_base", "1:100", "-loop", "0") + if animated { + delays := pngFrameDelays(data) + outArgs = append(outArgs, "-final_delay", strconv.Itoa(delays[len(delays)-1])) + } else { + outArgs = append(outArgs, "-frames:v", "2") + } + outArgs = append(outArgs, "-f", "gif", "pipe:1") + output := gifOutputBuffer{limit: maxConvertedGIFSize} + cmd := exec.CommandContext(ctx, "ffmpeg", outArgs...) + cmd.Stdout = &output + if err = cmd.Run(); err != nil { + return nil, fmt.Errorf("sticker GIF conversion failed: %w", err) + } + if !isAnimatedGif(output.buffer.Bytes()) { + return nil, fmt.Errorf("sticker conversion did not produce a multi-frame GIF") + } + return output.buffer.Bytes(), nil +} + +func pngFrameDelays(data []byte) []int { + var delays []int + for pos := 8; pos+12 <= len(data); { + if string(data[pos+4:pos+8]) == "IEND" { + return delays + } + n := int(binary.BigEndian.Uint32(data[pos:])) + if string(data[pos+4:pos+8]) == "fcTL" { + chunk := data[pos+8 : pos+8+n] + num := int(binary.BigEndian.Uint16(chunk[20:22])) + den := int(binary.BigEndian.Uint16(chunk[22:24])) + if den == 0 { + den = 100 + } + delays = append(delays, max(1, (num*100+den/2)/den)) + } + pos += n + 12 + } + return delays +} diff --git a/pkg/connector/gif_capabilities_test.go b/pkg/connector/gif_capabilities_test.go new file mode 100644 index 00000000..e491045d --- /dev/null +++ b/pkg/connector/gif_capabilities_test.go @@ -0,0 +1,24 @@ +package connector + +import ( + "testing" + + "maunium.net/go/mautrix/event" +) + +func TestGIFPickerCapabilities(t *testing.T) { + features := (&LineClient{}).GetCapabilities(t.Context(), nil) + gifFeature := features.File[event.CapMsgGIF] + if gifFeature == nil { + t.Fatal("GIF picker capability missing") + } + for mimeType, want := range map[string]event.CapabilitySupportLevel{ + "image/gif": event.CapLevelFullySupported, + "video/mp4": event.CapLevelPartialSupport, + "video/webm": event.CapLevelPartialSupport, + } { + if got := gifFeature.GetMimeSupport(mimeType); got != want { + t.Errorf("%s = %v, want %v", mimeType, got, want) + } + } +} diff --git a/pkg/connector/gif_test.go b/pkg/connector/gif_test.go new file mode 100644 index 00000000..aa002bb9 --- /dev/null +++ b/pkg/connector/gif_test.go @@ -0,0 +1,167 @@ +package connector + +import ( + "bytes" + "context" + "image" + "image/color" + "image/gif" + "io" + "os" + "os/exec" + "path/filepath" + "testing" +) + +func testGIF(t *testing.T, frames int) []byte { + t.Helper() + + palette := color.Palette{color.RGBA{44, 44, 44, 255}, color.White} + animation := &gif.GIF{LoopCount: 0} + for range frames { + animation.Image = append(animation.Image, image.NewPaletted(image.Rect(0, 0, 8, 6), palette)) + animation.Delay = append(animation.Delay, 10) + } + var buf bytes.Buffer + if err := gif.EncodeAll(&buf, animation); err != nil { + t.Fatal(err) + } + return buf.Bytes() +} + +func TestOriginalGIFInfo(t *testing.T) { + for _, frames := range []int{1, 2} { + data := testGIF(t, frames) + info := originalImageInfo(data, "gif") + if info.Category != "original" || info.Extension != "gif" || info.FileSize != len(data) || + info.Width != 8 || info.Height != 6 || info.Animated != (frames > 1) { + t.Fatalf("frames=%d: unexpected metadata: %+v", frames, info) + } + } +} + +func TestGIFFrameDetectionIgnoresCommentsAndTruncation(t *testing.T) { + data := testGIF(t, 1) + + commented := append(bytes.Clone(data[:len(data)-1]), 0x21, 0xfe, 3, 0x2c, 0x2c, 0x2c, 0, 0x3b) + if isAnimatedGif(commented) { + t.Fatal("static GIF with comment was detected as animated") + } + animated := testGIF(t, 2) + for i := range len(animated) { + + isAnimatedGif(animated[:i]) + } + for _, bad := range [][]byte{nil, []byte("GIFxxx,,"), animated[:13]} { + if isAnimatedGif(bad) { + t.Fatalf("invalid GIF detected as animated: %x", bad) + } + } +} + +func TestConvertVideoToGIF(t *testing.T) { + if _, err := exec.LookPath("ffmpeg"); err != nil { + t.Skip("ffmpeg not installed") + } + path := filepath.Join(t.TempDir(), "animation.mp4") + cmd := exec.CommandContext(t.Context(), "ffmpeg", "-hide_banner", "-loglevel", "error", + "-f", "lavfi", "-i", "testsrc2=size=64x48:rate=10:duration=0.5", + "-c:v", "mpeg4", path) + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("create video fixture: %v: %s", err, out) + } + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + converted, err := convertVideoToGIF(t.Context(), data) + if err != nil { + t.Fatal(err) + } + animation, err := gif.DecodeAll(bytes.NewReader(converted)) + if err != nil { + t.Fatal(err) + } + if len(animation.Image) < 2 || animation.LoopCount != 0 || animation.Config.Width != 64 || animation.Config.Height != 48 { + t.Fatalf("conversion did not preserve a looping animation: frames=%d loops=%d config=%+v", + len(animation.Image), animation.LoopCount, animation.Config) + } + if _, err := convertVideoToGIF(t.Context(), []byte("not a video")); err == nil { + t.Fatal("expected invalid video to fail") + } + ctx, cancel := context.WithCancel(t.Context()) + cancel() + if _, err := convertVideoToGIF(ctx, data); err != context.Canceled { + t.Fatalf("canceled conversion: %v", err) + } +} + +func TestGIFOutputLimit(t *testing.T) { + output := gifOutputBuffer{limit: 4} + if _, err := output.Write([]byte("GIF")); err != nil { + t.Fatal(err) + } + if _, err := output.Write([]byte("89a")); err == nil { + t.Fatal("oversized output accepted") + } + if output.buffer.Len() != 3 { + t.Fatal("oversized write changed the buffer") + } +} + +func TestConvertLongVideoToGIF(t *testing.T) { + if _, err := exec.LookPath("ffmpeg"); err != nil { + t.Skip("ffmpeg not installed") + } + path := filepath.Join(t.TempDir(), "long.mp4") + cmd := exec.CommandContext(t.Context(), "ffmpeg", "-hide_banner", "-loglevel", "error", + "-f", "lavfi", "-i", "color=c=red:size=640x640:rate=15:duration=20", "-c:v", "mpeg4", path) + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("fixture: %v: %s", err, out) + } + data, err := os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + converted, err := convertVideoToGIF(t.Context(), data) + if err != nil { + t.Fatal(err) + } + animation, err := gif.DecodeAll(bytes.NewReader(converted)) + if err != nil { + t.Fatal(err) + } + duration := 0 + for _, delay := range animation.Delay { + duration += delay + } + if duration < 1900 || duration > 2100 { + t.Fatalf("animation duration = %d centiseconds", duration) + } + + path = filepath.Join(t.TempDir(), "too-long.mp4") + cmd = exec.CommandContext(t.Context(), "ffmpeg", "-hide_banner", "-loglevel", "error", + "-f", "lavfi", "-i", "color=size=16x16:rate=1:duration=61", "-c:v", "mpeg4", path) + if out, err := cmd.CombinedOutput(); err != nil { + t.Fatalf("fixture: %v: %s", err, out) + } + data, err = os.ReadFile(path) + if err != nil { + t.Fatal(err) + } + if _, err := convertVideoToGIF(t.Context(), data); err == nil { + t.Fatal("long input accepted") + } +} + +func TestGIFOutputLimitThroughCopy(t *testing.T) { + output := gifOutputBuffer{limit: 4} + + input := struct{ io.Reader }{bytes.NewBufferString("GIF89a")} + if _, err := io.Copy(&output, input); err == nil { + t.Fatal("io.Copy bypassed output limit") + } + if output.buffer.Len() > output.limit { + t.Fatal("output exceeded limit") + } +} diff --git a/pkg/connector/handlers/directmedia.go b/pkg/connector/handlers/directmedia.go index f8a47cb1..ba28f4b5 100644 --- a/pkg/connector/handlers/directmedia.go +++ b/pkg/connector/handlers/directmedia.go @@ -4,6 +4,7 @@ import ( "context" "encoding/json" "fmt" + "path/filepath" "strconv" "strings" @@ -141,7 +142,20 @@ func directMediaDuration(contentMetadata map[string]string) int { func directMediaContent(kind, decryptedBody string, metadata map[string]string) (name, contentType string, err error) { switch kind { case "image": - return "image.jpg", "image/jpeg", nil + var original struct { + Extension string `json:"extension"` + } + _ = json.Unmarshal([]byte(metadata["MEDIA_CONTENT_INFO"]), &original) + ext := strings.ToLower(original.Extension) + if ext == "" { + ext = strings.TrimPrefix(strings.ToLower(filepath.Ext(metadata["FILE_NAME"])), ".") + } + mimeType := "image/jpeg" + switch ext { + case "gif", "png", "webp", "bmp": + mimeType = "image/" + ext + } + return "image" + imageSuffix(mimeType), mimeType, nil case "audio": return "audio.m4a", "audio/mp4", nil case "video": diff --git a/pkg/connector/handlers/image.go b/pkg/connector/handlers/image.go index 48ffe2a7..41b64880 100644 --- a/pkg/connector/handlers/image.go +++ b/pkg/connector/handlers/image.go @@ -1,9 +1,12 @@ package handlers import ( + "bytes" "context" "encoding/json" "fmt" + "image" + "net/http" "time" "maunium.net/go/mautrix/bridgev2" @@ -113,7 +116,13 @@ func (h *Handler) ConvertImage(ctx context.Context, portal *bridgev2.Portal, int // Upload to Matrix uploadStart := time.Now() - mxc, file, err := intent.UploadMedia(ctx, portal.MXID, imgData, "image.jpg", "image/jpeg") + mimeType := http.DetectContentType(imgData) + fileName := "image" + imageSuffix(mimeType) + info := &event.FileInfo{MimeType: mimeType, Size: len(imgData)} + if config, _, decodeErr := image.DecodeConfig(bytes.NewReader(imgData)); decodeErr == nil { + info.Width, info.Height = config.Width, config.Height + } + mxc, file, err := intent.UploadMedia(ctx, portal.MXID, imgData, fileName, mimeType) uploadDuration := time.Since(uploadStart) if err != nil { h.Log.Error(). @@ -145,7 +154,8 @@ func (h *Handler) ConvertImage(ctx context.Context, portal *bridgev2.Portal, int Type: event.EventMessage, Content: &event.MessageEventContent{ MsgType: event.MsgImage, - Body: "image.jpg", + Body: fileName, + Info: info, URL: mxc, File: file, RelatesTo: relatesTo, @@ -199,3 +209,22 @@ func lineOBSDownloadOptions(metadata map[string]string, isPlainMedia bool) line. } return opts } + +func imageSuffix(mimeType string) string { + switch mimeType { + case "image/gif": + return ".gif" + case "image/png": + return ".png" + case "image/jpeg": + return ".jpg" + case "image/webp": + return ".webp" + case "image/bmp": + return ".bmp" + case "image/tiff": + return ".tiff" + default: + return "" + } +} diff --git a/pkg/connector/handlers/image_format_test.go b/pkg/connector/handlers/image_format_test.go new file mode 100644 index 00000000..e56b3e6e --- /dev/null +++ b/pkg/connector/handlers/image_format_test.go @@ -0,0 +1,89 @@ +package handlers + +import ( + "bytes" + "context" + "image" + "image/color" + "image/gif" + "image/png" + "io" + "net/http" + "testing" + + "maunium.net/go/mautrix/bridgev2" + "maunium.net/go/mautrix/bridgev2/database" + "maunium.net/go/mautrix/event" + "maunium.net/go/mautrix/id" + + "github.com/highesttt/matrix-line-messenger/pkg/line" +) + +type imageTestTransport func(*http.Request) (*http.Response, error) + +func (f imageTestTransport) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) } + +type incomingImageMatrix struct { + bridgev2.MatrixAPI + data []byte + name, mime string +} + +func (m *incomingImageMatrix) UploadMedia(_ context.Context, _ id.RoomID, data []byte, name, mime string) (id.ContentURIString, *event.EncryptedFileInfo, error) { + m.data, m.name, m.mime = bytes.Clone(data), name, mime + return "mxc://test/original", nil, nil +} + +func TestIncomingImageKeepsOriginalFormat(t *testing.T) { + var pngData, gifData bytes.Buffer + png.Encode(&pngData, image.NewNRGBA(image.Rect(0, 0, 4, 3))) + first := image.NewPaletted(image.Rect(0, 0, 4, 3), color.Palette{color.Transparent, color.White}) + second := image.NewPaletted(first.Bounds(), first.Palette) + second.SetColorIndex(1, 1, 1) + gif.EncodeAll(&gifData, &gif.GIF{Image: []*image.Paletted{first, second}, Delay: []int{10, 20}, LoopCount: 0}) + for _, tc := range []struct { + ext string + data []byte + }{{"png", pngData.Bytes()}, {"gif", gifData.Bytes()}} { + t.Run(tc.ext, func(t *testing.T) { + h := &Handler{HTTPClient: &http.Client{Transport: imageTestTransport(func(r *http.Request) (*http.Response, error) { + return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(bytes.NewReader(tc.data))}, nil + })}} + h.NewClient = func() *line.Client { + c := line.NewClient("test") + c.HTTPClient = h.HTTPClient + c.OBSClient = h.HTTPClient + return c + } + matrix := &incomingImageMatrix{} + converted, err := h.ConvertImage(t.Context(), &bridgev2.Portal{Portal: &database.Portal{MXID: "!room:test"}}, matrix, line.Message{ID: "original", ContentMetadata: map[string]string{"DOWNLOAD_URL": "/r/test/original"}}, "", nil) + if err != nil { + t.Fatal(err) + } + content := converted.Parts[0].Content + if matrix.name != "image."+tc.ext || matrix.mime != "image/"+tc.ext || !bytes.Equal(matrix.data, tc.data) { + t.Fatalf("upload changed format: %s %s", matrix.name, matrix.mime) + } + if content.Info.MimeType != matrix.mime || content.Body != matrix.name || content.Info.Width != 4 || content.Info.Height != 3 || content.Info.Size != len(tc.data) { + t.Fatalf("wrong content: %+v", content) + } + }) + } +} + +func TestDirectImageFormat(t *testing.T) { + for _, tc := range []struct { + metadata map[string]string + name, mime string + }{ + {map[string]string{"MEDIA_CONTENT_INFO": `{"category":"original","extension":"gif","animated":true}`, "FILE_NAME": "wrong.jpg"}, "image.gif", "image/gif"}, + {map[string]string{"MEDIA_CONTENT_INFO": `{"extension":"png"}`}, "image.png", "image/png"}, + {map[string]string{"FILE_NAME": "transparent.art.PNG"}, "image.png", "image/png"}, + {nil, "image.jpg", "image/jpeg"}, + } { + name, mime, err := directMediaContent("image", "", tc.metadata) + if err != nil || name != tc.name || mime != tc.mime { + t.Fatalf("got %q %q %v", name, mime, err) + } + } +} diff --git a/pkg/connector/media.go b/pkg/connector/media.go index a65c3add..98e0d9ef 100644 --- a/pkg/connector/media.go +++ b/pkg/connector/media.go @@ -13,7 +13,7 @@ import ( "image" _ "image/gif" "image/jpeg" - _ "image/png" + "image/png" "io" "os" "os/exec" @@ -35,13 +35,6 @@ func isHEICImage(mimeType, fileName string) bool { return extension == ".heic" || extension == ".heif" } -func jpegFileName(fileName string) string { - if fileName == "" { - return "image.jpg" - } - return strings.TrimSuffix(fileName, filepath.Ext(fileName)) + ".jpg" -} - func convertHEICToJPEG(ctx context.Context, imageData []byte) ([]byte, error) { if err := ctx.Err(); err != nil { return nil, err @@ -236,10 +229,10 @@ func generateThumbnail(imageData []byte) ([]byte, int, int, error) { if width > maxDim || height > maxDim { if width > height { newWidth = maxDim - newHeight = (height * maxDim) / width + newHeight = max(1, (height*maxDim)/width) } else { newHeight = maxDim - newWidth = (width * maxDim) / height + newWidth = max(1, (width*maxDim)/height) } } @@ -252,8 +245,14 @@ func generateThumbnail(imageData []byte) ([]byte, int, int, error) { } var buf bytes.Buffer - if err := jpeg.Encode(&buf, thumbnail, &jpeg.Options{Quality: 60}); err != nil { - return nil, 0, 0, fmt.Errorf("failed to encode thumbnail: %w", err) + var encodeErr error + if opaque, ok := thumbnail.(interface{ Opaque() bool }); !ok || !opaque.Opaque() { + encodeErr = png.Encode(&buf, thumbnail) + } else { + encodeErr = jpeg.Encode(&buf, thumbnail, &jpeg.Options{Quality: 60}) + } + if encodeErr != nil { + return nil, 0, 0, fmt.Errorf("failed to encode thumbnail: %w", encodeErr) } return buf.Bytes(), newWidth, newHeight, nil @@ -294,30 +293,6 @@ func encryptThumbnail(thumbnailData []byte, keyMaterialB64 string) ([]byte, erro return append(encrypted, h.Sum(nil)...), nil } -func isAnimatedGif(data []byte) bool { - // GIF header: "GIF89a" or "GIF87a" - if len(data) < 6 { - return false - } - - if string(data[0:3]) != "GIF" { - return false - } - - // Count image descriptors (0x2C) which indicate frames - frameCount := 0 - for i := 0; i < len(data)-1; i++ { - if data[i] == 0x2C { // Image descriptor separator - frameCount++ - if frameCount > 1 { - return true - } - } - } - - return false -} - // generates the first frame of a video and resizes it to fit within 384x384 func extractVideoThumbnail(videoData []byte) ([]byte, int, int, error) { tmpVideoFile, err := os.CreateTemp("", "video-*.mp4") @@ -385,3 +360,21 @@ func generateChunkHashes(encryptedData []byte) []byte { return allHashes } + +func imageExtension(mimeType string) string { + mimeType, _, _ = strings.Cut(strings.ToLower(mimeType), ";") + switch strings.TrimSpace(mimeType) { + case "image/jpeg", "image/jpg": + return "jpg" + case "image/png", "image/apng": + return "png" + case "image/gif": + return "gif" + case "image/webp": + return "webp" + case "image/bmp", "image/x-ms-bmp": + return "bmp" + default: + return "" + } +} diff --git a/pkg/connector/png_stickers_test.go b/pkg/connector/png_stickers_test.go new file mode 100644 index 00000000..6a041f9b --- /dev/null +++ b/pkg/connector/png_stickers_test.go @@ -0,0 +1,133 @@ +package connector + +import ( + "bytes" + "context" + "encoding/binary" + "hash/crc32" + "image" + "image/color" + "image/gif" + "image/png" + "os/exec" + "slices" + "testing" +) + +func testAPNG(t *testing.T) []byte { + t.Helper() + out := []byte("\x89PNG\r\n\x1a\n") + chunk := func(kind string, data []byte) { + out = binary.BigEndian.AppendUint32(out, uint32(len(data))) + out = append(out, kind...) + out = append(out, data...) + crc := crc32.NewIEEE() + crc.Write([]byte(kind)) + crc.Write(data) + out = binary.BigEndian.AppendUint32(out, crc.Sum32()) + } + var frames [2][]byte + for i := range frames { + img := image.NewNRGBA(image.Rect(0, 0, 8, 8)) + img.SetNRGBA(2+i, 2, color.NRGBA{R: 255, A: 255}) + var b bytes.Buffer + png.Encode(&b, img) + frames[i] = b.Bytes() + } + chunk("IHDR", frames[0][16:29]) + control := make([]byte, 8) + binary.BigEndian.PutUint32(control, 2) + chunk("acTL", control) + seq := uint32(0) + for i, data := range frames { + fc := make([]byte, 26) + binary.BigEndian.PutUint32(fc, seq) + seq++ + binary.BigEndian.PutUint32(fc[4:], 8) + binary.BigEndian.PutUint32(fc[8:], 8) + binary.BigEndian.PutUint16(fc[20:], uint16(i+1)) + binary.BigEndian.PutUint16(fc[22:], 10) + chunk("fcTL", fc) + for pos := 8; pos < len(data); { + n := int(binary.BigEndian.Uint32(data[pos:])) + if string(data[pos+4:pos+8]) == "IDAT" { + payload := data[pos+8 : pos+8+n] + if i == 0 { + chunk("IDAT", payload) + } else { + fd := binary.BigEndian.AppendUint32(nil, seq) + seq++ + chunk("fdAT", append(fd, payload...)) + } + } + pos += n + 12 + } + } + chunk("IEND", nil) + return out +} + +func checkTransparentGIF(t *testing.T, data []byte) *gif.GIF { + t.Helper() + g, err := gif.DecodeAll(bytes.NewReader(data)) + if err != nil { + t.Fatal(err) + } + if len(g.Image) < 2 || g.LoopCount != 0 { + t.Fatalf("not looping animation: frames=%d loop=%d", len(g.Image), g.LoopCount) + } + a := uint32(0) + if image.Pt(0, 0).In(g.Image[0].Bounds()) { + _, _, _, a = g.Image[0].At(0, 0).RGBA() + } + if a != 0 { + t.Fatalf("transparent corner flattened: alpha=%d pixel=%v bounds=%v", a, g.Image[0].At(0, 0), g.Image[0].Bounds()) + } + return g +} + +func TestPNGStickerConversion(t *testing.T) { + if _, err := exec.LookPath("ffmpeg"); err != nil { + t.Skip("ffmpeg required") + } + var static bytes.Buffer + img := image.NewNRGBA(image.Rect(0, 0, 8, 8)) + img.SetNRGBA(2, 2, color.NRGBA{R: 255, A: 255}) + png.Encode(&static, img) + for _, data := range [][]byte{static.Bytes(), testAPNG(t)} { + converted, err := preparePNGSticker(t.Context(), data, true) + if err != nil { + t.Fatal(err) + } + g := checkTransparentGIF(t, converted) + if len(g.Image) != 2 { + t.Fatal("frame count", len(g.Image)) + } + } + original, err := preparePNGSticker(t.Context(), static.Bytes(), false) + if err != nil || !bytes.Equal(original, static.Bytes()) { + t.Fatal("ordinary PNG changed", err) + } + ctx, cancel := context.WithCancel(t.Context()) + cancel() + if _, err := preparePNGSticker(ctx, testAPNG(t), true); err == nil { + t.Fatal("ignored cancellation") + } + fixture := testAPNG(t) + for i := 8; i < len(fixture); i++ { + _, _ = pngAnimation(fixture[:i]) + } +} + +func TestAPNGTrailingChunksIgnored(t *testing.T) { + valid := testAPNG(t) + + trailing := append(bytes.Clone(valid), 0, 0, 0, 0, 'f', 'c', 'T', 'L', 0, 0, 0, 0) + animated, err := pngAnimation(trailing) + if err != nil || !animated { + t.Fatal("valid APNG rejected", err) + } + if !slices.Equal(pngFrameDelays(trailing), pngFrameDelays(valid)) { + t.Fatal("parsed trailing bytes") + } +} diff --git a/pkg/connector/reaction_test.go b/pkg/connector/reaction_test.go index 6a0eeed7..3d509971 100644 --- a/pkg/connector/reaction_test.go +++ b/pkg/connector/reaction_test.go @@ -20,8 +20,8 @@ import ( func TestCapabilitiesAdvertiseSupportedReactions(t *testing.T) { infoVersion, capabilityVersion := (&LineConnector{}).GetBridgeInfoVersion() - if infoVersion != 1 || capabilityVersion != 3 { - t.Fatalf("bridge info/capability versions = %d/%d, want 1/3", infoVersion, capabilityVersion) + if infoVersion != 1 || capabilityVersion != 4 { + t.Fatalf("bridge info/capability versions = %d/%d, want 1/4", infoVersion, capabilityVersion) } caps := (&LineClient{}).GetCapabilities(context.Background(), nil) diff --git a/pkg/connector/send_message.go b/pkg/connector/send_message.go index fd542056..b369ec98 100644 --- a/pkg/connector/send_message.go +++ b/pkg/connector/send_message.go @@ -9,6 +9,8 @@ import ( "fmt" "image" "image/jpeg" + "net/http" + "path/filepath" "regexp" "strconv" "strings" @@ -123,6 +125,10 @@ func (lc *LineClient) HandleMatrixMessage(ctx context.Context, msg *bridgev2.Mat // Matrix clients often send media as MsgFile (e.g. drag-and-drop), which would otherwise // be sent as ContentFile (14) instead of the appropriate media type on LINE. effectiveMsgType := msg.Content.MsgType + isSticker := effectiveMsgType == event.CapMsgSticker + if isSticker { + effectiveMsgType = event.MsgImage + } if effectiveMsgType == event.MsgFile && msg.Content.Info != nil { mime := msg.Content.Info.MimeType if strings.HasPrefix(mime, "audio/") { @@ -134,6 +140,11 @@ func (lc *LineClient) HandleMatrixMessage(ctx context.Context, msg *bridgev2.Mat } } + videoGIF := effectiveMsgType == event.MsgVideo && msg.Content.Info != nil && msg.Content.Info.MauGIF + if videoGIF { + effectiveMsgType = event.MsgImage + } + // For non-text messages, check with the server whether to use E2EE or plain media upload. // This must happen before media processing since it affects the upload path. useE2EEMedia := !plainText @@ -185,73 +196,60 @@ func (lc *LineClient) HandleMatrixMessage(ctx context.Context, msg *bridgev2.Mat return nil, fmt.Errorf("failed to download media from matrix: %w", err) } - mimeType := msg.Content.Info.MimeType + mimeType := "" + if msg.Content.Info != nil { + mimeType = msg.Content.Info.MimeType + } fileName := msg.Content.GetFileName() - if isHEICImage(mimeType, fileName) { + if videoGIF { + data, err = convertVideoToGIF(ctx, data) + if err != nil { + return nil, fmt.Errorf("failed to convert GIF video: %w", err) + } + mimeType = "image/gif" + } + data, err = preparePNGSticker(ctx, data, isSticker) + if err != nil { + return nil, fmt.Errorf("failed to prepare PNG sticker: %w", err) + } + detectedMimeType := http.DetectContentType(data) + if strings.HasPrefix(detectedMimeType, "image/") { + mimeType = detectedMimeType + } else if isHEICImage(mimeType, fileName) { data, err = convertHEICToJPEG(ctx, data) if err != nil { return nil, fmt.Errorf("failed to convert HEIC image to JPEG: %w", err) } mimeType = "image/jpeg" - fileName = jpegFileName(fileName) } - isGif := mimeType == "image/gif" - isAnimated := isGif && isAnimatedGif(data) - - extension := "jpg" - if isGif { - extension = "gif" - } else if mimeType == "image/png" { - extension = "png" + extension := imageExtension(mimeType) + if extension == "" { + return nil, fmt.Errorf("unsupported image MIME type %q", mimeType) + } + if fileName == "" { + fileName = "image" } + fileName = strings.TrimSuffix(fileName, filepath.Ext(fileName)) + "." + extension contentType = int(ContentImage) + mediaContentInfo := originalImageInfo(data, extension) + if mediaInfoJSON, err := json.Marshal(mediaContentInfo); err == nil { + contentMetadata["MEDIA_CONTENT_INFO"] = string(mediaInfoJSON) + } + if plainText { // Plain media: save data for post-send upload to r/talk/m/{msgId} plainMediaData = data - - thumbnailData, thumbWidth, thumbHeight, err := generateThumbnail(data) - if err != nil { - lc.UserLogin.Bridge.Log.Warn().Err(err).Msg("Failed to generate thumbnail, continuing without it") - } else { - plainThumbData = thumbnailData - mediaThumbInfo := map[string]interface{}{ - "width": thumbWidth, - "height": thumbHeight, - } - if thumbInfoJSON, err := json.Marshal(mediaThumbInfo); err == nil { - contentMetadata["MEDIA_THUMB_INFO"] = string(thumbInfoJSON) - } - } - contentMetadata["FILE_SIZE"] = fmt.Sprintf("%d", len(data)) contentMetadata["contentType"] = fmt.Sprintf("%d", ContentImage) - if fileName == "" { - fileName = "image." + extension - } contentMetadata["FILE_NAME"] = fileName - - mediaContentInfo := map[string]interface{}{ - "category": "original", - "fileSize": len(data), - "extension": extension, - } - if isAnimated { - mediaContentInfo["animated"] = true - } - if mediaInfoJSON, err := json.Marshal(mediaContentInfo); err == nil { - contentMetadata["MEDIA_CONTENT_INFO"] = string(mediaInfoJSON) - } } else { // E2EE: encrypt, upload to OBS first, send with OID // Save original data for potential group E2EE fallback if isGroup { originalMediaData = data - if thumbData, _, _, tErr := generateThumbnail(data); tErr == nil { - originalThumbData = thumbData - } } uploadData, keyMaterialB64, err := lc.encryptFileData(data) @@ -297,27 +295,12 @@ func (lc *LineClient) HandleMatrixMessage(ctx context.Context, msg *bridgev2.Mat contentMetadata["OID"] = oid contentMetadata["SID"] = "emi" - contentMetadata["FILE_SIZE"] = fmt.Sprintf("%d", len(uploadData)) + contentMetadata["FILE_SIZE"] = fmt.Sprintf("%d", len(data)) contentMetadata["contentType"] = fmt.Sprintf("%d", ContentImage) contentMetadata["ENC_KM"] = keyMaterialB64 - if fileName == "" { - fileName = "image." + extension - } contentMetadata["FILE_NAME"] = fileName - mediaContentInfo := map[string]interface{}{ - "category": "original", - "fileSize": len(uploadData), - "extension": extension, - } - if isAnimated { - mediaContentInfo["animated"] = true - } - if mediaInfoJSON, err := json.Marshal(mediaContentInfo); err == nil { - contentMetadata["MEDIA_CONTENT_INFO"] = string(mediaInfoJSON) - } - imgPayload := map[string]string{"keyMaterial": keyMaterialB64} payload, _ = json.Marshal(imgPayload) } @@ -637,6 +620,9 @@ func (lc *LineClient) HandleMatrixMessage(ctx context.Context, msg *bridgev2.Mat delete(contentMetadata, "ENC_KM") plainMediaData = originalMediaData plainThumbData = originalThumbData + if contentType == int(ContentImage) { + delete(contentMetadata, "MEDIA_THUMB_INFO") + } } } } @@ -833,6 +819,9 @@ func (lc *LineClient) HandleMatrixMessage(ctx context.Context, msg *bridgev2.Mat } if err := callLineErr(func(client *line.Client) error { + if contentType == int(ContentImage) { + return client.UploadOBSPlainOriginalImage(plainMediaData, sentMsg.ID, contentMetadata["FILE_NAME"]) + } return client.UploadOBSPlain(plainMediaData, sentMsg.ID, obsType) }); err != nil { return nil, fmt.Errorf("failed to upload plain media to OBS: %w", err) diff --git a/pkg/connector/transparency_test.go b/pkg/connector/transparency_test.go new file mode 100644 index 00000000..cffaf123 --- /dev/null +++ b/pkg/connector/transparency_test.go @@ -0,0 +1,100 @@ +package connector + +import ( + "bytes" + "image" + "image/color" + "image/png" + "testing" +) + +func TestTransparentPreviewPreservesAlpha(t *testing.T) { + for _, size := range []image.Point{{8, 6}, {2560, 1280}, {1, 2560}} { + t.Run(size.String(), func(t *testing.T) { + src := image.NewNRGBA(image.Rect(0, 0, size.X, size.Y)) + src.SetNRGBA(0, 0, color.NRGBA{R: 255, A: 128}) + var encoded bytes.Buffer + if err := png.Encode(&encoded, src); err != nil { + t.Fatal(err) + } + data, w, h, err := generateThumbnail(encoded.Bytes()) + if err != nil { + t.Fatal(err) + } + decoded, format, err := image.Decode(bytes.NewReader(data)) + if err != nil { + t.Fatal(err) + } + if format != "png" || w < 1 || h < 1 || w > 1280 || h > 1280 { + t.Fatalf("format=%s dimensions=%dx%d", format, w, h) + } + _, _, _, alpha := decoded.At(w-1, h-1).RGBA() + if alpha != 0 { + t.Fatalf("transparent pixel alpha=%d", alpha) + } + if size.X == 8 { + _, _, _, a := decoded.At(0, 0).RGBA() + if a != 128*257 { + t.Fatalf("partial alpha changed: %d", a) + } + } + }) + } +} + +func TestOpaquePreviewRemainsJPEG(t *testing.T) { + src := image.NewNRGBA(image.Rect(0, 0, 2, 2)) + for y := range 2 { + for x := range 2 { + src.SetNRGBA(x, y, color.NRGBA{R: 255, A: 255}) + } + } + var encoded bytes.Buffer + png.Encode(&encoded, src) + data, _, _, err := generateThumbnail(encoded.Bytes()) + if err != nil { + t.Fatal(err) + } + _, format, err := image.Decode(bytes.NewReader(data)) + if err != nil || format != "jpeg" { + t.Fatalf("format=%s err=%v", format, err) + } +} + +func TestEncryptedImageOriginalAndPreviewPreserveAlpha(t *testing.T) { + var original bytes.Buffer + png.Encode(&original, image.NewNRGBA(image.Rect(0, 0, 8, 6))) + client := &LineClient{} + for _, data := range [][]byte{original.Bytes(), testGIF(t, 2)} { + encrypted, key, err := client.encryptFileData(data) + if err != nil { + t.Fatal(err) + } + decrypted, err := client.decryptImageData(encrypted, key) + if err != nil || !bytes.Equal(data, decrypted) { + t.Fatal("encrypted original changed", err) + } + preview, _, _, err := generateThumbnail(data) + if err != nil { + t.Fatal(err) + } + ciphertext, err := encryptThumbnail(preview, key) + if err != nil { + t.Fatal(err) + } + decodedPreview, err := client.decryptImageData(ciphertext, key) + if err != nil || !bytes.Equal(preview, decodedPreview) { + t.Fatal("encrypted preview changed", err) + } + if bytes.Equal(data, original.Bytes()) { + img, err := png.Decode(bytes.NewReader(decodedPreview)) + if err != nil { + t.Fatal(err) + } + _, _, _, alpha := img.At(0, 0).RGBA() + if alpha != 0 { + t.Fatal("encrypted preview lost transparency") + } + } + } +} diff --git a/pkg/connector/userinfo.go b/pkg/connector/userinfo.go index 835af9ed..c80a5262 100644 --- a/pkg/connector/userinfo.go +++ b/pkg/connector/userinfo.go @@ -63,6 +63,24 @@ func (lc *LineClient) GetCapabilities(ctx context.Context, portal *bridgev2.Port AcceptWithButton: event.CapLevelFullySupported, }, File: event.FileFeatureMap{ + event.CapMsgSticker: { + Caption: event.CapLevelRejected, + MaxSize: handlers.BeeperMaxFileSize, + MimeTypes: map[string]event.CapabilitySupportLevel{ + "image/png": event.CapLevelPartialSupport, + "image/gif": event.CapLevelPartialSupport, + "image/webp": event.CapLevelPartialSupport, + }, + }, + event.CapMsgGIF: { + Caption: event.CapLevelRejected, + MaxSize: handlers.BeeperMaxFileSize, + MimeTypes: map[string]event.CapabilitySupportLevel{ + "image/gif": event.CapLevelFullySupported, + "video/mp4": event.CapLevelPartialSupport, + "video/webm": event.CapLevelPartialSupport, + }, + }, event.MsgImage: { Caption: event.CapLevelRejected, MaxSize: handlers.BeeperMaxFileSize, diff --git a/pkg/line/client.go b/pkg/line/client.go index 91bcfdc6..06350fc1 100644 --- a/pkg/line/client.go +++ b/pkg/line/client.go @@ -537,6 +537,14 @@ func obsTypeFromSID(sid string) string { // UploadOBSPlain uploads plain (non-E2EE) media to a specific OID via the "m" endpoint. // obsType should be "image", "video", "audio", or "file". func (c *Client) UploadOBSPlain(data []byte, oid string, obsType string) error { + return c.uploadOBSPlain(data, oid, obsType, "", "") +} + +func (c *Client) UploadOBSPlainOriginalImage(data []byte, oid, fileName string) error { + return c.uploadOBSPlain(data, oid, "image", fileName, "original") +} + +func (c *Client) uploadOBSPlain(data []byte, oid, obsType, fileName, category string) error { obsToken, err := c.AcquireEncryptedAccessToken() if err != nil { return fmt.Errorf("failed to acquire OBS token: %w", err) @@ -554,6 +562,12 @@ func (c *Client) UploadOBSPlain(data []byte, oid string, obsType string) error { "name": fmt.Sprintf("%d", time.Now().UnixMilli()), "type": obsType, } + if fileName != "" { + obsParams["name"] = fileName + } + if category != "" { + obsParams["cat"] = category + } obsParamsJSON, _ := json.Marshal(obsParams) obsParamsB64 := base64.StdEncoding.EncodeToString(obsParamsJSON) diff --git a/pkg/line/obs_test.go b/pkg/line/obs_test.go index 5716da27..187386c7 100644 --- a/pkg/line/obs_test.go +++ b/pkg/line/obs_test.go @@ -2,6 +2,8 @@ package line import ( "context" + "encoding/base64" + "encoding/json" "errors" "io" "net/http" @@ -10,6 +12,51 @@ import ( "time" ) +func TestUploadOBSPlainOriginalImage(t *testing.T) { + installCachedOBSToken(t) + client := NewClient("line-token") + requests := 0 + client.OBSClient = &http.Client{Transport: roundTripFunc(func(req *http.Request) (*http.Response, error) { + requests++ + paramsJSON, err := base64.StdEncoding.DecodeString(req.Header.Get("X-Obs-Params")) + if err != nil { + t.Fatal(err) + } + var params map[string]string + if err = json.Unmarshal(paramsJSON, ¶ms); err != nil { + t.Fatal(err) + } + if params["ver"] != "2.0" || params["type"] != "image" { + t.Fatalf("unexpected OBS params: %v", params) + } + if requests == 1 { + if params["cat"] != "original" || params["name"] != "animation.gif" { + t.Fatalf("original GIF upload missing Chrome parameters: %v", params) + } + } else if _, ok := params["cat"]; ok { + t.Fatal("regular upload unexpectedly has original category") + } + body, _ := io.ReadAll(req.Body) + if string(body) != "GIF89a-original-bytes" || req.Method != http.MethodPost || req.URL.Path != "/r/talk/m/message-id" { + t.Fatalf("unexpected upload: %s %s %q", req.Method, req.URL.Path, body) + } + if req.Header.Get("X-Line-Access") != "obs-token" { + t.Fatal("missing OBS token") + } + return obsResponse(http.StatusCreated, ""), nil + })} + data := []byte("GIF89a-original-bytes") + if err := client.UploadOBSPlainOriginalImage(data, "message-id", "animation.gif"); err != nil { + t.Fatal(err) + } + if err := client.UploadOBSPlain(data, "message-id", "image"); err != nil { + t.Fatal(err) + } + if requests != 2 { + t.Fatalf("requests = %d, want 2", requests) + } +} + type observedOBSRequest struct { path string query string From 659fb81b21699a8e01494d56f7c8016603298996 Mon Sep 17 00:00:00 2001 From: formeo14 Date: Sun, 27 Sep 2026 13:20:33 +0900 Subject: [PATCH 2/4] feat: bridge owned LINE sticker packs --- pkg/connector/client.go | 21 +- pkg/connector/connector.go | 6 +- pkg/connector/creategroup.go | 4 +- pkg/connector/handlers/sticker.go | 25 +- pkg/connector/handlers/sticker_test.go | 57 +++ pkg/connector/send_message.go | 23 +- pkg/connector/sticker_commands.go | 31 ++ pkg/connector/stickers.go | 645 +++++++++++++++++++++++++ pkg/connector/stickers_test.go | 324 +++++++++++++ pkg/connector/sync.go | 6 +- pkg/connector/userinfo.go | 4 +- pkg/line/stickers.go | 77 +++ pkg/line/stickers_test.go | 57 +++ 13 files changed, 1260 insertions(+), 20 deletions(-) create mode 100644 pkg/connector/handlers/sticker_test.go create mode 100644 pkg/connector/sticker_commands.go create mode 100644 pkg/connector/stickers.go create mode 100644 pkg/connector/stickers_test.go create mode 100644 pkg/line/stickers.go create mode 100644 pkg/line/stickers_test.go diff --git a/pkg/connector/client.go b/pkg/connector/client.go index b0e3786f..ea4687a0 100644 --- a/pkg/connector/client.go +++ b/pkg/connector/client.go @@ -82,10 +82,16 @@ type LineClient struct { paidReactionIconMXC map[string]string // LINE sticon URL -> cached MXC URI unblockBackfills sync.Map // chat MID -> *unblockBackfillState while unblock history restoration is active + stickerSyncMu sync.Mutex + stickerMu sync.Mutex + stickerCatalogs map[string]stickerCatalog + pendingStickerRooms sync.Map + wg sync.WaitGroup } type lineClientRun struct { + ctx context.Context cancel context.CancelFunc } @@ -156,7 +162,7 @@ func (lc *LineClient) isSessionInvalidated() bool { func (lc *LineClient) beginRun(parent context.Context) (context.Context, *lineClientRun, bool) { ctx, cancel := context.WithCancel(parent) - run := &lineClientRun{cancel: cancel} + run := &lineClientRun{ctx: ctx, cancel: cancel} lc.runMu.Lock() if lc.stopped { lc.runMu.Unlock() @@ -184,6 +190,16 @@ func (lc *LineClient) cancelActiveRun() { } } +func (lc *LineClient) startRunTask() (context.Context, bool) { + lc.runMu.Lock() + defer lc.runMu.Unlock() + if lc.stopped || lc.activeRun == nil || lc.activeRun.ctx.Err() != nil { + return nil, false + } + lc.wg.Add(1) + return lc.activeRun.ctx, true +} + func (lc *LineClient) retire() { lc.Disconnect() } @@ -641,7 +657,8 @@ func (lc *LineClient) Connect(ctx context.Context) { return } - lc.wg.Add(3) + lc.wg.Add(4) + go lc.syncStickerPacks(ctx) go lc.syncDMChats(ctx) go lc.prefetchMessages(ctx) go lc.pollLoop(ctx) diff --git a/pkg/connector/connector.go b/pkg/connector/connector.go index 6e7a8d70..dbb71582 100644 --- a/pkg/connector/connector.go +++ b/pkg/connector/connector.go @@ -14,6 +14,7 @@ import ( "go.mau.fi/util/configupgrade" "maunium.net/go/mautrix/bridgev2" + "maunium.net/go/mautrix/bridgev2/commands" "maunium.net/go/mautrix/bridgev2/database" "maunium.net/go/mautrix/bridgev2/networkid" "maunium.net/go/mautrix/bridgev2/status" @@ -41,6 +42,9 @@ func (lc *LineConnector) Init(bridge *bridgev2.Bridge) { // user doesn't already have one. bridge.Config.BridgeStatusNotices = "none" lc.br = bridge + if processor, ok := bridge.Commands.(*commands.Processor); ok { + processor.AddHandlers(commandStickers) + } } func (lc *LineConnector) Start(ctx context.Context) error { @@ -59,7 +63,7 @@ func (lc *LineConnector) GetCapabilities() *bridgev2.NetworkGeneralCapabilities return &bridgev2.NetworkGeneralCapabilities{ AggressiveUpdateInfo: true, Provisioning: bridgev2.ProvisioningCapabilities{ - ImagePackImport: false, + ImagePackImport: true, ResolveIdentifier: bridgev2.ResolveIdentifierCapabilities{ Search: true, ContactList: true, diff --git a/pkg/connector/creategroup.go b/pkg/connector/creategroup.go index 9119908a..2b8420eb 100644 --- a/pkg/connector/creategroup.go +++ b/pkg/connector/creategroup.go @@ -119,14 +119,14 @@ func (lc *LineClient) CreateGroup(ctx context.Context, params *bridgev2.GroupCre return &bridgev2.CreateChatResponse{ PortalKey: portalKey, Portal: portal, - PortalInfo: &bridgev2.ChatInfo{ + PortalInfo: lc.withStickerPacks(&bridgev2.ChatInfo{ Type: &ct, Name: &chatName, Members: &bridgev2.ChatMemberList{ IsFull: true, MemberMap: chatMemberMapFromList(members), }, - }, + }), }, nil } diff --git a/pkg/connector/handlers/sticker.go b/pkg/connector/handlers/sticker.go index 330f51c4..86656452 100644 --- a/pkg/connector/handlers/sticker.go +++ b/pkg/connector/handlers/sticker.go @@ -1,10 +1,12 @@ package handlers import ( + "bytes" "context" "encoding/binary" "fmt" "hash/crc32" + "image" "io" "strings" @@ -14,7 +16,7 @@ import ( "github.com/highesttt/matrix-line-messenger/pkg/line" ) -// ConvertSticker converts a LINE sticker message to a Matrix image or text fallback. +// ConvertSticker converts a LINE sticker message to a Matrix sticker or text fallback. func (h *Handler) ConvertSticker(ctx context.Context, portal *bridgev2.Portal, intent bridgev2.MatrixAPI, data line.Message, relatesTo *event.RelatesTo) (*bridgev2.ConvertedMessage, error) { stkID := data.ContentMetadata["STKID"] stkTxt := data.ContentMetadata["STKTXT"] @@ -75,7 +77,12 @@ func (h *Handler) ConvertSticker(ctx context.Context, portal *bridgev2.Portal, i ext = "png" } + dimensions, _, _ := image.DecodeConfig(bytes.NewReader(stkData)) bodyName := "sticker." + ext + body := stkTxt + if body == "" { + body = bodyName + } mxc, file, err := intent.UploadMedia(ctx, portal.MXID, stkData, bodyName, mimeType) if err != nil { @@ -84,15 +91,17 @@ func (h *Handler) ConvertSticker(ctx context.Context, portal *bridgev2.Portal, i return &bridgev2.ConvertedMessage{ Parts: []*bridgev2.ConvertedMessagePart{ { - Type: event.EventMessage, + Type: event.EventSticker, Content: &event.MessageEventContent{ - MsgType: event.MsgImage, - Body: bodyName, - URL: mxc, - File: file, + Body: body, + URL: mxc, + File: file, Info: &event.FileInfo{ - MimeType: mimeType, - Size: len(stkData), + MimeType: mimeType, + Width: dimensions.Width, + Height: dimensions.Height, + IsAnimated: isAnimated, + Size: len(stkData), }, RelatesTo: relatesTo, }, diff --git a/pkg/connector/handlers/sticker_test.go b/pkg/connector/handlers/sticker_test.go new file mode 100644 index 00000000..7f18b14b --- /dev/null +++ b/pkg/connector/handlers/sticker_test.go @@ -0,0 +1,57 @@ +package handlers + +import ( + "bytes" + "context" + "image" + "image/png" + "io" + "net/http" + "testing" + + "maunium.net/go/mautrix/bridgev2" + "maunium.net/go/mautrix/bridgev2/database" + "maunium.net/go/mautrix/event" + "maunium.net/go/mautrix/id" + + "github.com/highesttt/matrix-line-messenger/pkg/line" +) + +type incomingStickerTransport func(*http.Request) (*http.Response, error) + +func (f incomingStickerTransport) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) } + +type incomingStickerMatrix struct{ bridgev2.MatrixAPI } + +func (m incomingStickerMatrix) UploadMedia(context.Context, id.RoomID, []byte, string, string) (id.ContentURIString, *event.EncryptedFileInfo, error) { + return "mxc://test/sticker", nil, nil +} + +func TestIncomingStickerUsesNativeMatrixEvent(t *testing.T) { + var data bytes.Buffer + if err := png.Encode(&data, image.NewRGBA(image.Rect(0, 0, 4, 3))); err != nil { + t.Fatal(err) + } + h := &Handler{HTTPClient: &http.Client{Transport: incomingStickerTransport(func(r *http.Request) (*http.Response, error) { + if r.URL.Path != "/stickershop/v1/sticker/100/android/sticker.png" { + t.Fatal(r.URL) + } + return &http.Response{ + StatusCode: http.StatusOK, + Header: http.Header{"Content-Type": []string{"image/png"}}, + Body: io.NopCloser(bytes.NewReader(data.Bytes())), + }, nil + })}} + portal := &bridgev2.Portal{Portal: &database.Portal{MXID: "!room:test"}} + msg := line.Message{ContentMetadata: map[string]string{"STKID": "100", "STKTXT": "waving cat"}} + converted, err := h.ConvertSticker(t.Context(), portal, incomingStickerMatrix{}, msg, nil) + if err != nil { + t.Fatal(err) + } + part := converted.Parts[0] + content := part.Content + if part.Type != event.EventSticker || content.MsgType != "" || content.Body != "waving cat" || + content.URL != "mxc://test/sticker" || content.Info.Width != 4 || content.Info.Height != 3 { + t.Fatalf("incorrect sticker event: %+v", part) + } +} diff --git a/pkg/connector/send_message.go b/pkg/connector/send_message.go index b369ec98..355063e2 100644 --- a/pkg/connector/send_message.go +++ b/pkg/connector/send_message.go @@ -94,9 +94,22 @@ func (lc *LineClient) HandleMatrixMessage(ctx context.Context, msg *bridgev2.Mat WithErrorReason(event.MessageStatusGenericError) } + var nativeSticker *lineSticker + if (msg.Content.MsgType == event.CapMsgSticker || msg.Content.MsgType == event.MsgImage) && msg.Content.File == nil { + sticker, err := lc.resolveSticker(ctx, msg.Content.URL) + if err != nil { + lc.UserLogin.Log.Warn().Err(err).Msg("Failed to resolve LINE sticker, sending as image") + } + if sticker != nil && sticker.Shop == line.StickerShop { + nativeSticker = sticker + } + } + // Determine whether we need to send as plain text (peer/group has Letter Sealing off). plainText := false - if lc.E2EE == nil { + if nativeSticker != nil { + plainText = true + } else if lc.E2EE == nil { plainText = true lc.UserLogin.Bridge.Log.Warn().Msg("E2EE not initialized, sending as plain text") } else if isGroup { @@ -126,7 +139,9 @@ func (lc *LineClient) HandleMatrixMessage(ctx context.Context, msg *bridgev2.Mat // be sent as ContentFile (14) instead of the appropriate media type on LINE. effectiveMsgType := msg.Content.MsgType isSticker := effectiveMsgType == event.CapMsgSticker - if isSticker { + if nativeSticker != nil { + effectiveMsgType = event.CapMsgSticker + } else if isSticker { effectiveMsgType = event.MsgImage } if effectiveMsgType == event.MsgFile && msg.Content.Info != nil { @@ -179,6 +194,10 @@ func (lc *LineClient) HandleMatrixMessage(ctx context.Context, msg *bridgev2.Mat var rawFileName string switch effectiveMsgType { + case event.CapMsgSticker: + contentType = int(ContentSticker) + contentMetadata = nativeSticker.metadata() + case event.MsgText: contentType = int(ContentText) if plainText { diff --git a/pkg/connector/sticker_commands.go b/pkg/connector/sticker_commands.go new file mode 100644 index 00000000..0a06086b --- /dev/null +++ b/pkg/connector/sticker_commands.go @@ -0,0 +1,31 @@ +package connector + +import "maunium.net/go/mautrix/bridgev2/commands" + +var commandStickers = &commands.FullHandler{ + Name: "stickers", + Help: commands.HelpMeta{ + Section: commands.HelpSectionMisc, + Description: "Add your LINE sticker packs to this room and refresh them", + }, + RequiresPortal: true, + RequiresLogin: true, + Func: func(ce *commands.Event) { + login, _, err := ce.Portal.FindPreferredLogin(ce.Ctx, ce.User, false) + if err != nil || login == nil { + ce.Reply("Could not find your LINE login for this room.") + return + } + client, ok := login.Client.(*LineClient) + if !ok { + ce.Reply("This room does not use a LINE login.") + return + } + client.resetStickerCatalogs() + if err = client.enableRoomStickers(ce.Ctx, ce.RoomID); err != nil { + ce.Reply("Failed to synchronize LINE sticker packs: %s", err) + return + } + ce.Reply("LINE sticker packs are now available in this room. Pack images are not end-to-end encrypted.") + }, +} diff --git a/pkg/connector/stickers.go b/pkg/connector/stickers.go new file mode 100644 index 00000000..2cdc7fa7 --- /dev/null +++ b/pkg/connector/stickers.go @@ -0,0 +1,645 @@ +package connector + +import ( + "bytes" + "context" + "crypto/sha256" + "encoding/json" + "fmt" + "image" + "io" + "net/http" + "net/url" + "regexp" + "slices" + "strconv" + "strings" + "time" + + "maunium.net/go/mautrix/bridgev2" + "maunium.net/go/mautrix/bridgev2/database" + "maunium.net/go/mautrix/bridgev2/provisionutil" + "maunium.net/go/mautrix/event" + "maunium.net/go/mautrix/id" + + "github.com/rs/zerolog" + + "github.com/highesttt/matrix-line-messenger/pkg/line" +) + +var _ bridgev2.StickerImportingNetworkAPI = (*LineClient)(nil) + +var lineAssetID = regexp.MustCompile(`^[A-Za-z0-9_-]{1,100}$`) + +var stickerShops = []string{line.StickerShop} + +const ( + stickerCDN = "https://stickershop.line-scdn.net" + maxStickerAssetSize = 10 * 1024 * 1024 + maxStickersPerPack = 1000 + stickerCatalogTTL = 5 * time.Minute + stickerSyncInterval = time.Hour + stickerRoomCreateTimeout = 10 * time.Minute +) + +type stickerCatalog struct { + Products []line.ShopProduct + Fetched time.Time +} + +type lineSticker struct { + Shop string `json:"shop"` + ProductID string `json:"product_id"` + ID string `json:"id"` + Version string `json:"version"` + Option string `json:"option,omitempty"` + Hash string `json:"hash,omitempty"` + Body string `json:"body"` +} + +func (s lineSticker) packURL() string { + return "line://" + s.Shop + "/" + s.ProductID +} + +func (s lineSticker) metadata() map[string]string { + metadata := map[string]string{ + "STKPKGID": s.ProductID, + "STKID": s.ID, + "STKVER": s.Version, + "STKTXT": s.Body, + } + if s.Option != "" { + metadata["STKOPT"] = s.Option + } + if s.Hash != "" { + metadata["STKHASH"] = s.Hash + } + return metadata +} + +func (s lineSticker) assetPath() string { + prefix := "/stickershop/v1/sticker/" + s.ID + if s.Hash != "" { + prefix = "/stickershop/v2/sticker/" + s.ID + "/" + s.Hash + } + name := "sticker.png" + if strings.Contains(s.Option, "A") { + name = "sticker_animation.png" + } else if strings.Contains(s.Option, "P") { + name = "sticker_popup.png" + } + return prefix + "/android/" + name +} + +func stickerOption(resourceType int) (string, error) { + switch resourceType { + case 1: + return "", nil + case 2: + return "A", nil + case 3: + return "S", nil + case 4: + return "AS", nil + case 5: + return "P", nil + case 6: + return "PS", nil + default: + return "", fmt.Errorf("LINE sticker type %d requires unsupported personalization", resourceType) + } +} + +func activeProduct(product line.ShopProduct, now time.Time) bool { + expiry, err := product.ValidUntil.Int64() + return err == nil && (expiry == -1 || expiry > now.UnixMilli()) +} + +func parseLinePackURL(rawURL string) (shop, productID string, err error) { + parsed, err := url.Parse(rawURL) + if err != nil { + return "", "", err + } + if parsed.User != nil || parsed.RawQuery != "" || parsed.Fragment != "" { + return "", "", fmt.Errorf("invalid LINE pack URL") + } + parts := strings.Split(strings.Trim(parsed.Path, "/"), "/") + switch { + case parsed.Scheme == "line" && slices.Contains(stickerShops, parsed.Host) && len(parts) == 1: + shop, productID = parsed.Host, parts[0] + case parsed.Scheme == "https" && parsed.Host == "store.line.me" && len(parts) >= 3 && len(parts) <= 4 && parts[1] == "product": + shop, err = shopFromStorePath(parts[0]) + if err != nil { + return "", "", err + } + productID = parts[2] + default: + return "", "", fmt.Errorf("use a LINE Store product URL or line://stickershop/PACK_ID") + } + if !lineAssetID.MatchString(productID) { + return "", "", fmt.Errorf("invalid LINE product ID") + } + return shop, productID, nil +} + +func shopFromStorePath(path string) (string, error) { + switch path { + case "stickershop": + return line.StickerShop, nil + default: + return "", fmt.Errorf("unsupported LINE Store section %q", path) + } +} + +func (lc *LineClient) ownedStickerProducts(ctx context.Context, shop string) ([]line.ShopProduct, error) { + lc.stickerMu.Lock() + defer lc.stickerMu.Unlock() + if catalog, ok := lc.stickerCatalogs[shop]; ok && time.Since(catalog.Fetched) < stickerCatalogTTL { + return catalog.Products, nil + } + _, profile, err := callLineResult(lc, ctx, func(client *line.Client) (*line.Profile, error) { + return client.GetProfileContext(ctx) + }) + if err != nil { + return nil, err + } + if profile.RegionCode == "" { + return nil, fmt.Errorf("LINE profile has no shop region") + } + _, products, err := callLineResult(lc, ctx, func(client *line.Client) ([]line.ShopProduct, error) { + return client.GetOwnedShopProducts(ctx, shop, profile.RegionCode) + }) + if err != nil { + return nil, err + } + if lc.stickerCatalogs == nil { + lc.stickerCatalogs = make(map[string]stickerCatalog) + } + lc.stickerCatalogs[shop] = stickerCatalog{Products: products, Fetched: time.Now()} + return products, nil +} + +func (lc *LineClient) resetStickerCatalogs() { + lc.stickerMu.Lock() + lc.stickerCatalogs = nil + lc.stickerMu.Unlock() +} + +func packMetadata(shop string, product line.ShopProduct) *event.ImagePackMetadata { + return &event.ImagePackMetadata{ + DisplayName: product.Name, + Usage: []event.ImagePackUsage{event.ImagePackUsageSticker}, + BridgedPack: &event.BridgedStickerPack{ + Network: "line", + URL: "line://" + shop + "/" + product.ID, + }, + } +} + +func supportedProduct(shop string, product line.ShopProduct) bool { + if !lineAssetID.MatchString(product.ID) || !activeProduct(product, time.Now()) { + return false + } + summary := product.ProductTypeSummary.Sticker + if shop != line.StickerShop || summary == nil { + return false + } + _, err := stickerOption(summary.ResourceType) + return err == nil +} + +func (lc *LineClient) ListImagePacks(ctx context.Context) ([]*event.ImagePackMetadata, error) { + var packs []*event.ImagePackMetadata + for _, shop := range stickerShops { + products, err := lc.ownedStickerProducts(ctx, shop) + if err != nil { + return nil, err + } + for _, product := range products { + if supportedProduct(shop, product) { + packs = append(packs, packMetadata(shop, product)) + } + } + } + return packs, nil +} + +func (lc *LineClient) ownedStickerProduct(ctx context.Context, shop, productID string) (*line.ShopProduct, error) { + products, err := lc.ownedStickerProducts(ctx, shop) + if err != nil { + return nil, err + } + for _, product := range products { + if product.ID == productID && supportedProduct(shop, product) { + return &product, nil + } + } + return nil, fmt.Errorf("LINE pack %s is not owned, has expired, or requires unsupported personalization", productID) +} + +func (lc *LineClient) fetchStickerAsset(ctx context.Context, path string, limit int64) ([]byte, error) { + req, err := http.NewRequestWithContext(ctx, http.MethodGet, stickerCDN+path, nil) + if err != nil { + return nil, err + } + client := lc.HTTPClient + if client == nil { + client = &http.Client{Timeout: 30 * time.Second} + } + noRedirectClient := *client + noRedirectClient.CheckRedirect = func(*http.Request, []*http.Request) error { + return http.ErrUseLastResponse + } + resp, err := noRedirectClient.Do(req) + if err != nil { + return nil, err + } + defer resp.Body.Close() + if resp.StatusCode != http.StatusOK { + return nil, fmt.Errorf("LINE sticker CDN returned %d", resp.StatusCode) + } + data, err := io.ReadAll(io.LimitReader(resp.Body, limit+1)) + if err != nil { + return nil, err + } + if int64(len(data)) > limit { + return nil, fmt.Errorf("LINE sticker asset too large") + } + return data, nil +} + +func (lc *LineClient) productStickers(_ context.Context, shop string, product line.ShopProduct) ([]lineSticker, error) { + version := product.LatestVersion.String() + if n, err := strconv.ParseUint(version, 10, 32); err != nil || n == 0 { + return nil, fmt.Errorf("invalid LINE pack version") + } + summary := product.ProductTypeSummary.Sticker + if summary == nil { + return nil, fmt.Errorf("missing sticker summary") + } + option, err := stickerOption(summary.ResourceType) + if err != nil { + return nil, err + } + if summary.Hash != "" && !lineAssetID.MatchString(summary.Hash) { + return nil, fmt.Errorf("invalid sticker hash") + } + base := lineSticker{ + Shop: shop, + ProductID: product.ID, + Version: version, + Option: option, + Hash: summary.Hash, + Body: product.Name, + } + var stickers []lineSticker + for _, idRange := range summary.Ranges { + start, err := strconv.ParseUint(idRange.Start.String(), 10, 63) + if err != nil || idRange.Size < 0 || len(stickers)+idRange.Size > maxStickersPerPack || + start > uint64(1<<63-1)-uint64(idRange.Size) { + return nil, fmt.Errorf("invalid sticker range") + } + for i := range idRange.Size { + sticker := base + sticker.ID = strconv.FormatUint(start+uint64(i), 10) + stickers = append(stickers, sticker) + } + } + if len(stickers) == 0 { + return nil, fmt.Errorf("LINE pack is empty") + } + return stickers, nil +} + +func stickerKey(kind, value string) database.Key { + return database.Key(fmt.Sprintf("line.sticker.%s.%x", kind, sha256.Sum256([]byte(value)))) +} + +func (lc *LineClient) saveStickerKV(ctx context.Context, key database.Key, value string) error { + kv := lc.UserLogin.Bridge.DB.KV + kv.Set(ctx, key, value) + if kv.Get(ctx, key) != value { + return fmt.Errorf("failed to persist LINE sticker mapping") + } + return nil +} + +func (lc *LineClient) importSticker(ctx context.Context, sticker lineSticker) (*event.ImagePackImage, error) { + kv := lc.UserLogin.Bridge.DB.KV + raw, err := json.Marshal(sticker) + if err != nil { + return nil, err + } + assetKey := stickerKey("asset", string(raw)) + var img event.ImagePackImage + if cached := kv.Get(ctx, assetKey); cached != "" && json.Unmarshal([]byte(cached), &img) == nil && img.URL != "" { + return &img, nil + } + data, err := lc.fetchStickerAsset(ctx, sticker.assetPath(), maxStickerAssetSize) + if err != nil { + return nil, err + } + config, format, err := image.DecodeConfig(bytes.NewReader(data)) + if err != nil || format != "png" { + return nil, fmt.Errorf("invalid LINE sticker PNG") + } + mxc, file, err := lc.UserLogin.Bridge.Bot.UploadMedia(ctx, "", data, "sticker.png", "image/png") + if err != nil { + return nil, err + } + if file != nil { + return nil, fmt.Errorf("image packs require unencrypted media") + } + img = event.ImagePackImage{ + URL: mxc, + Body: sticker.Body, + Info: &event.FileInfo{ + MimeType: "image/png", + IsAnimated: strings.ContainsAny(sticker.Option, "AP"), + Size: len(data), + Width: config.Width, + Height: config.Height, + BridgedSticker: &event.BridgedSticker{ + Network: "line", + ID: sticker.ID, + PackURL: sticker.packURL(), + }, + }, + } + if err = lc.saveStickerKV(ctx, stickerKey("mxc", string(mxc)), string(raw)); err != nil { + return nil, err + } + imgJSON, err := json.Marshal(img) + if err != nil { + return nil, err + } + if err = lc.saveStickerKV(ctx, assetKey, string(imgJSON)); err != nil { + return nil, err + } + return &img, nil +} + +func (lc *LineClient) DownloadImagePack(ctx context.Context, rawURL string) (*bridgev2.ImportedImagePack, error) { + shop, productID, err := parseLinePackURL(rawURL) + if err != nil { + return nil, err + } + product, err := lc.ownedStickerProduct(ctx, shop, productID) + if err != nil { + return nil, err + } + stickers, err := lc.productStickers(ctx, shop, *product) + if err != nil { + return nil, err + } + pack := &bridgev2.ImportedImagePack{ + Shortcode: "line_" + shop + "_" + productID, + Content: &event.ImagePackEventContent{ + Metadata: *packMetadata(shop, *product), + Images: make(map[string]*event.ImagePackImage, len(stickers)), + }, + } + for _, sticker := range stickers { + img, err := lc.importSticker(ctx, sticker) + if err != nil { + return nil, err + } + pack.Content.Images["line_"+sticker.ID] = img + } + return pack, nil +} + +func (lc *LineClient) resolveSticker(ctx context.Context, mxc id.ContentURIString) (*lineSticker, error) { + if mxc == "" || lc.UserLogin == nil || lc.UserLogin.Bridge == nil || lc.UserLogin.Bridge.DB == nil { + return nil, nil + } + raw := lc.UserLogin.Bridge.DB.KV.Get(ctx, stickerKey("mxc", string(mxc))) + if raw == "" { + return nil, nil + } + var stored lineSticker + if err := json.Unmarshal([]byte(raw), &stored); err != nil { + return nil, err + } + product, err := lc.ownedStickerProduct(ctx, stored.Shop, stored.ProductID) + if err != nil { + return nil, err + } + stickers, err := lc.productStickers(ctx, stored.Shop, *product) + if err != nil { + return nil, err + } + for _, sticker := range stickers { + if sticker.ID == stored.ID { + return &sticker, nil + } + } + return nil, fmt.Errorf("sticker no longer belongs to the LINE pack") +} + +type publishedStickerPack struct { + Digest string `json:"digest"` + StateKeys []string `json:"state_keys"` +} + +func (lc *LineClient) syncStickerRoom(ctx context.Context, room id.RoomID, packs []*event.ImagePackMetadata) error { + manifestKey := stickerKey("published", string(lc.UserLogin.ID)+"/"+string(room)) + manifest := map[string]publishedStickerPack{} + if raw := lc.UserLogin.Bridge.DB.KV.Get(ctx, manifestKey); raw != "" { + if err := json.Unmarshal([]byte(raw), &manifest); err != nil { + return err + } + } + saveManifest := func() error { + raw, err := json.Marshal(manifest) + if err != nil { + return err + } + return lc.saveStickerKV(ctx, manifestKey, string(raw)) + } + active := make(map[string]bool, len(packs)) + var firstErr error + for _, meta := range packs { + packURL := meta.BridgedPack.URL + active[packURL] = true + pack, err := lc.DownloadImagePack(ctx, packURL) + if err != nil { + if firstErr == nil { + firstErr = err + } + continue + } + content, err := json.Marshal(pack.Content) + if err != nil { + return err + } + digest := fmt.Sprintf("%x", sha256.Sum256(content)) + previous := manifest[packURL] + if previous.Digest == digest { + continue + } + parts, err := provisionutil.SplitPack(pack) + if err != nil { + return err + } + prefix := fmt.Sprintf("line_%x_%s", sha256.Sum256([]byte(lc.UserLogin.ID)), pack.Shortcode) + next := publishedStickerPack{Digest: digest} + for i, part := range parts { + stateKey := fmt.Sprintf("%s_%d", prefix, i) + for _, stateType := range []event.Type{event.StateImagePack, event.StateUnstableImagePack} { + if _, err = lc.UserLogin.Bridge.Bot.SendState(ctx, room, stateType, stateKey, &event.Content{VeryRaw: part}, time.Now()); err != nil { + return err + } + } + next.StateKeys = append(next.StateKeys, stateKey) + if !slices.Contains(previous.StateKeys, stateKey) { + previous.StateKeys = append(previous.StateKeys, stateKey) + } + previous.Digest = "" + manifest[packURL] = previous + if err = saveManifest(); err != nil { + return err + } + } + for _, stateKey := range previous.StateKeys { + if slices.Contains(next.StateKeys, stateKey) { + continue + } + if err = lc.clearStickerPack(ctx, room, stateKey); err != nil { + return err + } + } + manifest[packURL] = next + if err = saveManifest(); err != nil { + return err + } + } + for packURL, previous := range manifest { + if active[packURL] { + continue + } + for _, stateKey := range previous.StateKeys { + if err := lc.clearStickerPack(ctx, room, stateKey); err != nil { + return err + } + } + delete(manifest, packURL) + if err := saveManifest(); err != nil { + return err + } + } + return firstErr +} + +func (lc *LineClient) clearStickerPack(ctx context.Context, room id.RoomID, stateKey string) error { + empty := &event.ImagePackEventContent{Images: map[string]*event.ImagePackImage{}} + for _, stateType := range []event.Type{event.StateImagePack, event.StateUnstableImagePack} { + if _, err := lc.UserLogin.Bridge.Bot.SendState(ctx, room, stateType, stateKey, &event.Content{Parsed: empty}, time.Now()); err != nil { + return err + } + } + return nil +} + +func (lc *LineClient) stickerRooms(ctx context.Context) ([]id.RoomID, error) { + var rooms []id.RoomID + if raw := lc.UserLogin.Bridge.DB.KV.Get(ctx, stickerKey("rooms", string(lc.UserLogin.ID))); raw != "" { + if err := json.Unmarshal([]byte(raw), &rooms); err != nil { + return nil, err + } + } + return rooms, nil +} + +func (lc *LineClient) enableRoomStickers(ctx context.Context, room id.RoomID) error { + lc.stickerSyncMu.Lock() + defer lc.stickerSyncMu.Unlock() + packs, err := lc.ListImagePacks(ctx) + if err != nil { + return err + } + rooms, err := lc.stickerRooms(ctx) + if err != nil { + return err + } + if !slices.Contains(rooms, room) { + rooms = append(rooms, room) + raw, err := json.Marshal(rooms) + if err != nil { + return err + } + if err = lc.saveStickerKV(ctx, stickerKey("rooms", string(lc.UserLogin.ID)), string(raw)); err != nil { + return err + } + } + return lc.syncStickerRoom(ctx, room, packs) +} + +func (lc *LineClient) syncStickerPacksOnce(ctx context.Context) error { + lc.stickerSyncMu.Lock() + defer lc.stickerSyncMu.Unlock() + rooms, err := lc.stickerRooms(ctx) + if err != nil || len(rooms) == 0 { + return err + } + packs, err := lc.ListImagePacks(ctx) + if err != nil { + return err + } + var firstErr error + for _, room := range rooms { + if err = lc.syncStickerRoom(ctx, room, packs); err != nil && firstErr == nil { + firstErr = err + } + } + return firstErr +} + +func (lc *LineClient) syncStickerPacks(ctx context.Context) { + defer lc.wg.Done() + ticker := time.NewTicker(stickerSyncInterval) + defer ticker.Stop() + for { + if err := lc.syncStickerPacksOnce(ctx); err != nil && ctx.Err() == nil { + lc.UserLogin.Log.Warn().Err(err).Msg("Failed to synchronize LINE sticker packs") + } + select { + case <-ctx.Done(): + return + case <-ticker.C: + } + } +} + +func (lc *LineClient) withStickerPacks(info *bridgev2.ChatInfo) *bridgev2.ChatInfo { + if info != nil { + info.ExtraUpdates = bridgev2.MergeExtraUpdaters(info.ExtraUpdates, lc.enableStickersOnRoomCreate) + } + return info +} + +func (lc *LineClient) enableStickersOnRoomCreate(ctx context.Context, portal *bridgev2.Portal) bool { + if portal.MXID != "" || portal.RoomType == database.RoomTypeSpace { + return false + } + if _, pending := lc.pendingStickerRooms.LoadOrStore(portal.PortalKey, struct{}{}); pending { + return false + } + runCtx, ok := lc.startRunTask() + if !ok { + lc.pendingStickerRooms.Delete(portal.PortalKey) + return false + } + go func() { + defer lc.wg.Done() + defer lc.pendingStickerRooms.Delete(portal.PortalKey) + ctx := zerolog.Ctx(ctx).WithContext(runCtx) + if err := portal.RoomCreated.WaitTimeoutCtx(ctx, stickerRoomCreateTimeout); err != nil { + return + } + if err := lc.enableRoomStickers(ctx, portal.MXID); err != nil { + lc.UserLogin.Log.Warn().Err(err).Stringer("room_id", portal.MXID).Msg("Failed to add LINE sticker packs to new room") + } + }() + return false +} diff --git a/pkg/connector/stickers_test.go b/pkg/connector/stickers_test.go new file mode 100644 index 00000000..f4238bcb --- /dev/null +++ b/pkg/connector/stickers_test.go @@ -0,0 +1,324 @@ +package connector + +import ( + "bytes" + "context" + "database/sql" + "encoding/json" + "fmt" + "image" + "image/png" + "io" + "net/http" + "strings" + "testing" + "time" + + _ "github.com/mattn/go-sqlite3" + "github.com/rs/zerolog" + "go.mau.fi/util/dbutil" + "go.mau.fi/util/exsync" + "maunium.net/go/mautrix" + "maunium.net/go/mautrix/bridgev2" + "maunium.net/go/mautrix/bridgev2/database" + "maunium.net/go/mautrix/bridgev2/networkid" + "maunium.net/go/mautrix/event" + "maunium.net/go/mautrix/id" + + "github.com/highesttt/matrix-line-messenger/pkg/e2ee" + "github.com/highesttt/matrix-line-messenger/pkg/line" +) + +type stickerTestTransport func(*http.Request) (*http.Response, error) + +func (f stickerTestTransport) RoundTrip(r *http.Request) (*http.Response, error) { return f(r) } + +func stickerResponse(data string) *http.Response { + return &http.Response{StatusCode: 200, Header: make(http.Header), Body: io.NopCloser(strings.NewReader(data))} +} + +type stickerTestMatrix struct { + bridgev2.MatrixAPI + uploads int + states map[string]event.ImagePackEventContent + stateWrites int +} + +func (m *stickerTestMatrix) UploadMedia(_ context.Context, room id.RoomID, data []byte, name, mime string) (id.ContentURIString, *event.EncryptedFileInfo, error) { + if room != "" || mime != "image/png" { + return "", nil, fmt.Errorf("bad upload") + } + m.uploads++ + return id.ContentURIString(fmt.Sprintf("mxc://test/sticker%d", m.uploads)), nil, nil +} + +func stickerTestProduct(t *testing.T) line.ShopProduct { + t.Helper() + var p line.ShopProduct + if err := json.Unmarshal([]byte(`{"id":"123","name":"Cats","latestVersion":"1","validUntil":"-1","productTypeSummary":{"stickerSummary":{"stickerResourceType":2,"stickerIdRanges":[{"start":"100","size":2}]}}}`), &p); err != nil { + t.Fatal(err) + } + return p +} + +func testMatrixMessage(portalID networkid.PortalID, content *event.MessageEventContent) *bridgev2.MatrixMessage { + return &bridgev2.MatrixMessage{ + MatrixEventBase: bridgev2.MatrixEventBase[*event.MessageEventContent]{ + Portal: &bridgev2.Portal{Portal: &database.Portal{PortalKey: networkid.PortalKey{ID: portalID}}}, + Content: content, + }, + } +} + +func stickerTestClient(t *testing.T) (*LineClient, *stickerTestMatrix) { + t.Helper() + raw, err := sql.Open("sqlite3", ":memory:") + if err != nil { + t.Fatal(err) + } + raw.SetMaxOpenConns(1) + t.Cleanup(func() { raw.Close() }) + db, err := dbutil.NewWithDB(raw, "sqlite3") + if err != nil { + t.Fatal(err) + } + if _, err = raw.Exec(`CREATE TABLE kv_store (bridge_id TEXT,key TEXT,value TEXT,PRIMARY KEY(bridge_id,key))`); err != nil { + t.Fatal(err) + } + bdb := database.New("line", database.MetaTypes{}, db) + matrix := &stickerTestMatrix{} + lc := &LineClient{ + Mid: "ume", + AccessToken: "test", + UserLogin: &bridgev2.UserLogin{ + UserLogin: &database.UserLogin{ID: "ume"}, + Bridge: &bridgev2.Bridge{DB: bdb, Bot: matrix, Log: zerolog.Nop()}, + }, + stickerCatalogs: map[string]stickerCatalog{ + line.StickerShop: {Products: []line.ShopProduct{stickerTestProduct(t)}, Fetched: time.Now()}, + }, + } + var pngData bytes.Buffer + png.Encode(&pngData, image.NewRGBA(image.Rect(0, 0, 2, 2))) + lc.HTTPClient = &http.Client{Transport: stickerTestTransport(func(r *http.Request) (*http.Response, error) { + if r.URL.Host != "stickershop.line-scdn.net" { + t.Fatal(r.URL) + } + return stickerResponse(pngData.String()), nil + })} + return lc, matrix +} + +func TestStickerImportPersistenceAndNativeSend(t *testing.T) { + lc, matrix := stickerTestClient(t) + pack, err := lc.DownloadImagePack(t.Context(), "https://store.line.me/stickershop/product/123/en") + if err != nil { + t.Fatal(err) + } + if len(pack.Content.Images) != 2 || matrix.uploads != 2 { + t.Fatal(pack) + } + img := pack.Content.Images["line_100"] + if img.Info.Width != 2 || img.Info.BridgedSticker.ID != "100" { + t.Fatal(img) + } + + if _, err = lc.DownloadImagePack(t.Context(), "line://stickershop/123"); err != nil { + t.Fatal(err) + } + if matrix.uploads != 2 { + t.Fatal("reuploaded cached pack") + } + fresh := &LineClient{Mid: lc.Mid, AccessToken: lc.AccessToken, UserLogin: lc.UserLogin, stickerCatalogs: lc.stickerCatalogs, E2EE: &e2ee.Manager{}} + previous := newLineAPIClient + t.Cleanup(func() { newLineAPIClient = previous }) + sent := 0 + newLineAPIClient = func(token string) *line.Client { + c := line.NewClient(token) + c.HTTPClient = &http.Client{Transport: stickerTestTransport(func(r *http.Request) (*http.Response, error) { + if !strings.HasSuffix(r.URL.Path, "/sendMessage") { + t.Fatalf("native sticker tried E2EE or upload: %s", r.URL) + } + var args []json.RawMessage + json.NewDecoder(r.Body).Decode(&args) + var msg line.Message + json.Unmarshal(args[1], &msg) + if msg.ContentType != 7 || !msg.HasContent || msg.ContentMetadata["STKPKGID"] != "123" || msg.ContentMetadata["STKID"] != "100" || msg.ContentMetadata["STKOPT"] != "A" || len(msg.Chunks) > 0 || msg.ContentMetadata["e2eeVersion"] != "" { + t.Fatalf("bad native message: %+v", msg) + } + sent++ + return stickerResponse(`{"code":0,"data":{"id":"remote-sticker"}}`), nil + })} + return c + } + for _, kind := range []event.MessageType{event.CapMsgSticker, event.MsgImage} { + content := &event.MessageEventContent{MsgType: kind, Body: "sticker", URL: img.URL} + response, err := fresh.HandleMatrixMessage(t.Context(), testMatrixMessage("ufriend", content)) + if err != nil { + t.Fatal(err) + } + if response.DB.ID != "remote-sticker" { + t.Fatal(response) + } + } + if sent != 2 { + t.Fatal(sent) + } +} + +func TestStickerOwnershipAndURLValidation(t *testing.T) { + lc, _ := stickerTestClient(t) + pack, err := lc.DownloadImagePack(t.Context(), "line://stickershop/123") + if err != nil { + t.Fatal(err) + } + lc.stickerCatalogs[line.StickerShop] = stickerCatalog{Fetched: time.Now()} + if _, err = lc.resolveSticker(t.Context(), pack.Content.Images["line_100"].URL); err == nil { + t.Fatal("allowed unowned sticker") + } + if s, err := lc.resolveSticker(t.Context(), "mxc://test/unknown"); s != nil || err != nil { + t.Fatal("unknown image should remain image") + } + for _, u := range []string{"https://evil.test/stickershop/product/123/en", "line://stickershop/../123", "https://store.line.me@evil.test/stickershop/product/123", "line://stickershop/123?x=1"} { + if _, _, err := parseLinePackURL(u); err == nil { + t.Fatalf("accepted %s", u) + } + } + p := stickerTestProduct(t) + p.ValidUntil = json.Number(fmt.Sprint(time.Now().Add(-time.Second).UnixMilli())) + if activeProduct(p, time.Now()) { + t.Fatal("accepted expired pack") + } +} + +func (m *stickerTestMatrix) SendState(_ context.Context, room id.RoomID, typ event.Type, key string, content *event.Content, _ time.Time) (*mautrix.RespSendEvent, error) { + data, err := json.Marshal(content) + if err != nil { + return nil, err + } + var pack event.ImagePackEventContent + if err = json.Unmarshal(data, &pack); err != nil { + return nil, err + } + if m.states == nil { + m.states = make(map[string]event.ImagePackEventContent) + } + m.states[string(room)+"/"+typ.Type+"/"+key] = pack + m.stateWrites++ + return &mautrix.RespSendEvent{EventID: "$pack"}, nil +} + +func TestStickerRoomSyncUpdatesAndRemovesOnlyManagedPacks(t *testing.T) { + lc, matrix := stickerTestClient(t) + room := id.RoomID("!room:test") + packs, err := lc.ListImagePacks(t.Context()) + if err != nil { + t.Fatal(err) + } + if err = lc.syncStickerRoom(t.Context(), room, packs); err != nil { + t.Fatal(err) + } + if matrix.stateWrites != 2 { + t.Fatalf("state writes %d", matrix.stateWrites) + } + if err = lc.syncStickerRoom(t.Context(), room, packs); err != nil { + t.Fatal(err) + } + if matrix.stateWrites != 2 { + t.Fatal("unchanged pack republished") + } + matrix.states["unrelated"] = event.ImagePackEventContent{Images: map[string]*event.ImagePackImage{"keep": {Body: "keep"}}} + if err = lc.syncStickerRoom(t.Context(), room, nil); err != nil { + t.Fatal(err) + } + if matrix.stateWrites != 4 || len(matrix.states["unrelated"].Images) != 1 { + t.Fatal("removed unrelated state") + } + for key, pack := range matrix.states { + if key != "unrelated" && len(pack.Images) != 0 { + t.Fatal("expired pack remained") + } + } +} + +func TestStickerDownloadRejectsOversizeAndRedirect(t *testing.T) { + lc, _ := stickerTestClient(t) + lc.HTTPClient.Transport = stickerTestTransport(func(*http.Request) (*http.Response, error) { return stickerResponse("12345"), nil }) + if _, err := lc.fetchStickerAsset(t.Context(), "/test", 4); err == nil { + t.Fatal("accepted oversized asset") + } + lc.HTTPClient.Transport = stickerTestTransport(func(*http.Request) (*http.Response, error) { + r := stickerResponse("") + r.StatusCode = 302 + r.Header.Set("Location", "http://localhost/private") + return r, nil + }) + if _, err := lc.fetchStickerAsset(t.Context(), "/test", 4); err == nil { + t.Fatal("followed redirect") + } +} + +func TestStickerPacksAppliedToNewRooms(t *testing.T) { + lc, matrix := stickerTestClient(t) + portal := &bridgev2.Portal{ + Portal: &database.Portal{PortalKey: networkid.PortalKey{ID: "ufriend"}}, + RoomCreated: exsync.NewEvent(), + } + runCtx, cancel := context.WithCancel(t.Context()) + defer cancel() + lc.activeRun = &lineClientRun{ctx: runCtx, cancel: cancel} + info := lc.withStickerPacks(&bridgev2.ChatInfo{}) + info.ExtraUpdates(t.Context(), portal) + info.ExtraUpdates(t.Context(), portal) + portal.MXID = "!new:test" + portal.RoomCreated.Set() + lc.wg.Wait() + rooms, err := lc.stickerRooms(t.Context()) + if err != nil { + t.Fatal(err) + } + if len(rooms) != 1 || rooms[0] != portal.MXID || matrix.stateWrites != 2 { + t.Fatalf("rooms=%v state writes=%d", rooms, matrix.stateWrites) + } + for key, pack := range matrix.states { + if !strings.HasPrefix(key, "!new:test/") || len(pack.Images) != 2 { + t.Fatalf("unexpected pack state %s: %+v", key, pack) + } + } + + existing := &bridgev2.Portal{ + Portal: &database.Portal{PortalKey: networkid.PortalKey{ID: "uother"}, MXID: "!existing:test"}, + RoomCreated: exsync.NewEvent(), + } + info.ExtraUpdates(t.Context(), existing) + if _, ok := lc.pendingStickerRooms.Load(existing.PortalKey); ok { + t.Fatal("existing room was scheduled for packs") + } +} + +func TestStickerPacksNotAppliedAfterDisconnect(t *testing.T) { + lc, matrix := stickerTestClient(t) + runCtx, cancel := context.WithCancel(t.Context()) + lc.activeRun = &lineClientRun{ctx: runCtx, cancel: cancel} + pending := &bridgev2.Portal{ + Portal: &database.Portal{PortalKey: networkid.PortalKey{ID: "upending"}}, + RoomCreated: exsync.NewEvent(), + } + info := lc.withStickerPacks(&bridgev2.ChatInfo{}) + info.ExtraUpdates(t.Context(), pending) + lc.Disconnect() + if _, ok := lc.pendingStickerRooms.Load(pending.PortalKey); ok { + t.Fatal("pending room task outlived the client") + } + late := &bridgev2.Portal{ + Portal: &database.Portal{PortalKey: networkid.PortalKey{ID: "ulate"}}, + RoomCreated: exsync.NewEvent(), + } + info.ExtraUpdates(t.Context(), late) + if _, ok := lc.pendingStickerRooms.Load(late.PortalKey); ok { + t.Fatal("room task started after disconnect") + } + if matrix.stateWrites != 0 { + t.Fatalf("state writes after disconnect: %d", matrix.stateWrites) + } +} diff --git a/pkg/connector/sync.go b/pkg/connector/sync.go index 4aa1cab5..efc73317 100644 --- a/pkg/connector/sync.go +++ b/pkg/connector/sync.go @@ -283,7 +283,7 @@ func (lc *LineClient) queueDMChatResync(ctx context.Context, mid string, createP Timestamp: time.Now(), CreatePortal: createPortal, }, - ChatInfo: &bridgev2.ChatInfo{ + ChatInfo: lc.withStickerPacks(&bridgev2.ChatInfo{ Type: &dmType, Name: &chatName, Avatar: lc.avatarFromPicturePath(contact.PicturePath), @@ -310,7 +310,7 @@ func (lc *LineClient) queueDMChatResync(ctx context.Context, mid string, createP }), }, ExcludeChangesFromTimeline: true, - }, + }), CheckNeedsBackfillFunc: checkNeedsBackfill, }) } @@ -1074,7 +1074,7 @@ func (lc *LineClient) chatToChatInfo(ctx context.Context, chat *line.Chat, exclu if selfInvitePending { info.MessageRequest = ptr.Ptr(true) } - return info + return lc.withStickerPacks(info) } func (lc *LineClient) generateNameFromMemberList(ctx context.Context, members []string) string { diff --git a/pkg/connector/userinfo.go b/pkg/connector/userinfo.go index c80a5262..4e1be151 100644 --- a/pkg/connector/userinfo.go +++ b/pkg/connector/userinfo.go @@ -183,7 +183,7 @@ func (lc *LineClient) GetChatInfo(ctx context.Context, portal *bridgev2.Portal) contact := lc.getContact(ctx, string(portal.ID)) dmType := database.RoomTypeDM chatName := contact.EffectiveDisplayName() - return &bridgev2.ChatInfo{ + return lc.withStickerPacks(&bridgev2.ChatInfo{ Type: &dmType, Name: &chatName, Avatar: lc.avatarFromPicturePath(contact.PicturePath), @@ -207,7 +207,7 @@ func (lc *LineClient) GetChatInfo(ctx context.Context, portal *bridgev2.Portal) }, }), }, - }, nil + }), nil } func (lc *LineClient) GetUserInfo(ctx context.Context, ghost *bridgev2.Ghost) (*bridgev2.UserInfo, error) { diff --git a/pkg/line/stickers.go b/pkg/line/stickers.go new file mode 100644 index 00000000..c525a8fa --- /dev/null +++ b/pkg/line/stickers.go @@ -0,0 +1,77 @@ +package line + +import ( + "context" + "encoding/json" + "fmt" +) + +const ( + StickerShop = "stickershop" + SticonShop = "sticonshop" + + shopPageSize = 1000 + maxShopProducts = 100000 +) + +type ShopProduct struct { + ID string `json:"id"` + Name string `json:"name"` + LatestVersion json.Number `json:"latestVersion"` + ValidUntil json.Number `json:"validUntil"` + ProductTypeSummary struct { + Sticker *StickerSummary `json:"stickerSummary"` + Sticon *SticonSummary `json:"sticonSummary"` + } `json:"productTypeSummary"` +} + +type StickerSummary struct { + ResourceType int `json:"stickerResourceType"` + Hash string `json:"stickerHash"` + Ranges []struct { + Start json.Number `json:"start"` + Size int `json:"size"` + } `json:"stickerIdRanges"` +} + +type SticonSummary struct { + ResourceType int `json:"sticonResourceType"` +} + +func (c *Client) GetOwnedShopProducts(ctx context.Context, shop, country string) ([]ShopProduct, error) { + if shop != StickerShop && shop != SticonShop { + return nil, fmt.Errorf("unknown LINE shop") + } + var products []ShopProduct + for offset := 0; offset < maxShopProducts; { + resp, err := c.callRPCWithBaseURLContext(ctx, ShopBaseURL, "ShopService", "getOwnedProductSummaries", + shop, offset, shopPageSize, map[string]string{"language": "en", "country": country}) + if err != nil { + return nil, err + } + var result struct { + Code int `json:"code"` + Message string `json:"message"` + Data struct { + Products []ShopProduct `json:"productList"` + Offset int `json:"offset"` + Total int `json:"totalSize"` + } `json:"data"` + } + if err = json.Unmarshal(resp, &result); err != nil { + return nil, err + } + if result.Code != 0 { + return nil, fmt.Errorf("getOwnedProductSummaries: %s", result.Message) + } + products = append(products, result.Data.Products...) + if result.Data.Offset+len(result.Data.Products) >= result.Data.Total { + return products, nil + } + if len(result.Data.Products) == 0 || result.Data.Offset != offset { + return nil, fmt.Errorf("invalid LINE shop pagination") + } + offset += shopPageSize + } + return nil, fmt.Errorf("LINE shop catalog exceeded pagination limit") +} diff --git a/pkg/line/stickers_test.go b/pkg/line/stickers_test.go new file mode 100644 index 00000000..3ae94389 --- /dev/null +++ b/pkg/line/stickers_test.go @@ -0,0 +1,57 @@ +package line + +import ( + "context" + "encoding/json" + "fmt" + "net/http" + "testing" +) + +func TestOwnedShopProductsChromePagination(t *testing.T) { + c := NewClient("token") + calls := 0 + c.HTTPClient = &http.Client{Transport: roundTripFunc(func(r *http.Request) (*http.Response, error) { + var args []json.RawMessage + if err := json.NewDecoder(r.Body).Decode(&args); err != nil { + t.Fatal(err) + } + if len(args) != 4 || string(args[0]) != `"stickershop"` || string(args[1]) != fmt.Sprint(calls*1000) || string(args[2]) != "1000" || r.URL.Path != "/api/shop/thrift/ShopService/ShopService/getOwnedProductSummaries" { + t.Fatalf("wrong Chrome request %s %s", r.URL.Path, args) + } + var locale map[string]string + json.Unmarshal(args[3], &locale) + if locale["country"] != "JP" || locale["language"] != "en" { + t.Fatal(locale) + } + offset := calls * 1000 + calls++ + return obsResponse(200, fmt.Sprintf(`{"code":0,"data":{"offset":%d,"totalSize":1001,"productList":[{"id":"%d","validUntil":"-1","latestVersion":1,"productTypeSummary":{"stickerSummary":{"stickerResourceType":1,"stickerIdRanges":[{"start":"10","size":2}]}}}]}}`, offset, calls)), nil + })} + products, err := c.GetOwnedShopProducts(t.Context(), StickerShop, "JP") + if err != nil { + t.Fatal(err) + } + if len(products) != 2 || calls != 2 || products[0].ValidUntil.String() != "-1" { + t.Fatalf("%+v %d", products, calls) + } +} + +func TestOwnedShopProductsRejectsStalledPage(t *testing.T) { + c := NewClient("") + c.HTTPClient = &http.Client{Transport: roundTripFunc(func(*http.Request) (*http.Response, error) { + return obsResponse(200, `{"code":0,"data":{"offset":0,"totalSize":10,"productList":[]}}`), nil + })} + if _, err := c.GetOwnedShopProducts(t.Context(), StickerShop, "JP"); err == nil { + t.Fatal("accepted stalled page") + } +} + +func TestOwnedShopProductsCancellation(t *testing.T) { + ctx, cancel := context.WithCancel(t.Context()) + cancel() + c := NewClient("") + if _, err := c.GetOwnedShopProducts(ctx, StickerShop, "JP"); err == nil { + t.Fatal("ignored cancellation") + } +} From 8e3a412481e14ca73203a66db42914c5319c4237 Mon Sep 17 00:00:00 2001 From: formeo14 Date: Sun, 27 Sep 2026 13:20:33 +0900 Subject: [PATCH 3/4] feat: bridge LINE emoji packs as custom emotes --- pkg/connector/client.go | 2 + pkg/connector/reaction.go | 28 ++++ pkg/connector/reaction_test.go | 4 +- pkg/connector/send_message.go | 53 +++++--- pkg/connector/sticker_commands.go | 6 +- pkg/connector/sticker_reactions_test.go | 84 ++++++++++++ pkg/connector/stickers.go | 165 ++++++++++++++++++++++-- pkg/connector/stickers_test.go | 126 ++++++++++++++++++ pkg/connector/sticon_send.go | 143 ++++++++++++++++++++ pkg/connector/sticon_send_test.go | 88 +++++++++++++ pkg/connector/userinfo.go | 4 +- 11 files changed, 670 insertions(+), 33 deletions(-) create mode 100644 pkg/connector/sticker_reactions_test.go create mode 100644 pkg/connector/sticon_send.go create mode 100644 pkg/connector/sticon_send_test.go diff --git a/pkg/connector/client.go b/pkg/connector/client.go index ea4687a0..7e6a280b 100644 --- a/pkg/connector/client.go +++ b/pkg/connector/client.go @@ -86,6 +86,8 @@ type LineClient struct { stickerMu sync.Mutex stickerCatalogs map[string]stickerCatalog pendingStickerRooms sync.Map + sticonMetaMu sync.Mutex + sticonMeta map[string]sticonMetaCache wg sync.WaitGroup } diff --git a/pkg/connector/reaction.go b/pkg/connector/reaction.go index 6e210a83..3fca4370 100644 --- a/pkg/connector/reaction.go +++ b/pkg/connector/reaction.go @@ -18,6 +18,7 @@ import ( "maunium.net/go/mautrix/bridgev2/networkid" "maunium.net/go/mautrix/bridgev2/simplevent" "maunium.net/go/mautrix/event" + "maunium.net/go/mautrix/id" "github.com/highesttt/matrix-line-messenger/pkg/line" ) @@ -571,6 +572,13 @@ func (lc *LineClient) resolveMatrixReaction(ctx context.Context, msg *bridgev2.M if !strings.HasPrefix(key, "mxc://") { return lineReactionRef{}, unsupportedMatrixReactionError(key) } + emote, err := lc.resolveSticker(ctx, id.ContentURIString(key)) + if err != nil { + lc.UserLogin.Log.Warn().Err(err).Str("key", key).Msg("Failed to resolve LINE emoji reaction") + } + if emote != nil { + return lineEmoteReaction(key, emote) + } if msg.TargetMessage == nil || msg.Portal == nil || msg.Portal.Bridge == nil || msg.Portal.Bridge.DB == nil { return lineReactionRef{}, errors.New("reaction target database context is missing") } @@ -591,6 +599,26 @@ func (lc *LineClient) resolveMatrixReaction(ctx context.Context, msg *bridgev2.M return ref, nil } +func lineEmoteReaction(key string, emote *lineSticker) (lineReactionRef, error) { + if emote.Shop != line.SticonShop { + return lineReactionRef{}, unsupportedMatrixReactionError(key) + } + version, err := strconv.ParseInt(emote.Version, 10, 32) + if err != nil || version <= 0 { + return lineReactionRef{}, fmt.Errorf("invalid LINE emoji reaction version") + } + resourceType := 1 + if emote.Option == "A" { + resourceType = 2 + } + return newLineReactionRef(linePaidReactionRef{ + ProductID: emote.ProductID, + EmojiID: emote.ID, + ResourceType: resourceType, + Version: int(version), + }.reactionType()) +} + func unsupportedMatrixReactionError(key string) error { return bridgev2.WrapErrorInStatus(fmt.Errorf("LINE does not support Matrix reaction %q", key)). WithStatus(event.MessageStatusFail). diff --git a/pkg/connector/reaction_test.go b/pkg/connector/reaction_test.go index 3d509971..2e09762b 100644 --- a/pkg/connector/reaction_test.go +++ b/pkg/connector/reaction_test.go @@ -31,8 +31,8 @@ func TestCapabilitiesAdvertiseSupportedReactions(t *testing.T) { if caps.ReactionCount != 1 { t.Fatalf("ReactionCount = %d, want 1", caps.ReactionCount) } - if caps.CustomEmojiReactions { - t.Fatal("CustomEmojiReactions must stay disabled because arbitrary Matrix custom emojis are unsupported") + if !caps.CustomEmojiReactions { + t.Fatal("CustomEmojiReactions must be enabled for imported LINE emoji") } if len(caps.AllowedReactions) != len(lineEmojiReactionURLs) { t.Fatalf("AllowedReactions has %d entries, want %d", len(caps.AllowedReactions), len(lineEmojiReactionURLs)) diff --git a/pkg/connector/send_message.go b/pkg/connector/send_message.go index 355063e2..06bdc9b0 100644 --- a/pkg/connector/send_message.go +++ b/pkg/connector/send_message.go @@ -7,8 +7,10 @@ import ( "encoding/json" "errors" "fmt" + "html" "image" "image/jpeg" + "maps" "net/http" "path/filepath" "regexp" @@ -20,6 +22,7 @@ import ( "maunium.net/go/mautrix/bridgev2/database" "maunium.net/go/mautrix/bridgev2/networkid" "maunium.net/go/mautrix/event" + "maunium.net/go/mautrix/id" "github.com/highesttt/matrix-line-messenger/pkg/e2ee" "github.com/highesttt/matrix-line-messenger/pkg/line" @@ -94,6 +97,11 @@ func (lc *LineClient) HandleMatrixMessage(ctx context.Context, msg *bridgev2.Mat WithErrorReason(event.MessageStatusGenericError) } + localMessage := *msg + localContent := *msg.Content + localMessage.Content = &localContent + msg = &localMessage + var nativeSticker *lineSticker if (msg.Content.MsgType == event.CapMsgSticker || msg.Content.MsgType == event.MsgImage) && msg.Content.File == nil { sticker, err := lc.resolveSticker(ctx, msg.Content.URL) @@ -102,6 +110,27 @@ func (lc *LineClient) HandleMatrixMessage(ctx context.Context, msg *bridgev2.Mat } if sticker != nil && sticker.Shop == line.StickerShop { nativeSticker = sticker + } else if sticker != nil && sticker.Shop == line.SticonShop { + msg.Content.MsgType = event.MsgText + msg.Content.Body = sticker.Body + msg.Content.Format = event.FormatHTML + msg.Content.FormattedBody = `` + html.EscapeString(sticker.Body) + `` + } + } + var sticonMetadata map[string]string + if msg.Content.MsgType == event.MsgText { + var err error + msg.Content.Body, sticonMetadata, err = convertOutgoingSticons(ctx, msg.Content, func(mxc id.ContentURIString) (*lineSticker, error) { + emote, err := lc.resolveSticker(ctx, mxc) + if err != nil { + lc.UserLogin.Log.Warn().Err(err).Str("mxc", string(mxc)).Msg("Failed to resolve LINE emoji, sending alt text") + return nil, nil + } + return emote, nil + }) + if err != nil { + return nil, err } } @@ -130,6 +159,7 @@ func (lc *LineClient) HandleMatrixMessage(ctx context.Context, msg *bridgev2.Mat var err error contentType := int(ContentText) contentMetadata := map[string]string{} + maps.Copy(contentMetadata, sticonMetadata) if !plainText { contentMetadata["e2eeVersion"] = "2" } @@ -203,7 +233,7 @@ func (lc *LineClient) HandleMatrixMessage(ctx context.Context, msg *bridgev2.Mat if plainText { plainTextBody = msg.Content.Body } else { - payload, err = json.Marshal(map[string]string{"text": msg.Content.Body}) + payload, err = lineTextPayload(msg.Content.Body, contentMetadata) if err != nil { return nil, fmt.Errorf("failed to marshal text payload: %w", err) } @@ -599,21 +629,13 @@ func (lc *LineClient) HandleMatrixMessage(ctx context.Context, msg *bridgev2.Mat return nil, errFetch } } - if contentType != int(ContentText) { - chunks, err = lc.E2EE.EncryptGroupMessageRaw(portalMid, fromMid, contentType, payload) - } else { - chunks, err = lc.E2EE.EncryptGroupMessage(portalMid, fromMid, msg.Content.Body) - } + chunks, err = lc.E2EE.EncryptGroupMessageRaw(portalMid, fromMid, contentType, payload) if err != nil { if errors.Is(err, ltsm.ErrAbort) { return nil, err } if errFetch := lc.fetchAndUnwrapGroupKey(ctx, portalMid, 0); errFetch == nil { - if contentType != int(ContentText) { - chunks, err = lc.E2EE.EncryptGroupMessageRaw(portalMid, fromMid, contentType, payload) - } else { - chunks, err = lc.E2EE.EncryptGroupMessage(portalMid, fromMid, msg.Content.Body) - } + chunks, err = lc.E2EE.EncryptGroupMessageRaw(portalMid, fromMid, contentType, payload) } else if errors.Is(errFetch, ltsm.ErrAbort) { return nil, errFetch } else if errFetch = lineGroupE2EEFetchFailureError(errFetch); errFetch != nil { @@ -668,6 +690,9 @@ func (lc *LineClient) HandleMatrixMessage(ctx context.Context, msg *bridgev2.Mat lc.UserLogin.Bridge.Log.Info().Str("portal", portalMid).Int("content_type", contentType).Msg("Sending plain text message (no E2EE)") } + if !plainText { + delete(contentMetadata, "REPLACE") + } now := time.Now().UnixMilli() lineMsg := &line.Message{ ID: fmt.Sprintf("local-%d", now), @@ -785,11 +810,7 @@ func (lc *LineClient) HandleMatrixMessage(ctx context.Context, msg *bridgev2.Mat Msg("autoRegisterGroupKey succeeded, retrying send") if !plainText && lc.E2EE != nil { if fetchErr := lc.fetchAndUnwrapGroupKey(ctx, portalMid, 0); fetchErr == nil { - if contentType != int(ContentText) { - chunks, err = lc.E2EE.EncryptGroupMessageRaw(portalMid, fromMid, contentType, payload) - } else { - chunks, err = lc.E2EE.EncryptGroupMessage(portalMid, fromMid, msg.Content.Body) - } + chunks, err = lc.E2EE.EncryptGroupMessageRaw(portalMid, fromMid, contentType, payload) if err == nil { lineMsg.Chunks = chunks lineMsg.Text = "" diff --git a/pkg/connector/sticker_commands.go b/pkg/connector/sticker_commands.go index 0a06086b..e3964d1b 100644 --- a/pkg/connector/sticker_commands.go +++ b/pkg/connector/sticker_commands.go @@ -6,7 +6,7 @@ var commandStickers = &commands.FullHandler{ Name: "stickers", Help: commands.HelpMeta{ Section: commands.HelpSectionMisc, - Description: "Add your LINE sticker packs to this room and refresh them", + Description: "Add your LINE sticker and emoji packs to this room and refresh them", }, RequiresPortal: true, RequiresLogin: true, @@ -23,9 +23,9 @@ var commandStickers = &commands.FullHandler{ } client.resetStickerCatalogs() if err = client.enableRoomStickers(ce.Ctx, ce.RoomID); err != nil { - ce.Reply("Failed to synchronize LINE sticker packs: %s", err) + ce.Reply("Failed to synchronize LINE sticker and emoji packs: %s", err) return } - ce.Reply("LINE sticker packs are now available in this room. Pack images are not end-to-end encrypted.") + ce.Reply("LINE sticker and emoji packs are now available in this room. Pack images are not end-to-end encrypted.") }, } diff --git a/pkg/connector/sticker_reactions_test.go b/pkg/connector/sticker_reactions_test.go new file mode 100644 index 00000000..49e119de --- /dev/null +++ b/pkg/connector/sticker_reactions_test.go @@ -0,0 +1,84 @@ +package connector + +import ( + "encoding/json" + "net/http" + "strings" + "testing" + "time" + + "maunium.net/go/mautrix/bridgev2" + "maunium.net/go/mautrix/bridgev2/database" + "maunium.net/go/mautrix/bridgev2/networkid" + "maunium.net/go/mautrix/event" + + "github.com/highesttt/matrix-line-messenger/pkg/line" +) + +func TestImportedLINEEmojiReaction(t *testing.T) { + lc, _ := stickerTestClient(t) + pack, _ := emojiTestPack(t, lc) + key := string(pack.Content.Images["brown_bear_cat"].URL) + msg := &bridgev2.MatrixReaction{ + MatrixEventBase: bridgev2.MatrixEventBase[*event.ReactionEventContent]{ + Content: &event.ReactionEventContent{ + RelatesTo: event.RelatesTo{Type: event.RelAnnotation, Key: key}, + }, + Portal: &bridgev2.Portal{Portal: &database.Portal{PortalKey: networkid.PortalKey{ID: "ufriend"}}}, + }, + TargetMessage: &database.Message{ID: "123456789"}, + } + pre, err := lc.PreHandleMatrixReaction(t.Context(), msg) + if err != nil { + t.Fatal(err) + } + if pre.EmojiID != "paid:emojipack:001" || pre.MaxReactions != 1 { + t.Fatalf("unexpected pre-handled reaction: %+v", pre) + } + + previous := newLineAPIClient + t.Cleanup(func() { newLineAPIClient = previous }) + calls := 0 + newLineAPIClient = func(token string) *line.Client { + c := line.NewClient(token) + c.HTTPClient = &http.Client{Transport: stickerTestTransport(func(r *http.Request) (*http.Response, error) { + if !strings.HasSuffix(r.URL.Path, "/react") { + t.Fatalf("unexpected request: %s", r.URL) + } + var args []line.ReactRequest + if err := json.NewDecoder(r.Body).Decode(&args); err != nil { + t.Fatal(err) + } + if len(args) != 1 || args[0].MessageID != "123456789" { + t.Fatalf("unexpected reaction request: %+v", args) + } + paid := args[0].ReactionType.PaidReactionType + if paid == nil || paid.ProductID != "emojipack" || paid.EmojiID != "001" || paid.Version != 2 || paid.ResourceType != 2 { + t.Fatalf("unexpected paid reaction: %+v", paid) + } + calls++ + return stickerResponse(`{"code":0}`), nil + })} + return c + } + reaction, err := lc.HandleMatrixReaction(t.Context(), msg) + if err != nil { + t.Fatal(err) + } + if calls != 1 || reaction.Emoji != key || reaction.EmojiID != pre.EmojiID { + t.Fatalf("reaction=%+v calls=%d", reaction, calls) + } + + lc.stickerCatalogs[line.SticonShop] = stickerCatalog{Fetched: time.Now()} + if _, err = lc.resolveMatrixReaction(t.Context(), msg); err == nil { + t.Fatal("unowned emoji accepted") + } + stickerPack, err := lc.DownloadImagePack(t.Context(), "line://stickershop/123") + if err != nil { + t.Fatal(err) + } + msg.Content.RelatesTo.Key = string(stickerPack.Content.Images["line_100"].URL) + if _, err = lc.resolveMatrixReaction(t.Context(), msg); err == nil { + t.Fatal("full-size sticker treated as LINE emoji") + } +} diff --git a/pkg/connector/stickers.go b/pkg/connector/stickers.go index 2cdc7fa7..44639622 100644 --- a/pkg/connector/stickers.go +++ b/pkg/connector/stickers.go @@ -31,12 +31,18 @@ var _ bridgev2.StickerImportingNetworkAPI = (*LineClient)(nil) var lineAssetID = regexp.MustCompile(`^[A-Za-z0-9_-]{1,100}$`) -var stickerShops = []string{line.StickerShop} +var stickerShops = []string{line.StickerShop, line.SticonShop} const ( stickerCDN = "https://stickershop.line-scdn.net" maxStickerAssetSize = 10 * 1024 * 1024 maxStickersPerPack = 1000 + maxSticonMetaSize = 1024 * 1024 + maxEmotesPerMessage = 250 + maxTextLength = 5000 + maxShortcodeLength = 100 + sticonMetaTTL = time.Hour + maxSticonMetaCacheSize = 256 stickerCatalogTTL = 5 * time.Minute stickerSyncInterval = time.Hour stickerRoomCreateTimeout = 10 * time.Minute @@ -47,6 +53,11 @@ type stickerCatalog struct { Fetched time.Time } +type sticonMetaCache struct { + Emotes []lineSticker + Fetched time.Time +} + type lineSticker struct { Shop string `json:"shop"` ProductID string `json:"product_id"` @@ -78,6 +89,13 @@ func (s lineSticker) metadata() map[string]string { } func (s lineSticker) assetPath() string { + if s.Shop == line.SticonShop { + suffix := "" + if s.Option == "A" { + suffix = "_animation" + } + return "/sticonshop/v1/sticon/" + s.ProductID + "/android/" + s.ID + suffix + ".png" + } prefix := "/stickershop/v1/sticker/" + s.ID if s.Hash != "" { prefix = "/stickershop/v2/sticker/" + s.ID + "/" + s.Hash @@ -134,7 +152,7 @@ func parseLinePackURL(rawURL string) (shop, productID string, err error) { } productID = parts[2] default: - return "", "", fmt.Errorf("use a LINE Store product URL or line://stickershop/PACK_ID") + return "", "", fmt.Errorf("use a LINE Store product URL or line://stickershop/PACK_ID or line://sticonshop/PACK_ID") } if !lineAssetID.MatchString(productID) { return "", "", fmt.Errorf("invalid LINE product ID") @@ -146,6 +164,8 @@ func shopFromStorePath(path string) (string, error) { switch path { case "stickershop": return line.StickerShop, nil + case "emojishop": + return line.SticonShop, nil default: return "", fmt.Errorf("unsupported LINE Store section %q", path) } @@ -186,9 +206,13 @@ func (lc *LineClient) resetStickerCatalogs() { } func packMetadata(shop string, product line.ShopProduct) *event.ImagePackMetadata { + usage := []event.ImagePackUsage{event.ImagePackUsageSticker} + if shop == line.SticonShop { + usage = []event.ImagePackUsage{event.ImagePackUsageEmoji, event.ImagePackUsageSticker} + } return &event.ImagePackMetadata{ DisplayName: product.Name, - Usage: []event.ImagePackUsage{event.ImagePackUsageSticker}, + Usage: usage, BridgedPack: &event.BridgedStickerPack{ Network: "line", URL: "line://" + shop + "/" + product.ID, @@ -200,12 +224,20 @@ func supportedProduct(shop string, product line.ShopProduct) bool { if !lineAssetID.MatchString(product.ID) || !activeProduct(product, time.Now()) { return false } - summary := product.ProductTypeSummary.Sticker - if shop != line.StickerShop || summary == nil { + switch shop { + case line.StickerShop: + summary := product.ProductTypeSummary.Sticker + if summary == nil { + return false + } + _, err := stickerOption(summary.ResourceType) + return err == nil + case line.SticonShop: + summary := product.ProductTypeSummary.Sticon + return summary != nil && (summary.ResourceType == 1 || summary.ResourceType == 2) + default: return false } - _, err := stickerOption(summary.ResourceType) - return err == nil } func (lc *LineClient) ListImagePacks(ctx context.Context) ([]*event.ImagePackMetadata, error) { @@ -268,11 +300,14 @@ func (lc *LineClient) fetchStickerAsset(ctx context.Context, path string, limit return data, nil } -func (lc *LineClient) productStickers(_ context.Context, shop string, product line.ShopProduct) ([]lineSticker, error) { +func (lc *LineClient) productStickers(ctx context.Context, shop string, product line.ShopProduct) ([]lineSticker, error) { version := product.LatestVersion.String() if n, err := strconv.ParseUint(version, 10, 32); err != nil || n == 0 { return nil, fmt.Errorf("invalid LINE pack version") } + if shop == line.SticonShop { + return lc.productEmotes(ctx, product, version) + } summary := product.ProductTypeSummary.Sticker if summary == nil { return nil, fmt.Errorf("missing sticker summary") @@ -311,6 +346,106 @@ func (lc *LineClient) productStickers(_ context.Context, shop string, product li return stickers, nil } +func (lc *LineClient) productEmotes(ctx context.Context, product line.ShopProduct, version string) ([]lineSticker, error) { + cacheKey := product.ID + "/" + version + lc.sticonMetaMu.Lock() + defer lc.sticonMetaMu.Unlock() + if cached, ok := lc.sticonMeta[cacheKey]; ok && time.Since(cached.Fetched) < sticonMetaTTL { + return cached.Emotes, nil + } + summary := product.ProductTypeSummary.Sticon + if summary == nil { + return nil, fmt.Errorf("missing sticon summary") + } + data, err := lc.fetchStickerAsset(ctx, "/sticonshop/v1/product/"+product.ID+"/android/meta.json", maxSticonMetaSize) + if err != nil { + return nil, err + } + var meta struct { + Orders []string `json:"orders"` + AltTexts map[string]string `json:"altTexts"` + } + if err = json.Unmarshal(data, &meta); err != nil { + return nil, err + } + if len(meta.Orders) == 0 { + return nil, fmt.Errorf("LINE pack is empty") + } + if len(meta.Orders) > maxStickersPerPack { + return nil, fmt.Errorf("emoji pack too large") + } + option := "" + if summary.ResourceType == 2 { + option = "A" + } + emotes := make([]lineSticker, 0, len(meta.Orders)) + for _, emoteID := range meta.Orders { + if !lineAssetID.MatchString(emoteID) { + return nil, fmt.Errorf("invalid sticon ID") + } + body := "(emoji)" + if alt := meta.AltTexts[emoteID]; alt != "" { + body = "(" + alt + ")" + } + emotes = append(emotes, lineSticker{ + Shop: line.SticonShop, + ProductID: product.ID, + ID: emoteID, + Version: version, + Option: option, + Body: body, + }) + } + if lc.sticonMeta == nil || len(lc.sticonMeta) >= maxSticonMetaCacheSize { + lc.sticonMeta = make(map[string]sticonMetaCache) + } + lc.sticonMeta[cacheKey] = sticonMetaCache{Emotes: emotes, Fetched: time.Now()} + return emotes, nil +} + +func shortcodeWords(text string) string { + var words strings.Builder + for _, r := range strings.ToLower(text) { + if (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') { + words.WriteRune(r) + } else if words.Len() > 0 && !strings.HasSuffix(words.String(), "_") { + words.WriteByte('_') + } + } + return strings.Trim(words.String(), "_") +} + +func truncateShortcode(shortcode string, limit int) string { + if len(shortcode) <= limit { + return shortcode + } + return strings.TrimRight(shortcode[:limit], "_") +} + +func emoteShortcodes(packName string, emotes []lineSticker) []string { + pack := shortcodeWords(packName) + if pack == "" { + pack = "line" + } + shortcodes := make([]string, len(emotes)) + used := make(map[string]bool, len(emotes)) + for i, emote := range emotes { + name := shortcodeWords(strings.TrimSuffix(strings.TrimPrefix(emote.Body, "("), ")")) + if name == "" { + name = strconv.Itoa(i + 1) + } + base := truncateShortcode(pack+"_"+name, maxShortcodeLength) + shortcode := base + for n := 2; used[shortcode]; n++ { + suffix := "_" + strconv.Itoa(n) + shortcode = truncateShortcode(base, maxShortcodeLength-len(suffix)) + suffix + } + used[shortcode] = true + shortcodes[i] = shortcode + } + return shortcodes +} + func stickerKey(kind, value string) database.Key { return database.Key(fmt.Sprintf("line.sticker.%s.%x", kind, sha256.Sum256([]byte(value)))) } @@ -399,12 +534,20 @@ func (lc *LineClient) DownloadImagePack(ctx context.Context, rawURL string) (*br Images: make(map[string]*event.ImagePackImage, len(stickers)), }, } - for _, sticker := range stickers { + var shortcodes []string + if shop == line.SticonShop { + shortcodes = emoteShortcodes(product.Name, stickers) + } + for i, sticker := range stickers { img, err := lc.importSticker(ctx, sticker) if err != nil { return nil, err } - pack.Content.Images["line_"+sticker.ID] = img + shortcode := "line_" + sticker.ID + if shortcodes != nil { + shortcode = shortcodes[i] + } + pack.Content.Images[shortcode] = img } return pack, nil } @@ -601,7 +744,7 @@ func (lc *LineClient) syncStickerPacks(ctx context.Context) { defer ticker.Stop() for { if err := lc.syncStickerPacksOnce(ctx); err != nil && ctx.Err() == nil { - lc.UserLogin.Log.Warn().Err(err).Msg("Failed to synchronize LINE sticker packs") + lc.UserLogin.Log.Warn().Err(err).Msg("Failed to synchronize LINE sticker and emoji packs") } select { case <-ctx.Done(): diff --git a/pkg/connector/stickers_test.go b/pkg/connector/stickers_test.go index f4238bcb..923e1332 100644 --- a/pkg/connector/stickers_test.go +++ b/pkg/connector/stickers_test.go @@ -9,7 +9,9 @@ import ( "image" "image/png" "io" + "maps" "net/http" + "slices" "strings" "testing" "time" @@ -96,6 +98,7 @@ func stickerTestClient(t *testing.T) (*LineClient, *stickerTestMatrix) { }, stickerCatalogs: map[string]stickerCatalog{ line.StickerShop: {Products: []line.ShopProduct{stickerTestProduct(t)}, Fetched: time.Now()}, + line.SticonShop: {Fetched: time.Now()}, }, } var pngData bytes.Buffer @@ -322,3 +325,126 @@ func TestStickerPacksNotAppliedAfterDisconnect(t *testing.T) { t.Fatalf("state writes after disconnect: %d", matrix.stateWrites) } } + +func emojiTestPack(t *testing.T, lc *LineClient) (*bridgev2.ImportedImagePack, *int) { + t.Helper() + var product line.ShopProduct + productJSON := `{"id":"emojipack","name":"Brown Bear","latestVersion":"2","validUntil":"-1",` + + `"productTypeSummary":{"sticonSummary":{"sticonResourceType":2}}}` + if err := json.Unmarshal([]byte(productJSON), &product); err != nil { + t.Fatal(err) + } + lc.stickerCatalogs[line.SticonShop] = stickerCatalog{Products: []line.ShopProduct{product}, Fetched: time.Now()} + assetTransport := lc.HTTPClient.Transport + metadataFetches := 0 + lc.HTTPClient.Transport = stickerTestTransport(func(r *http.Request) (*http.Response, error) { + if strings.HasSuffix(r.URL.Path, "meta.json") { + metadataFetches++ + return stickerResponse(`{"orders":["001","002"],"altTexts":{"001":"cat","002":"dog"}}`), nil + } + return assetTransport.RoundTrip(r) + }) + pack, err := lc.DownloadImagePack(t.Context(), "line://sticonshop/emojipack") + if err != nil { + t.Fatal(err) + } + return pack, &metadataFetches +} + +func TestEmojiPackSendsNativeInlineEmoji(t *testing.T) { + lc, _ := stickerTestClient(t) + pack, metadataFetches := emojiTestPack(t, lc) + if len(pack.Content.Metadata.Usage) != 2 || pack.Content.Metadata.Usage[0] != event.ImagePackUsageEmoji { + t.Fatalf("emoji pack usage = %v", pack.Content.Metadata.Usage) + } + cat := pack.Content.Images["brown_bear_cat"] + if cat == nil || pack.Content.Images["brown_bear_dog"] == nil { + t.Fatalf("unexpected emote keys: %v", slices.Collect(maps.Keys(pack.Content.Images))) + } + for range 20 { + for _, img := range pack.Content.Images { + if _, err := lc.resolveSticker(t.Context(), img.URL); err != nil { + t.Fatal(err) + } + } + } + if *metadataFetches != 1 { + t.Fatalf("downloaded meta.json %d times", *metadataFetches) + } + + previous := newLineAPIClient + t.Cleanup(func() { newLineAPIClient = previous }) + var sent []line.Message + newLineAPIClient = func(token string) *line.Client { + c := line.NewClient(token) + c.HTTPClient = &http.Client{Transport: stickerTestTransport(func(r *http.Request) (*http.Response, error) { + var args []json.RawMessage + if err := json.NewDecoder(r.Body).Decode(&args); err != nil { + t.Fatal(err) + } + var msg line.Message + if err := json.Unmarshal(args[1], &msg); err != nil { + t.Fatal(err) + } + sent = append(sent, msg) + return stickerResponse(`{"code":0,"data":{"id":"text"}}`), nil + })} + return c + } + inline := &event.MessageEventContent{ + MsgType: event.MsgText, + Body: "fallback", + Format: event.FormatHTML, + FormattedBody: fmt.Sprintf(`🙂 (cat)`, cat.URL), + } + standalone := &event.MessageEventContent{MsgType: event.CapMsgSticker, Body: "cat", URL: cat.URL} + for _, content := range []*event.MessageEventContent{inline, standalone} { + if _, err := lc.HandleMatrixMessage(t.Context(), testMatrixMessage("ufriend", content)); err != nil { + t.Fatal(err) + } + } + if inline.Body != "fallback" || standalone.MsgType != event.CapMsgSticker { + t.Fatal("original event content was mutated") + } + if len(sent) != 2 || sent[0].Text != "🙂 (cat)" || sent[1].Text != "(cat)" { + t.Fatalf("unexpected messages: %+v", sent) + } + for _, msg := range sent { + if msg.ContentType != 0 || msg.ContentMetadata["REPLACE"] == "" || + msg.ContentMetadata["STICON_OWNERSHIP"] != `["emojipack"]` { + t.Fatalf("invalid inline emoji message: %+v", msg) + } + } + + lc.stickerCatalogs[line.SticonShop] = stickerCatalog{Fetched: time.Now()} + if _, err := lc.HandleMatrixMessage(t.Context(), testMatrixMessage("ufriend", inline)); err != nil { + t.Fatalf("unowned emoji failed the send: %v", err) + } + fallback := sent[len(sent)-1] + if fallback.Text != "fallback" || fallback.ContentMetadata["REPLACE"] != "" { + t.Fatalf("unowned emoji was not sent as plain text: %+v", fallback) + } +} + +func TestEmoteShortcodes(t *testing.T) { + emotes := []lineSticker{ + {Body: "(Happy!)"}, + {Body: "(happy)"}, + {Body: "(猫)"}, + {Body: "(" + strings.Repeat("very long description ", 10) + ")"}, + } + shortcodes := emoteShortcodes("Brown Bear", emotes) + expected := []string{"brown_bear_happy", "brown_bear_happy_2", "brown_bear_3"} + for i, want := range expected { + if shortcodes[i] != want { + t.Fatalf("shortcode %d = %q, want %q", i, shortcodes[i], want) + } + } + long := shortcodes[3] + if len(long) > maxShortcodeLength || !lineAssetID.MatchString(long) || !strings.HasPrefix(long, "brown_bear_very_long") { + t.Fatalf("invalid long shortcode %q", long) + } + if got := emoteShortcodes("", []lineSticker{{Body: "(cat)"}}); got[0] != "line_cat" { + t.Fatalf("unnamed pack shortcode = %q", got[0]) + } +} diff --git a/pkg/connector/sticon_send.go b/pkg/connector/sticon_send.go new file mode 100644 index 00000000..dbeb3e3e --- /dev/null +++ b/pkg/connector/sticon_send.go @@ -0,0 +1,143 @@ +package connector + +import ( + "context" + "crypto/rand" + "encoding/hex" + "encoding/json" + "fmt" + "slices" + "strconv" + "strings" + + "maunium.net/go/mautrix/event" + "maunium.net/go/mautrix/format" + "maunium.net/go/mautrix/id" + + "github.com/highesttt/matrix-line-messenger/pkg/connector/handlers" + "github.com/highesttt/matrix-line-messenger/pkg/line" +) + +type sticonReplacement struct { + marker string + emote lineSticker + alt string +} + +func utf16Length(text string) int { + length, _ := byteIndexToUTF16Offset(text, len(text)) + return length +} + +func convertOutgoingSticons( + ctx context.Context, + content *event.MessageEventContent, + resolve func(id.ContentURIString) (*lineSticker, error), +) (string, map[string]string, error) { + if content.Format != event.FormatHTML || !strings.Contains(content.FormattedBody, "data-mx-emoticon") { + return content.Body, nil, nil + } + nonce := make([]byte, 16) + if _, err := rand.Read(nonce); err != nil { + return "", nil, err + } + markerPrefix := "LINESTICON" + hex.EncodeToString(nonce) + var replacements []sticonReplacement + var resolveErr error + resolved := make(map[id.ContentURIString]*lineSticker) + parser := *format.TextHTMLParser + parser.ImageConverter = func(src, alt, _, _, _ string, isEmoji bool) string { + if !isEmoji || resolveErr != nil { + return alt + } + if len(replacements) >= maxEmotesPerMessage { + resolveErr = fmt.Errorf("LINE supports at most %d inline emoji per message", maxEmotesPerMessage) + return alt + } + mxc := id.ContentURIString(src) + emote, ok := resolved[mxc] + if !ok { + emote, resolveErr = resolve(mxc) + if resolveErr != nil { + return alt + } + resolved[mxc] = emote + } + if emote == nil || emote.Shop != line.SticonShop { + return alt + } + if alt == "" { + alt = emote.Body + } + marker := fmt.Sprintf("%s_%d_END", markerPrefix, len(replacements)) + replacements = append(replacements, sticonReplacement{marker: marker, emote: *emote, alt: alt}) + return marker + } + text := parser.Parse(content.FormattedBody, format.NewContext(ctx)) + if resolveErr != nil { + return "", nil, resolveErr + } + if len(replacements) == 0 { + return content.Body, nil, nil + } + + var body strings.Builder + resources := make([]handlers.SticonResource, 0, len(replacements)) + var ownership []string + offset := 0 + for _, replacement := range replacements { + before, after, found := strings.Cut(text, replacement.marker) + if !found { + return "", nil, fmt.Errorf("could not locate inline emoji") + } + version, err := strconv.Atoi(replacement.emote.Version) + if err != nil { + return "", nil, fmt.Errorf("invalid LINE emoji version: %w", err) + } + resourceType := "STATIC" + if replacement.emote.Option == "A" { + resourceType = "ANIMATION" + } + body.WriteString(before) + offset += utf16Length(before) + end := offset + utf16Length(replacement.alt) + resources = append(resources, handlers.SticonResource{ + Start: offset, + End: end, + ProductID: replacement.emote.ProductID, + SticonID: replacement.emote.ID, + Version: version, + ResourceType: resourceType, + }) + body.WriteString(replacement.alt) + offset = end + text = after + if !slices.Contains(ownership, replacement.emote.ProductID) { + ownership = append(ownership, replacement.emote.ProductID) + } + } + body.WriteString(text) + if offset+utf16Length(text) > maxTextLength { + return "", nil, fmt.Errorf("LINE text exceeds %d UTF-16 units", maxTextLength) + } + replace, err := json.Marshal(map[string]any{"sticon": map[string]any{"resources": resources}}) + if err != nil { + return "", nil, err + } + ownershipJSON, err := json.Marshal(ownership) + if err != nil { + return "", nil, err + } + return body.String(), map[string]string{ + "REPLACE": string(replace), + "STICON_OWNERSHIP": string(ownershipJSON), + }, nil +} + +func lineTextPayload(body string, metadata map[string]string) ([]byte, error) { + payload := map[string]any{"text": body} + if replace := metadata["REPLACE"]; replace != "" { + payload["REPLACE"] = json.RawMessage(replace) + } + return json.Marshal(payload) +} diff --git a/pkg/connector/sticon_send_test.go b/pkg/connector/sticon_send_test.go new file mode 100644 index 00000000..e4f3a4e2 --- /dev/null +++ b/pkg/connector/sticon_send_test.go @@ -0,0 +1,88 @@ +package connector + +import ( + "encoding/json" + "errors" + "strings" + "testing" + + "maunium.net/go/mautrix/event" + "maunium.net/go/mautrix/id" + + "github.com/highesttt/matrix-line-messenger/pkg/connector/handlers" + "github.com/highesttt/matrix-line-messenger/pkg/line" +) + +func TestOutgoingSticonUTF16AndEncryptedPayload(t *testing.T) { + emoji := `(猫)` + content := &event.MessageEventContent{ + Body: "fallback", + Format: event.FormatHTML, + FormattedBody: "

🙂 Hello " + emoji + " & " + emoji + "

", + } + calls := 0 + body, meta, err := convertOutgoingSticons(t.Context(), content, func(id.ContentURIString) (*lineSticker, error) { + calls++ + return &lineSticker{Shop: line.SticonShop, ID: "001", ProductID: "cats", Version: "2", Option: "A"}, nil + }) + if err != nil { + t.Fatal(err) + } + if body != "🙂 Hello (猫) & (猫)" || calls != 1 { + t.Fatalf("%q calls %d", body, calls) + } + var replace struct { + Sticon struct { + Resources []handlers.SticonResource `json:"resources"` + } `json:"sticon"` + } + if err = json.Unmarshal([]byte(meta["REPLACE"]), &replace); err != nil { + t.Fatal(err) + } + r := replace.Sticon.Resources + if len(r) != 2 || r[0].Start != 9 || r[0].End != 12 || r[1].Start != 15 || r[1].End != 18 || + r[0].ResourceType != "ANIMATION" || r[0].Version != 2 { + t.Fatalf("%+v", r) + } + if meta["STICON_OWNERSHIP"] != `["cats"]` { + t.Fatal(meta) + } + payload, err := lineTextPayload(body, meta) + if err != nil { + t.Fatal(err) + } + var decoded map[string]any + json.Unmarshal(payload, &decoded) + if _, ok := decoded["REPLACE"].(map[string]any); !ok { + t.Fatalf("REPLACE must be encrypted JSON object: %s", payload) + } + if content.Body != "fallback" { + t.Fatal("mutated input") + } +} + +func TestOutgoingSticonFallbackAndLimits(t *testing.T) { + content := &event.MessageEventContent{ + Body: "fallback", + Format: event.FormatHTML, + FormattedBody: `image unknown`, + } + body, meta, err := convertOutgoingSticons(t.Context(), content, func(id.ContentURIString) (*lineSticker, error) { return nil, nil }) + if body != "fallback" || meta != nil || err != nil { + t.Fatalf("%q %v %v", body, meta, err) + } + content.FormattedBody = strings.Repeat(`x`, 251) + calls := 0 + _, _, err = convertOutgoingSticons(t.Context(), content, func(id.ContentURIString) (*lineSticker, error) { + calls++ + return &lineSticker{Shop: line.SticonShop, ID: "001", ProductID: "cats", Version: "1"}, nil + }) + if err == nil || calls != 1 { + t.Fatalf("limit failed %v calls %d", err, calls) + } + expected := errors.New("not owned") + _, _, err = convertOutgoingSticons(t.Context(), content, func(id.ContentURIString) (*lineSticker, error) { return nil, expected }) + if !errors.Is(err, expected) { + t.Fatal(err) + } +} diff --git a/pkg/connector/userinfo.go b/pkg/connector/userinfo.go index 4e1be151..c032018a 100644 --- a/pkg/connector/userinfo.go +++ b/pkg/connector/userinfo.go @@ -46,7 +46,9 @@ func (lc *LineClient) HandleMatrixReadReceipt(ctx context.Context, read *bridgev func (lc *LineClient) GetCapabilities(ctx context.Context, portal *bridgev2.Portal) *event.RoomFeatures { return &event.RoomFeatures{ - MaxTextLength: 5000, + MaxTextLength: maxTextLength, + Formatting: event.FormattingFeatureMap{event.FmtCustomEmoji: event.CapLevelPartialSupport}, + CustomEmojiReactions: true, Reply: event.CapLevelFullySupported, Reaction: event.CapLevelPartialSupport, ReactionCount: 1, From c09b6f94a71bc171bf161b3172256afdcbcdee83 Mon Sep 17 00:00:00 2001 From: formeo14 Date: Sun, 27 Sep 2026 13:20:34 +0900 Subject: [PATCH 4/4] fix: verify LINE media HMAC before decrypting --- pkg/connector/handle_message.go | 2 +- pkg/connector/handlers/handler.go | 4 +- pkg/connector/handlers/handler_test.go | 6 +-- pkg/connector/media.go | 24 ++++++++++- pkg/connector/media_auth_test.go | 57 ++++++++++++++++++++++++++ 5 files changed, 85 insertions(+), 8 deletions(-) create mode 100644 pkg/connector/media_auth_test.go diff --git a/pkg/connector/handle_message.go b/pkg/connector/handle_message.go index cb190c77..80e6348a 100644 --- a/pkg/connector/handle_message.go +++ b/pkg/connector/handle_message.go @@ -35,7 +35,7 @@ func (lc *LineClient) newMessageHandler() *handlers.Handler { HTTPClient: lc.HTTPClient, RecoverClient: lc.recoverClientAfterAuthError, NewClient: func() *line.Client { return lc.newClient() }, - DecryptMedia: lc.decryptImageData, + DecryptMedia: lc.decryptMediaData, } connector, ok := lc.UserLogin.Bridge.Network.(*LineConnector) if ok && connector.directMedia.Load() { diff --git a/pkg/connector/handlers/handler.go b/pkg/connector/handlers/handler.go index 9a8829d0..cc7d98fa 100644 --- a/pkg/connector/handlers/handler.go +++ b/pkg/connector/handlers/handler.go @@ -39,7 +39,7 @@ type Handler struct { DownloadAlbumPreview func(ctx context.Context, client *line.Client, oid, chatID, albumID string) ([]byte, error) // DecryptMedia decrypts E2EE encrypted media data using the given key material. - DecryptMedia func(data []byte, keyMaterial string) ([]byte, error) + DecryptMedia func(data []byte, keyMaterial, kind string) ([]byte, error) } func (h *Handler) decryptDownloadedMedia(data []byte, decryptedBody string, metadata map[string]string, kind string) ([]byte, error) { @@ -96,7 +96,7 @@ func (h *Handler) decryptMediaWithKeys(data []byte, keys []string, encrypted boo decryptErrors := make([]error, 0, len(keys)) for _, key := range keys { - decrypted, err := h.DecryptMedia(data, key) + decrypted, err := h.DecryptMedia(data, key, kind) if err == nil { return decrypted, nil } diff --git a/pkg/connector/handlers/handler_test.go b/pkg/connector/handlers/handler_test.go index 92881a96..39d1a7aa 100644 --- a/pkg/connector/handlers/handler_test.go +++ b/pkg/connector/handlers/handler_test.go @@ -17,7 +17,7 @@ func TestDecryptDownloadedMediaUsesBodyKey(t *testing.T) { ciphertext := []byte("ciphertext") plaintext := []byte("plaintext") var keys []string - h := &Handler{DecryptMedia: func(data []byte, key string) ([]byte, error) { + h := &Handler{DecryptMedia: func(data []byte, key, kind string) ([]byte, error) { if !bytes.Equal(data, ciphertext) { t.Fatalf("decrypt input = %q, want ciphertext", data) } @@ -40,7 +40,7 @@ func TestDecryptDownloadedMediaUsesBodyKey(t *testing.T) { func TestDecryptDownloadedMediaFallsBackToENCKM(t *testing.T) { ciphertext := []byte("ciphertext") var keys []string - h := &Handler{DecryptMedia: func(data []byte, key string) ([]byte, error) { + h := &Handler{DecryptMedia: func(data []byte, key, kind string) ([]byte, error) { if !bytes.Equal(data, ciphertext) { t.Fatalf("decrypt input = %q, want original ciphertext", data) } @@ -88,7 +88,7 @@ func TestDecryptDownloadedMediaFailsClosed(t *testing.T) { ciphertext := []byte("ciphertext") decryptErr := errors.New("invalid media key") var calls int - h := &Handler{DecryptMedia: func([]byte, string) ([]byte, error) { + h := &Handler{DecryptMedia: func([]byte, string, string) ([]byte, error) { calls++ return nil, decryptErr }} diff --git a/pkg/connector/media.go b/pkg/connector/media.go index 98e0d9ef..28656fd5 100644 --- a/pkg/connector/media.go +++ b/pkg/connector/media.go @@ -85,6 +85,15 @@ func convertHEICToJPEG(ctx context.Context, imageData []byte) ([]byte, error) { // LINE's E2EE file format: [encrypted_data][32-byte HMAC] // The keyMaterial is derived using HKDF to get encKey (32), macKey (32), and nonce (12 bytes) func (lc *LineClient) decryptImageData(encryptedData []byte, keyMaterialB64 string) ([]byte, error) { + return lc.decryptMediaData(encryptedData, keyMaterialB64, "image") +} + +func (lc *LineClient) decryptMediaData(encryptedData []byte, keyMaterialB64, kind string) ([]byte, error) { + switch kind { + case "image", "file", "audio", "video": + default: + return nil, fmt.Errorf("unsupported encrypted media kind %q", kind) + } keyMaterial, err := base64.StdEncoding.DecodeString(keyMaterialB64) if err != nil { return nil, fmt.Errorf("failed to decode key material: %w", err) @@ -99,7 +108,7 @@ func (lc *LineClient) decryptImageData(encryptedData []byte, keyMaterialB64 stri } encKey := derived[0:32] - // macKey := derived[32:64] // for HMAC verification + macKey := derived[32:64] nonce := derived[64:76] // Create 16-byte counter: nonce(12 bytes) + zero counter(4 bytes) @@ -110,7 +119,18 @@ func (lc *LineClient) decryptImageData(encryptedData []byte, keyMaterialB64 stri if len(encryptedData) < 32 { return nil, fmt.Errorf("encrypted data too short (< 32 bytes for HMAC)") } - encryptedData = encryptedData[:len(encryptedData)-32] + tag := encryptedData[len(encryptedData)-sha256.Size:] + encryptedData = encryptedData[:len(encryptedData)-sha256.Size] + + mac := hmac.New(sha256.New, macKey) + if kind == "video" { + mac.Write(generateChunkHashes(encryptedData)) + } else { + mac.Write(encryptedData) + } + if !hmac.Equal(tag, mac.Sum(nil)) { + return nil, fmt.Errorf("LINE media HMAC verification failed") + } block, err := aes.NewCipher(encKey) if err != nil { diff --git a/pkg/connector/media_auth_test.go b/pkg/connector/media_auth_test.go new file mode 100644 index 00000000..61d9a50d --- /dev/null +++ b/pkg/connector/media_auth_test.go @@ -0,0 +1,57 @@ +package connector + +import ( + "bytes" + "encoding/base64" + "testing" +) + +func TestMediaAuthentication(t *testing.T) { + client := &LineClient{} + + plain := bytes.Repeat([]byte("authenticated media"), 15000) + for _, streaming := range []bool{false, true} { + kind := "image" + if streaming { + kind = "video" + } + var encrypted []byte + var key string + var err error + if streaming { + encrypted, key, err = client.encryptVideoData(plain) + } else { + encrypted, key, err = client.encryptFileData(plain) + } + if err != nil { + t.Fatal(err) + } + decoded, err := client.decryptMediaData(encrypted, key, kind) + if err != nil || !bytes.Equal(decoded, plain) { + t.Fatalf("valid media rejected, streaming=%v: %v", streaming, err) + } + for _, pos := range []int{0, 131072, len(encrypted) - 33, len(encrypted) - 1} { + damaged := bytes.Clone(encrypted) + damaged[pos] ^= 1 + decoded, err = client.decryptMediaData(damaged, key, kind) + if err == nil || decoded != nil { + t.Fatalf("unauthenticated plaintext returned at %d streaming=%v", pos, streaming) + } + } + if streaming { + forged := append(generateChunkHashes(encrypted[:len(encrypted)-32]), encrypted[len(encrypted)-32:]...) + if output, err := client.decryptMediaData(forged, key, "video"); err == nil || output != nil { + t.Fatal("chunk hash substitution accepted") + } + } + wrongKey := base64.StdEncoding.EncodeToString(bytes.Repeat([]byte{42}, 32)) + if decoded, err = client.decryptMediaData(encrypted, wrongKey, kind); err == nil || decoded != nil { + t.Fatal("wrong key accepted") + } + for _, size := range []int{0, 31, 32, len(encrypted) - 1} { + if decoded, err = client.decryptMediaData(encrypted[:size], key, kind); err == nil || decoded != nil { + t.Fatalf("truncated ciphertext accepted: %d", size) + } + } + } +}