From 62d6145c149086ffa112dc43fdba0e24f589dfcc Mon Sep 17 00:00:00 2001 From: Sumner Evans Date: Wed, 10 Jul 2024 00:41:21 -0600 Subject: [PATCH] stickers: support receiving and converting Signed-off-by: Sumner Evans --- go.mod | 2 +- go.sum | 4 +- pkg/connector/config.go | 14 ++-- pkg/connector/directdownload.go | 11 +++ pkg/connector/media/sticker.go | 85 ++++++++++++++++++++++ pkg/connector/media/transfer.go | 94 +++++++++++++----------- pkg/connector/msgconv/tomatrix.go | 114 ++++++++++++++++++++++-------- 7 files changed, 247 insertions(+), 77 deletions(-) create mode 100644 pkg/connector/media/sticker.go diff --git a/go.mod b/go.mod index 6334645c..1ae0047f 100644 --- a/go.mod +++ b/go.mod @@ -6,7 +6,7 @@ require ( github.com/gotd/td v0.102.0 github.com/rs/zerolog v1.33.0 github.com/stretchr/testify v1.9.0 - go.mau.fi/util v0.5.1-0.20240708233020-c2f9af6fecf8 + go.mau.fi/util v0.5.1-0.20240710154926-931b33d6d530 go.mau.fi/zerozap v0.1.1 go.uber.org/zap v1.27.0 maunium.net/go/mautrix v0.19.0-beta.1.0.20240706124659-b4057a26c3ed diff --git a/go.sum b/go.sum index e0c88fe4..0ccaf372 100644 --- a/go.sum +++ b/go.sum @@ -67,8 +67,8 @@ github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= github.com/yuin/goldmark v1.7.4 h1:BDXOHExt+A7gwPCJgPIIq7ENvceR7we7rOS9TNoLZeg= github.com/yuin/goldmark v1.7.4/go.mod h1:uzxRWxtg69N339t3louHJ7+O03ezfj6PlliRlaOzY1E= -go.mau.fi/util v0.5.1-0.20240708233020-c2f9af6fecf8 h1:7ntkhSR0G/dIwAPjcoOoJz+bPne0gA4PypEn8euMMx0= -go.mau.fi/util v0.5.1-0.20240708233020-c2f9af6fecf8/go.mod h1:DsJzUrJAG53lCZnnYvq9/mOyLuPScWwYhvETiTrpdP4= +go.mau.fi/util v0.5.1-0.20240710154926-931b33d6d530 h1:ZWMrLC+Fn2AmKL8HM04YY0zyMDMOagQZVukpxp0rmic= +go.mau.fi/util v0.5.1-0.20240710154926-931b33d6d530/go.mod h1:DsJzUrJAG53lCZnnYvq9/mOyLuPScWwYhvETiTrpdP4= go.mau.fi/zeroconfig v0.1.2 h1:DKOydWnhPMn65GbXZOafgkPm11BvFashZWLct0dGFto= go.mau.fi/zeroconfig v0.1.2/go.mod h1:NcSJkf180JT+1IId76PcMuLTNa1CzsFFZ0nBygIQM70= go.mau.fi/zerozap v0.1.1 h1:mxE/dW4wtkqBYOXOEEzXldk5qKB+ahsZXjoTGnvEhZQ= diff --git a/pkg/connector/config.go b/pkg/connector/config.go index 78ead697..4149aa36 100644 --- a/pkg/connector/config.go +++ b/pkg/connector/config.go @@ -3,6 +3,7 @@ package connector import ( _ "embed" "fmt" + "slices" up "go.mau.fi/util/configupgrade" "maunium.net/go/mautrix/bridgev2" @@ -25,11 +26,11 @@ var ExampleConfig string func upgradeConfig(helper up.Helper) { helper.Copy(up.Int, "app_id") helper.Copy(up.Str, "app_hash") - helper.Copy(up.Str, "animated_sticker.target") - helper.Copy(up.Bool, "animated_sticker.convert_from_webm") - helper.Copy(up.Int, "animated_sticker.args.width") - helper.Copy(up.Int, "animated_sticker.args.height") - helper.Copy(up.Int, "animated_sticker.args.fps") + helper.Copy(up.Str, "animated_sticker", "target") + helper.Copy(up.Bool, "animated_sticker", "convert_from_webm") + helper.Copy(up.Int, "animated_sticker", "args", "width") + helper.Copy(up.Int, "animated_sticker", "args", "height") + helper.Copy(up.Int, "animated_sticker", "args", "fps") } func (tg *TelegramConnector) GetConfig() (example string, data any, upgrader up.Upgrader) { @@ -43,5 +44,8 @@ func (tg *TelegramConnector) ValidateConfig() error { if tg.Config.AppHash == "" { return fmt.Errorf("app_hash is required") } + if !slices.Contains([]string{"disable", "gif", "png", "webp", "webm"}, tg.Config.AnimatedSticker.Target) { + return fmt.Errorf("unsupported animated sticker target: %s", tg.Config.AnimatedSticker.Target) + } return nil } diff --git a/pkg/connector/directdownload.go b/pkg/connector/directdownload.go index 898c9473..f0c41065 100644 --- a/pkg/connector/directdownload.go +++ b/pkg/connector/directdownload.go @@ -86,6 +86,16 @@ func (tc *TelegramConnector) Download(ctx context.Context, mediaID networkid.Med case *tg.MessageMediaPhoto: readyTransferer = transferer.WithPhoto(msgMedia.Photo) case *tg.MessageMediaDocument: + document, ok := msgMedia.Document.(*tg.Document) + if !ok { + return nil, fmt.Errorf("unknown document type %T", msgMedia.Document) + } + for _, attr := range document.GetAttributes() { + if attr.TypeID() == tg.DocumentAttributeStickerTypeID { + transferer = transferer.WithStickerConfig(tc.Config.AnimatedSticker) + } + } + readyTransferer = transferer.WithDocument(msgMedia.Document, info.Thumbnail) default: return nil, fmt.Errorf("unhandled media type %T", msgMedia) @@ -93,6 +103,7 @@ func (tc *TelegramConnector) Download(ctx context.Context, mediaID networkid.Med data, fileInfo, err := readyTransferer.Download(ctx) if err != nil { + log.Err(err).Msg("failed to download media") return nil, err } diff --git a/pkg/connector/media/sticker.go b/pkg/connector/media/sticker.go new file mode 100644 index 00000000..069a4f25 --- /dev/null +++ b/pkg/connector/media/sticker.go @@ -0,0 +1,85 @@ +package media + +import ( + "bytes" + "context" + "fmt" + "strconv" + + "github.com/rs/zerolog" + "go.mau.fi/util/ffmpeg" + "go.mau.fi/util/lottie" +) + +type AnimatedStickerConfig struct { + Target string `yaml:"target"` + ConvertFromWebm bool `yaml:"convert_from_webm"` + Args struct { + Width int `yaml:"width"` + Height int `yaml:"height"` + FPS int `yaml:"fps"` + } `yaml:"args"` +} + +type ConvertedSticker struct { + Data []byte + MIMEType string + ThumbnailData []byte + ThumbnailMIMEType string + Width int + Height int +} + +func (c AnimatedStickerConfig) convert(ctx context.Context, data []byte) ConvertedSticker { + if c.Target == "disable" { + return ConvertedSticker{Data: data, MIMEType: "application/x-tgsticker"} + } + + log := zerolog.Ctx(ctx).With().Str("animated_sticker_target", c.Target).Logger() + + if !lottie.Supported() { + log.Warn().Msg("lottie not supported, cannot convert animated stickers") + return ConvertedSticker{Data: data, MIMEType: "application/x-tgsticker"} + } else if (c.Target == "webp" || c.Target == "webm") && !ffmpeg.Supported() { + log.Warn().Msg("ffmpeg not supported, cannot convert animated stickers") + return ConvertedSticker{Data: data, MIMEType: "application/x-tgsticker"} + } + + input := bytes.NewBuffer(data) + outputWriter := new(bytes.Buffer) + var thumbnailData []byte + var mimeType, thumbnailMIMEType string + + var err error + switch c.Target { + case "png": + mimeType = "image/png" + err = lottie.Convert(ctx, input, "", outputWriter, c.Target, c.Args.Width, c.Args.Height, "1") + case "gif": + mimeType = "image/gif" + err = lottie.Convert(ctx, input, "", outputWriter, c.Target, c.Args.Width, c.Args.Height, strconv.Itoa(c.Args.FPS)) + case "webm", "webp": + thumbnailMIMEType = "image/png" + outputWriter, mimeType, thumbnailData, err = lottie.FfmpegConvert(ctx, input, c.Target, c.Args.Width, c.Args.Height, c.Args.FPS) + default: + err = fmt.Errorf("unsupported target format %s", c.Target) + } + if err != nil { + log.Err(err). + Str("target", c.Target). + Msg("failed to convert animated sticker to target format") + + // Fallback to original data + return ConvertedSticker{Data: data, MIMEType: "application/x-tgsticker"} + } + + return ConvertedSticker{ + Data: outputWriter.Bytes(), + MIMEType: mimeType, + ThumbnailData: thumbnailData, + ThumbnailMIMEType: thumbnailMIMEType, + Width: c.Args.Width, + Height: c.Args.Height, + } + +} diff --git a/pkg/connector/media/transfer.go b/pkg/connector/media/transfer.go index f9493356..7de07ad1 100644 --- a/pkg/connector/media/transfer.go +++ b/pkg/connector/media/transfer.go @@ -13,7 +13,7 @@ import ( "maunium.net/go/mautrix/event" "maunium.net/go/mautrix/id" - "go.mau.fi/util/lottie" + "go.mau.fi/util/gnuzip" "go.mau.fi/mautrix-telegram/pkg/connector/ids" "go.mau.fi/mautrix-telegram/pkg/connector/store" @@ -80,24 +80,6 @@ func getLocationID(loc any) (locID store.TelegramFileLocationID) { return store.TelegramFileLocationID(id) } -type AnimatedStickerConfig struct { - Target string `yaml:"target"` - ConvertFromWebm bool `yaml:"convert_from_webm"` - Args struct { - Width int `yaml:"width"` - Height int `yaml:"height"` - FPS int `yaml:"fps"` - } `yaml:"args"` -} - -func (c AnimatedStickerConfig) TGSConvert() bool { - return c.Target == "gif" || c.Target == "png" -} - -func (c AnimatedStickerConfig) WebmConvert() bool { - return c.ConvertFromWebm && c.Target != "webm" -} - // Transferer is a utility for downloading media from Telegram and uploading it // to Matrix. // TODO better name? @@ -134,14 +116,19 @@ func (t *Transferer) WithFilename(filename string) *Transferer { return t } -func (t *Transferer) WithMIMEType(mimeType string) *Transferer { - t.fileInfo.MimeType = mimeType - return t -} - // WithStickerConfig sets the animated sticker config for the [Transferer]. func (t *Transferer) WithStickerConfig(cfg AnimatedStickerConfig) *Transferer { t.animatedStickerConfig = &cfg + switch cfg.Target { + case "png": + t.fileInfo.MimeType = "image/png" + case "gif": + t.fileInfo.MimeType = "image/gif" + case "webp": + t.fileInfo.MimeType = "image/webp" + case "webm": + t.fileInfo.MimeType = "video/webm" + } return t } @@ -158,6 +145,11 @@ func (t *Transferer) WithVideo(attr *tg.DocumentAttributeVideo) *Transferer { return t } +func (t *Transferer) WithImageSize(attr *tg.DocumentAttributeImageSize) *Transferer { + t.fileInfo.Width, t.fileInfo.Height = attr.W, attr.H + return t +} + // WithDocument transforms a [Transferer] to a [ReadyTransferer] by setting the // given document as the location that will be downloaded by the // [ReadyTransferer]. @@ -173,7 +165,9 @@ func (t *Transferer) WithDocument(doc tg.DocumentClass, thumbnail bool) *ReadyTr documentFileLocation.ThumbSize = largestThumbnail.GetType() } else { t.fileInfo.Size = int(document.Size) - t.fileInfo.MimeType = document.GetMimeType() + if t.fileInfo.MimeType == "" { + t.fileInfo.MimeType = document.GetMimeType() + } } return &ReadyTransferer{t, &documentFileLocation} } @@ -225,6 +219,7 @@ func (t *ReadyTransferer) Transfer(ctx context.Context, store *store.Container, Str("component", "media_transfer"). Str("location_id", string(locationID)). Logger() + ctx = log.WithContext(ctx) if file, err := store.TelegramFile.GetByLocationID(ctx, locationID); err != nil { return "", nil, nil, fmt.Errorf("failed to search for Telegram file by location ID: %w", err) @@ -238,22 +233,26 @@ func (t *ReadyTransferer) Transfer(ctx context.Context, store *store.Container, return "", nil, nil, fmt.Errorf("downloading file failed: %w", err) } - if t.inner.animatedStickerConfig != nil { - if lottie.Supported() && t.inner.animatedStickerConfig.TGSConvert() && t.inner.fileInfo.MimeType == "application/x-tgsticker" { - newData, err := lottie.ConvertBytes(ctx, data, - t.inner.animatedStickerConfig.Target, - t.inner.animatedStickerConfig.Args.Width, - t.inner.animatedStickerConfig.Args.Height, - fmt.Sprintf("%d", t.inner.animatedStickerConfig.Args.FPS)) + if t.inner.animatedStickerConfig != nil && t.inner.fileInfo.MimeType == "application/x-tgsticker" { + converted := t.inner.animatedStickerConfig.convert(ctx, data) + data = converted.Data + t.inner.fileInfo.MimeType = converted.MIMEType + t.inner.fileInfo.Width = converted.Width + t.inner.fileInfo.Height = converted.Height + t.inner.fileInfo.Size = len(data) + + if len(converted.ThumbnailData) > 0 { + thumbnailMXC, thumbnailFileInfo, err := intent.UploadMedia(ctx, t.inner.roomID, converted.ThumbnailData, t.inner.filename, converted.ThumbnailMIMEType) if err != nil { - log.Err(err).Msg("failed to convert animated sticker") + log.Err(err).Msg("failed to upload animated sticker thumbnail to Matrix") } else { - data = newData - t.inner.fileInfo.Size = len(data) - t.inner.fileInfo.MimeType = fmt.Sprintf("image/%s", t.inner.animatedStickerConfig.Target) + t.inner = t.inner.WithThumbnail(thumbnailMXC, thumbnailFileInfo, &event.FileInfo{ + MimeType: converted.ThumbnailMIMEType, + Width: converted.Width, + Height: converted.Height, + Size: len(converted.ThumbnailData), + }) } - // TODO support ffmpeg conversion - // } else if ffmpeg.Supported() && t.Config.WebmConvert() && mimeType == "video/webm" { } } @@ -278,7 +277,7 @@ func (t *ReadyTransferer) Transfer(ctx context.Context, store *store.Container, } // Download downloads the media from Telegram. -func (t *ReadyTransferer) Download(ctx context.Context) (data []byte, fileInfo *event.FileInfo, err error) { +func (t *ReadyTransferer) Download(ctx context.Context) ([]byte, *event.FileInfo, error) { // TODO convert entire function to streaming? Maybe at least stream to file? var buf bytes.Buffer storageFileTypeClass, err := downloader.NewDownloader().Download(t.inner.client, t.loc).Stream(ctx, &buf) @@ -307,7 +306,22 @@ func (t *ReadyTransferer) Download(ctx context.Context) (data []byte, fileInfo * t.inner.fileInfo.MimeType = http.DetectContentType(buf.Bytes()) } } - t.inner.fileInfo.Size = len(data) + t.inner.fileInfo.Size = buf.Len() + + if t.inner.animatedStickerConfig != nil { + detected := http.DetectContentType(buf.Bytes()) + if detected == "application/x-tgsticker" || detected == "application/x-gzip" { + if unzipped, err := gnuzip.MaybeGUnzip(buf.Bytes()); err != nil { + zerolog.Ctx(ctx).Err(err).Msg("failed to unzip animated sticker") + } else { + converted := t.inner.animatedStickerConfig.convert(ctx, unzipped) + t.inner.fileInfo.MimeType = converted.MIMEType + t.inner.fileInfo.Size = len(converted.Data) + return converted.Data, &t.inner.fileInfo, nil + } + } + } + return buf.Bytes(), &t.inner.fileInfo, nil } diff --git a/pkg/connector/msgconv/tomatrix.go b/pkg/connector/msgconv/tomatrix.go index 52deed68..40a7b327 100644 --- a/pkg/connector/msgconv/tomatrix.go +++ b/pkg/connector/msgconv/tomatrix.go @@ -154,8 +154,11 @@ func (mc *MessageConverter) convertMediaRequiringUpload(ctx context.Context, por Str("portal_id", string(portal.ID)). Int("msg_id", msgID). Logger() + eventType := event.EventMessage var partID networkid.PartID var content event.MessageEventContent + var isSticker, isAnimatedSticker bool + extra := map[string]any{} transferer := media.NewTransferer(mc.client.API()).WithRoomID(portal.MXID) var mediaTransferer *media.ReadyTransferer @@ -176,7 +179,81 @@ func (mc *MessageConverter) convertMediaRequiringUpload(ctx context.Context, por partID = networkid.PartID("document") content.MsgType = event.MsgFile - if _, ok := document.GetThumbs(); ok { + extraInfo := map[string]any{} + for _, attr := range document.GetAttributes() { + switch a := attr.(type) { + case *tg.DocumentAttributeFilename: + if content.Body == "" { + content.Body = a.GetFileName() + } else { + content.FileName = a.GetFileName() + } + case *tg.DocumentAttributeVideo: + content.MsgType = event.MsgVideo + transferer = transferer.WithVideo(a) + case *tg.DocumentAttributeAudio: + content.MsgType = event.MsgAudio + content.MSC1767Audio = &event.MSC1767Audio{ + Duration: a.Duration * 1000, + } + if wf, ok := a.GetWaveform(); ok { + for _, v := range waveform.Decode(wf) { + content.MSC1767Audio.Waveform = append(content.MSC1767Audio.Waveform, int(v)<<5) + } + } + if a.Voice { + content.MSC3245Voice = &event.MSC3245Voice{} + } + case *tg.DocumentAttributeImageSize: + transferer = transferer.WithImageSize(a) + case *tg.DocumentAttributeSticker: + isSticker = true + if mc.animatedStickerConfig.Target == "webm" { + content.MsgType = event.MsgVideo + } else { + eventType = event.EventSticker + content.MsgType = "" + } + if content.Body == "" { + content.Body = a.Alt + } else { + content.FileName = content.Body + content.Body = a.Alt + } + stickerInfo := map[string]any{"alt": a.Alt, "id": document.ID} + + if setID, ok := a.Stickerset.(*tg.InputStickerSetID); ok { + stickerInfo["pack"] = map[string]any{ + "id": setID.ID, + "access_hash": setID.AccessHash, + } + } else if shortName, ok := a.Stickerset.(*tg.InputStickerSetShortName); ok { + stickerInfo["pack"] = map[string]any{ + "short_name": shortName.ShortName, + } + } + extraInfo["fi.mau.telegram.sticker"] = stickerInfo + extraInfo["fi.mau.gif"] = true + extraInfo["fi.mau.loop"] = true + extraInfo["fi.mau.autoplay"] = true + extraInfo["fi.mau.hide_controls"] = true + extraInfo["fi.mau.no_audio"] = true + transferer = transferer.WithStickerConfig(mc.animatedStickerConfig) + case *tg.DocumentAttributeAnimated: + isAnimatedSticker = true + } + } + + if isAnimatedSticker || (isSticker && mc.animatedStickerConfig.Target == "webm") { + if isAnimatedSticker { + extraInfo["fi.mau.telegram.gif"] = true + } else { + extraInfo["fi.mau.telegram.animated_sticker"] = true + } + } + extra["info"] = extraInfo + + if _, ok := document.GetThumbs(); ok && eventType != event.EventSticker { var thumbnailURL id.ContentURIString var thumbnailFile *event.EncryptedFileInfo var thumbnailInfo *event.FileInfo @@ -201,29 +278,6 @@ func (mc *MessageConverter) convertMediaRequiringUpload(ctx context.Context, por transferer = transferer.WithThumbnail(thumbnailURL, thumbnailFile, thumbnailInfo) } - for _, attr := range document.GetAttributes() { - switch a := attr.(type) { - case *tg.DocumentAttributeFilename: - content.Body = a.GetFileName() - case *tg.DocumentAttributeVideo: - content.MsgType = event.MsgVideo - transferer = transferer.WithVideo(a) - case *tg.DocumentAttributeAudio: - content.MsgType = event.MsgAudio - content.MSC1767Audio = &event.MSC1767Audio{ - Duration: a.Duration * 1000, - } - if wf, ok := a.GetWaveform(); ok { - for _, v := range waveform.Decode(wf) { - content.MSC1767Audio.Waveform = append(content.MSC1767Audio.Waveform, int(v)<<5) - } - } - if a.Voice { - content.MSC3245Voice = &event.MSC3245Voice{} - } - } - } - mediaTransferer = transferer. WithFilename(content.Body). WithDocument(msgMedia.Document, false) @@ -232,7 +286,7 @@ func (mc *MessageConverter) convertMediaRequiringUpload(ctx context.Context, por } var err error - if mc.useDirectMedia { + if mc.useDirectMedia && (!isSticker || mc.animatedStickerConfig.Target == "disable") { content.URL, content.Info, err = mediaTransferer.DirectDownloadURL(ctx, portal, msgID, false) if err != nil { log.Err(err).Msg("error getting direct download URL for media") @@ -248,15 +302,17 @@ func (mc *MessageConverter) convertMediaRequiringUpload(ctx context.Context, por } } - extra := map[string]any{} - // Handle spoilers // See: https://github.com/matrix-org/matrix-spec-proposals/pull/3725 if s, ok := msgMedia.(spoilable); ok && s.GetSpoiler() { extra["town.robin.msc3725.content_warning"] = map[string]any{ "type": "town.robin.msc3725.spoiler", } - extra["fi.mau.telegram.spoiler"] = true + if extra["info"] == nil { + extra["info"] = map[string]any{} + } + info := extra["info"].(map[string]any) + info["fi.mau.telegram.spoiler"] = true } // Handle disappearing messages @@ -275,7 +331,7 @@ func (mc *MessageConverter) convertMediaRequiringUpload(ctx context.Context, por return &bridgev2.ConvertedMessagePart{ ID: partID, - Type: event.EventMessage, + Type: eventType, Content: &content, Extra: extra, }, disappearingSetting, nil