iartamonov/mautrix-telegram
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

stickers: support receiving and converting

Browse Source 
Signed-off-by: Sumner Evans <sumner.evans@automattic.com>
This commit is contained in:
Sumner Evans
2024-07-10 00:41:21 -06:00
parent 58cc638058
commit 62d6145c14
7 changed files with 247 additions and 77 deletions
Download Patch File Download Diff File Expand all files Collapse all files
go.mod
+1 -1
View File
@@ -6,7 +6,7 @@ require (
github.com/gotd/td v0.102.0
github.com/rs/zerolog v1.33.0
github.com/stretchr/testify v1.9.0
go.mau.fi/util v0.5.1-0.20240708233020-c2f9af6fecf8
go.mau.fi/util v0.5.1-0.20240710154926-931b33d6d530
go.mau.fi/zerozap v0.1.1
go.uber.org/zap v1.27.0
maunium.net/go/mautrix v0.19.0-beta.1.0.20240706124659-b4057a26c3ed
go.sum
+2 -2
View File
@@ -67,8 +67,8 @@ github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY=
github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28=
github.com/yuin/goldmark v1.7.4 h1:BDXOHExt+A7gwPCJgPIIq7ENvceR7we7rOS9TNoLZeg=
github.com/yuin/goldmark v1.7.4/go.mod h1:uzxRWxtg69N339t3louHJ7+O03ezfj6PlliRlaOzY1E=
go.mau.fi/util v0.5.1-0.20240708233020-c2f9af6fecf8 h1:7ntkhSR0G/dIwAPjcoOoJz+bPne0gA4PypEn8euMMx0=
go.mau.fi/util v0.5.1-0.20240708233020-c2f9af6fecf8/go.mod h1:DsJzUrJAG53lCZnnYvq9/mOyLuPScWwYhvETiTrpdP4=
go.mau.fi/util v0.5.1-0.20240710154926-931b33d6d530 h1:ZWMrLC+Fn2AmKL8HM04YY0zyMDMOagQZVukpxp0rmic=
go.mau.fi/util v0.5.1-0.20240710154926-931b33d6d530/go.mod h1:DsJzUrJAG53lCZnnYvq9/mOyLuPScWwYhvETiTrpdP4=
go.mau.fi/zeroconfig v0.1.2 h1:DKOydWnhPMn65GbXZOafgkPm11BvFashZWLct0dGFto=
go.mau.fi/zeroconfig v0.1.2/go.mod h1:NcSJkf180JT+1IId76PcMuLTNa1CzsFFZ0nBygIQM70=
go.mau.fi/zerozap v0.1.1 h1:mxE/dW4wtkqBYOXOEEzXldk5qKB+ahsZXjoTGnvEhZQ=
pkg/connector/config.go
+9 -5
View File
@@ -3,6 +3,7 @@ package connector
import (
_ "embed"
"fmt"
"slices"
up "go.mau.fi/util/configupgrade"
"maunium.net/go/mautrix/bridgev2"
@@ -25,11 +26,11 @@ var ExampleConfig string
func upgradeConfig(helper up.Helper) {
helper.Copy(up.Int, "app_id")
helper.Copy(up.Str, "app_hash")
helper.Copy(up.Str, "animated_sticker.target")
helper.Copy(up.Bool, "animated_sticker.convert_from_webm")
helper.Copy(up.Int, "animated_sticker.args.width")
helper.Copy(up.Int, "animated_sticker.args.height")
helper.Copy(up.Int, "animated_sticker.args.fps")
helper.Copy(up.Str, "animated_sticker", "target")
helper.Copy(up.Bool, "animated_sticker", "convert_from_webm")
helper.Copy(up.Int, "animated_sticker", "args", "width")
helper.Copy(up.Int, "animated_sticker", "args", "height")
helper.Copy(up.Int, "animated_sticker", "args", "fps")
}
func (tg *TelegramConnector) GetConfig() (example string, data any, upgrader up.Upgrader) {
@@ -43,5 +44,8 @@ func (tg *TelegramConnector) ValidateConfig() error {
if tg.Config.AppHash == "" {
return fmt.Errorf("app_hash is required")
}
if !slices.Contains([]string{"disable", "gif", "png", "webp", "webm"}, tg.Config.AnimatedSticker.Target) {
return fmt.Errorf("unsupported animated sticker target: %s", tg.Config.AnimatedSticker.Target)
}
return nil
}
pkg/connector/directdownload.go
+11
View File
@@ -86,6 +86,16 @@ func (tc *TelegramConnector) Download(ctx context.Context, mediaID networkid.Med
case *tg.MessageMediaPhoto:
readyTransferer = transferer.WithPhoto(msgMedia.Photo)
case *tg.MessageMediaDocument:
document, ok := msgMedia.Document.(*tg.Document)
if !ok {
return nil, fmt.Errorf("unknown document type %T", msgMedia.Document)
}
for _, attr := range document.GetAttributes() {
if attr.TypeID() == tg.DocumentAttributeStickerTypeID {
transferer = transferer.WithStickerConfig(tc.Config.AnimatedSticker)
}
}
readyTransferer = transferer.WithDocument(msgMedia.Document, info.Thumbnail)
default:
return nil, fmt.Errorf("unhandled media type %T", msgMedia)
@@ -93,6 +103,7 @@ func (tc *TelegramConnector) Download(ctx context.Context, mediaID networkid.Med
data, fileInfo, err := readyTransferer.Download(ctx)
if err != nil {
log.Err(err).Msg("failed to download media")
return nil, err
}
pkg/connector/media/sticker.go
+85
View File
@@ -0,0 +1,85 @@
package media
import (
"bytes"
"context"
"fmt"
"strconv"
"github.com/rs/zerolog"
"go.mau.fi/util/ffmpeg"
"go.mau.fi/util/lottie"
)
type AnimatedStickerConfig struct {
Target string `yaml:"target"`
ConvertFromWebm bool `yaml:"convert_from_webm"`
Args struct {
Width int `yaml:"width"`
Height int `yaml:"height"`
FPS int `yaml:"fps"`
} `yaml:"args"`
}
type ConvertedSticker struct {
Data []byte
MIMEType string
ThumbnailData []byte
ThumbnailMIMEType string
Width int
Height int
}
func (c AnimatedStickerConfig) convert(ctx context.Context, data []byte) ConvertedSticker {
if c.Target == "disable" {
return ConvertedSticker{Data: data, MIMEType: "application/x-tgsticker"}
}
log := zerolog.Ctx(ctx).With().Str("animated_sticker_target", c.Target).Logger()
if !lottie.Supported() {
log.Warn().Msg("lottie not supported, cannot convert animated stickers")
return ConvertedSticker{Data: data, MIMEType: "application/x-tgsticker"}
} else if (c.Target == "webp" || c.Target == "webm") && !ffmpeg.Supported() {
log.Warn().Msg("ffmpeg not supported, cannot convert animated stickers")
return ConvertedSticker{Data: data, MIMEType: "application/x-tgsticker"}
}
input := bytes.NewBuffer(data)
outputWriter := new(bytes.Buffer)
var thumbnailData []byte
var mimeType, thumbnailMIMEType string
var err error
switch c.Target {
case "png":
mimeType = "image/png"
err = lottie.Convert(ctx, input, "", outputWriter, c.Target, c.Args.Width, c.Args.Height, "1")
case "gif":
mimeType = "image/gif"
err = lottie.Convert(ctx, input, "", outputWriter, c.Target, c.Args.Width, c.Args.Height, strconv.Itoa(c.Args.FPS))
case "webm", "webp":
thumbnailMIMEType = "image/png"
outputWriter, mimeType, thumbnailData, err = lottie.FfmpegConvert(ctx, input, c.Target, c.Args.Width, c.Args.Height, c.Args.FPS)
default:
err = fmt.Errorf("unsupported target format %s", c.Target)
}
if err != nil {
log.Err(err).
Str("target", c.Target).
Msg("failed to convert animated sticker to target format")
// Fallback to original data
return ConvertedSticker{Data: data, MIMEType: "application/x-tgsticker"}
}
return ConvertedSticker{
Data: outputWriter.Bytes(),
MIMEType: mimeType,
ThumbnailData: thumbnailData,
ThumbnailMIMEType: thumbnailMIMEType,
Width: c.Args.Width,
Height: c.Args.Height,
}
}
pkg/connector/media/transfer.go
+54 -40
View File
@@ -13,7 +13,7 @@ import (
"maunium.net/go/mautrix/event"
"maunium.net/go/mautrix/id"
"go.mau.fi/util/lottie"
"go.mau.fi/util/gnuzip"
"go.mau.fi/mautrix-telegram/pkg/connector/ids"
"go.mau.fi/mautrix-telegram/pkg/connector/store"
@@ -80,24 +80,6 @@ func getLocationID(loc any) (locID store.TelegramFileLocationID) {
return store.TelegramFileLocationID(id)
}
type AnimatedStickerConfig struct {
Target string `yaml:"target"`
ConvertFromWebm bool `yaml:"convert_from_webm"`
Args struct {
Width int `yaml:"width"`
Height int `yaml:"height"`
FPS int `yaml:"fps"`
} `yaml:"args"`
}
func (c AnimatedStickerConfig) TGSConvert() bool {
return c.Target == "gif" || c.Target == "png"
}
func (c AnimatedStickerConfig) WebmConvert() bool {
return c.ConvertFromWebm && c.Target != "webm"
}
// Transferer is a utility for downloading media from Telegram and uploading it
// to Matrix.
// TODO better name?
@@ -134,14 +116,19 @@ func (t *Transferer) WithFilename(filename string) *Transferer {
return t
}
func (t *Transferer) WithMIMEType(mimeType string) *Transferer {
t.fileInfo.MimeType = mimeType
return t
}
// WithStickerConfig sets the animated sticker config for the [Transferer].
func (t *Transferer) WithStickerConfig(cfg AnimatedStickerConfig) *Transferer {
t.animatedStickerConfig = &cfg
switch cfg.Target {
case "png":
t.fileInfo.MimeType = "image/png"
case "gif":
t.fileInfo.MimeType = "image/gif"
case "webp":
t.fileInfo.MimeType = "image/webp"
case "webm":
t.fileInfo.MimeType = "video/webm"
}
return t
}
@@ -158,6 +145,11 @@ func (t *Transferer) WithVideo(attr *tg.DocumentAttributeVideo) *Transferer {
return t
}
func (t *Transferer) WithImageSize(attr *tg.DocumentAttributeImageSize) *Transferer {
t.fileInfo.Width, t.fileInfo.Height = attr.W, attr.H
return t
}
// WithDocument transforms a [Transferer] to a [ReadyTransferer] by setting the
// given document as the location that will be downloaded by the
// [ReadyTransferer].
@@ -173,7 +165,9 @@ func (t *Transferer) WithDocument(doc tg.DocumentClass, thumbnail bool) *ReadyTr
documentFileLocation.ThumbSize = largestThumbnail.GetType()
} else {
t.fileInfo.Size = int(document.Size)
t.fileInfo.MimeType = document.GetMimeType()
if t.fileInfo.MimeType == "" {
t.fileInfo.MimeType = document.GetMimeType()
}
}
return &ReadyTransferer{t, &documentFileLocation}
}
@@ -225,6 +219,7 @@ func (t *ReadyTransferer) Transfer(ctx context.Context, store *store.Container,
Str("component", "media_transfer").
Str("location_id", string(locationID)).
Logger()
ctx = log.WithContext(ctx)
if file, err := store.TelegramFile.GetByLocationID(ctx, locationID); err != nil {
return "", nil, nil, fmt.Errorf("failed to search for Telegram file by location ID: %w", err)
@@ -238,22 +233,26 @@ func (t *ReadyTransferer) Transfer(ctx context.Context, store *store.Container,
return "", nil, nil, fmt.Errorf("downloading file failed: %w", err)
}
if t.inner.animatedStickerConfig != nil {
if lottie.Supported() && t.inner.animatedStickerConfig.TGSConvert() && t.inner.fileInfo.MimeType == "application/x-tgsticker" {
newData, err := lottie.ConvertBytes(ctx, data,
t.inner.animatedStickerConfig.Target,
t.inner.animatedStickerConfig.Args.Width,
t.inner.animatedStickerConfig.Args.Height,
fmt.Sprintf("%d", t.inner.animatedStickerConfig.Args.FPS))
if t.inner.animatedStickerConfig != nil && t.inner.fileInfo.MimeType == "application/x-tgsticker" {
converted := t.inner.animatedStickerConfig.convert(ctx, data)
data = converted.Data
t.inner.fileInfo.MimeType = converted.MIMEType
t.inner.fileInfo.Width = converted.Width
t.inner.fileInfo.Height = converted.Height
t.inner.fileInfo.Size = len(data)
if len(converted.ThumbnailData) > 0 {
thumbnailMXC, thumbnailFileInfo, err := intent.UploadMedia(ctx, t.inner.roomID, converted.ThumbnailData, t.inner.filename, converted.ThumbnailMIMEType)
if err != nil {
log.Err(err).Msg("failed to convert animated sticker")
log.Err(err).Msg("failed to upload animated sticker thumbnail to Matrix")
} else {
data = newData
t.inner.fileInfo.Size = len(data)
t.inner.fileInfo.MimeType = fmt.Sprintf("image/%s", t.inner.animatedStickerConfig.Target)
t.inner = t.inner.WithThumbnail(thumbnailMXC, thumbnailFileInfo, &event.FileInfo{
MimeType: converted.ThumbnailMIMEType,
Width: converted.Width,
Height: converted.Height,
Size: len(converted.ThumbnailData),
})
}
// TODO support ffmpeg conversion
// } else if ffmpeg.Supported() && t.Config.WebmConvert() && mimeType == "video/webm" {
}
}
@@ -278,7 +277,7 @@ func (t *ReadyTransferer) Transfer(ctx context.Context, store *store.Container,
}
// Download downloads the media from Telegram.
func (t *ReadyTransferer) Download(ctx context.Context) (data []byte, fileInfo *event.FileInfo, err error) {
func (t *ReadyTransferer) Download(ctx context.Context) ([]byte, *event.FileInfo, error) {
// TODO convert entire function to streaming? Maybe at least stream to file?
var buf bytes.Buffer
storageFileTypeClass, err := downloader.NewDownloader().Download(t.inner.client, t.loc).Stream(ctx, &buf)
@@ -307,7 +306,22 @@ func (t *ReadyTransferer) Download(ctx context.Context) (data []byte, fileInfo *
t.inner.fileInfo.MimeType = http.DetectContentType(buf.Bytes())
}
}
t.inner.fileInfo.Size = len(data)
t.inner.fileInfo.Size = buf.Len()
if t.inner.animatedStickerConfig != nil {
detected := http.DetectContentType(buf.Bytes())
if detected == "application/x-tgsticker" || detected == "application/x-gzip" {
if unzipped, err := gnuzip.MaybeGUnzip(buf.Bytes()); err != nil {
zerolog.Ctx(ctx).Err(err).Msg("failed to unzip animated sticker")
} else {
converted := t.inner.animatedStickerConfig.convert(ctx, unzipped)
t.inner.fileInfo.MimeType = converted.MIMEType
t.inner.fileInfo.Size = len(converted.Data)
return converted.Data, &t.inner.fileInfo, nil
}
}
}
return buf.Bytes(), &t.inner.fileInfo, nil
}
pkg/connector/msgconv/tomatrix.go
+85 -29
View File
@@ -154,8 +154,11 @@ func (mc *MessageConverter) convertMediaRequiringUpload(ctx context.Context, por
Str("portal_id", string(portal.ID)).
Int("msg_id", msgID).
Logger()
eventType := event.EventMessage
var partID networkid.PartID
var content event.MessageEventContent
var isSticker, isAnimatedSticker bool
extra := map[string]any{}
transferer := media.NewTransferer(mc.client.API()).WithRoomID(portal.MXID)
var mediaTransferer *media.ReadyTransferer
@@ -176,7 +179,81 @@ func (mc *MessageConverter) convertMediaRequiringUpload(ctx context.Context, por
partID = networkid.PartID("document")
content.MsgType = event.MsgFile
if _, ok := document.GetThumbs(); ok {
extraInfo := map[string]any{}
for _, attr := range document.GetAttributes() {
switch a := attr.(type) {
case *tg.DocumentAttributeFilename:
if content.Body == "" {
content.Body = a.GetFileName()
} else {
content.FileName = a.GetFileName()
}
case *tg.DocumentAttributeVideo:
content.MsgType = event.MsgVideo
transferer = transferer.WithVideo(a)
case *tg.DocumentAttributeAudio:
content.MsgType = event.MsgAudio
content.MSC1767Audio = &event.MSC1767Audio{
Duration: a.Duration * 1000,
}
if wf, ok := a.GetWaveform(); ok {
for _, v := range waveform.Decode(wf) {
content.MSC1767Audio.Waveform = append(content.MSC1767Audio.Waveform, int(v)<<5)
}
}
if a.Voice {
content.MSC3245Voice = &event.MSC3245Voice{}
}
case *tg.DocumentAttributeImageSize:
transferer = transferer.WithImageSize(a)
case *tg.DocumentAttributeSticker:
isSticker = true
if mc.animatedStickerConfig.Target == "webm" {
content.MsgType = event.MsgVideo
} else {
eventType = event.EventSticker
content.MsgType = ""
}
if content.Body == "" {
content.Body = a.Alt
} else {
content.FileName = content.Body
content.Body = a.Alt
}
stickerInfo := map[string]any{"alt": a.Alt, "id": document.ID}
if setID, ok := a.Stickerset.(*tg.InputStickerSetID); ok {
stickerInfo["pack"] = map[string]any{
"id": setID.ID,
"access_hash": setID.AccessHash,
}
} else if shortName, ok := a.Stickerset.(*tg.InputStickerSetShortName); ok {
stickerInfo["pack"] = map[string]any{
"short_name": shortName.ShortName,
}
}
extraInfo["fi.mau.telegram.sticker"] = stickerInfo
extraInfo["fi.mau.gif"] = true
extraInfo["fi.mau.loop"] = true
extraInfo["fi.mau.autoplay"] = true
extraInfo["fi.mau.hide_controls"] = true
extraInfo["fi.mau.no_audio"] = true
transferer = transferer.WithStickerConfig(mc.animatedStickerConfig)
case *tg.DocumentAttributeAnimated:
isAnimatedSticker = true
}
}
if isAnimatedSticker || (isSticker && mc.animatedStickerConfig.Target == "webm") {
if isAnimatedSticker {
extraInfo["fi.mau.telegram.gif"] = true
} else {
extraInfo["fi.mau.telegram.animated_sticker"] = true
}
}
extra["info"] = extraInfo
if _, ok := document.GetThumbs(); ok && eventType != event.EventSticker {
var thumbnailURL id.ContentURIString
var thumbnailFile *event.EncryptedFileInfo
var thumbnailInfo *event.FileInfo
@@ -201,29 +278,6 @@ func (mc *MessageConverter) convertMediaRequiringUpload(ctx context.Context, por
transferer = transferer.WithThumbnail(thumbnailURL, thumbnailFile, thumbnailInfo)
}
for _, attr := range document.GetAttributes() {
switch a := attr.(type) {
case *tg.DocumentAttributeFilename:
content.Body = a.GetFileName()
case *tg.DocumentAttributeVideo:
content.MsgType = event.MsgVideo
transferer = transferer.WithVideo(a)
case *tg.DocumentAttributeAudio:
content.MsgType = event.MsgAudio
content.MSC1767Audio = &event.MSC1767Audio{
Duration: a.Duration * 1000,
}
if wf, ok := a.GetWaveform(); ok {
for _, v := range waveform.Decode(wf) {
content.MSC1767Audio.Waveform = append(content.MSC1767Audio.Waveform, int(v)<<5)
}
}
if a.Voice {
content.MSC3245Voice = &event.MSC3245Voice{}
}
}
}
mediaTransferer = transferer.
WithFilename(content.Body).
WithDocument(msgMedia.Document, false)
@@ -232,7 +286,7 @@ func (mc *MessageConverter) convertMediaRequiringUpload(ctx context.Context, por
}
var err error
if mc.useDirectMedia {
if mc.useDirectMedia && (!isSticker || mc.animatedStickerConfig.Target == "disable") {
content.URL, content.Info, err = mediaTransferer.DirectDownloadURL(ctx, portal, msgID, false)
if err != nil {
log.Err(err).Msg("error getting direct download URL for media")
@@ -248,15 +302,17 @@ func (mc *MessageConverter) convertMediaRequiringUpload(ctx context.Context, por
}
}
extra := map[string]any{}
// Handle spoilers
// See: https://github.com/matrix-org/matrix-spec-proposals/pull/3725
if s, ok := msgMedia.(spoilable); ok && s.GetSpoiler() {
extra["town.robin.msc3725.content_warning"] = map[string]any{
"type": "town.robin.msc3725.spoiler",
}
extra["fi.mau.telegram.spoiler"] = true
if extra["info"] == nil {
extra["info"] = map[string]any{}
}
info := extra["info"].(map[string]any)
info["fi.mau.telegram.spoiler"] = true
}
// Handle disappearing messages
@@ -275,7 +331,7 @@ func (mc *MessageConverter) convertMediaRequiringUpload(ctx context.Context, por
return &bridgev2.ConvertedMessagePart{
ID: partID,
Type: event.EventMessage,
Type: eventType,
Content: &content,
Extra: extra,
}, disappearingSetting, nil