iartamonov/mautrix-telegram
1
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

media: refactor sticker conversion

Browse Source
This commit is contained in:
Tulir Asokan
2025-12-03 22:15:59 +02:00
parent b7e5078053
commit 2580e28bee
5 changed files with 196 additions and 115 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkg/connector/directdownload.go
+1 -21
View File
@@ -19,7 +19,6 @@ package connector
import (
"context"
"fmt"
"io"
"github.com/rs/zerolog"
"maunium.net/go/mautrix"
@@ -226,26 +225,7 @@ func (tc *TelegramConnector) Download(ctx context.Context, mediaID networkid.Med
WithDocument(customEmojiDocuments[0], false)
}
if readyTransferer == nil {
return nil, fmt.Errorf("invalid combination of direct media keys")
}
r, mimeType, size, err := readyTransferer.Stream(ctx)
if err != nil {
log.Err(err).Msg("failed to download media")
return nil, err
}
log.Debug().
Str("mime_type", mimeType).
Int("size", size).
Msg("Downloaded media successfully")
return &mediaproxy.GetMediaResponseData{
Reader: io.NopCloser(r),
ContentType: mimeType,
ContentLength: int64(size),
}, nil
return readyTransferer.ToDirectMediaResponse(ctx)
}
func (tg *TelegramConnector) SetUseDirectMedia() {
pkg/connector/media/sticker.go
+94 -36
View File
@@ -17,18 +17,13 @@
package media
import (
"bytes"
"context"
"fmt"
"io"
"os"
"path/filepath"
"strconv"
"github.com/rs/zerolog"
"go.mau.fi/util/ffmpeg"
"go.mau.fi/util/lottie"
"go.mau.fi/util/random"
)
type AnimatedStickerConfig struct {
@@ -42,7 +37,8 @@ type AnimatedStickerConfig struct {
}
type ConvertedSticker struct {
DataWriter io.Reader
Success bool
NewPath string
MIMEType string
ThumbnailData []byte
ThumbnailMIMEType string
@@ -51,23 +47,82 @@ type ConvertedSticker struct {
Size int
}
func (c AnimatedStickerConfig) convert(ctx context.Context, data []byte) ConvertedSticker {
input := bytes.NewBuffer(data)
func (c *AnimatedStickerConfig) convertWebm(ctx context.Context, src *os.File) *ConvertedSticker {
if !c.ConvertFromWebm || c.Target == "webm" {
return nil
}
log := zerolog.Ctx(ctx).With().Str("animated_sticker_target", c.Target).Logger()
if !ffmpeg.Supported() {
log.Warn().Msg("Not converting webm sticker as ffmpeg is not installed")
return nil
}
var newPath string
var err error
switch c.Target {
case "png":
newPath, err = ffmpeg.ConvertPath(
ctx, src.Name(), ".png",
[]string{"-ss", "0", "-c:v", "libvpx-vp9"},
[]string{"-frames:v", "1"},
false,
)
case "gif":
newPath, err = ffmpeg.ConvertPath(
ctx, src.Name(), ".gif",
[]string{"-c:v", "libvpx-vp9"},
[]string{"-vf", "split[s0][s1];[s0]palettegen[p];[s1][p]paletteuse"},
false,
)
case "webp":
newPath, err = ffmpeg.ConvertPath(
ctx, src.Name(), ".webp",
[]string{"-c:v", "libvpx-vp9"},
[]string{"-loop", "0"},
false,
)
default:
log.Error().Msg("Unknown target format for webm conversion")
return nil
}
if err != nil {
log.Err(err).Msg("Failed to convert webm sticker")
return nil
}
var outputSize int64
stat, err := os.Stat(newPath)
if err != nil {
log.Err(err).Msg("Failed to stat converted sticker")
} else {
outputSize = stat.Size()
}
_ = src.Close()
return &ConvertedSticker{
Success: true,
NewPath: newPath,
MIMEType: "image/" + c.Target,
Width: c.Args.Width,
Height: c.Args.Height,
Size: int(outputSize),
}
}
func (c *AnimatedStickerConfig) convert(ctx context.Context, src *os.File) *ConvertedSticker {
if c.Target == "disable" {
return ConvertedSticker{DataWriter: input, MIMEType: "video/lottie+json"}
return nil
}
log := zerolog.Ctx(ctx).With().Str("animated_sticker_target", c.Target).Logger()
if !lottie.Supported() {
log.Warn().Msg("lottie not supported, cannot convert animated stickers")
return ConvertedSticker{DataWriter: input, MIMEType: "video/lottie+json"}
log.Warn().Msg("Not converting lottie sticker as lottieconverter is not installed")
return nil
} else if (c.Target == "webp" || c.Target == "webm") && !ffmpeg.Supported() {
log.Warn().Msg("ffmpeg not supported, cannot convert animated stickers")
return ConvertedSticker{DataWriter: input, MIMEType: "video/lottie+json"}
log.Warn().Msg("Not converting lottie sticker as target is webp/webm, but ffmpeg is not installed")
return nil
}
outputFilename := src.Name() + "." + c.Target
dataWriter := new(bytes.Buffer)
var thumbnailData []byte
var mimeType, thumbnailMIMEType string
@@ -75,44 +130,47 @@ func (c AnimatedStickerConfig) convert(ctx context.Context, data []byte) Convert
switch c.Target {
case "png":
mimeType = "image/png"
err = lottie.Convert(ctx, input, "", dataWriter, c.Target, c.Args.Width, c.Args.Height, "1")
err = lottie.Convert(ctx, src, outputFilename, nil, c.Target, c.Args.Width, c.Args.Height, "1")
case "gif":
mimeType = "image/gif"
err = lottie.Convert(ctx, input, "", dataWriter, c.Target, c.Args.Width, c.Args.Height, strconv.Itoa(c.Args.FPS))
err = lottie.Convert(ctx, src, outputFilename, nil, c.Target, c.Args.Width, c.Args.Height, strconv.Itoa(c.Args.FPS))
case "webm", "webp":
tmpFile := filepath.Join(os.TempDir(), fmt.Sprintf("mautrix-telegram-lottieconverter-%s.%s", random.String(10), c.Target))
defer func() {
_ = os.Remove(tmpFile)
}()
thumbnailMIMEType = "image/png"
mimeType = "image/" + c.Target
thumbnailData, err = lottie.FFmpegConvert(ctx, input, tmpFile, c.Args.Width, c.Args.Height, c.Args.FPS)
if c.Target == "webm" {
mimeType = "video/webm"
} else {
mimeType = "image/webp"
}
thumbnailData, err = lottie.FFmpegConvert(ctx, src, outputFilename, c.Args.Width, c.Args.Height, c.Args.FPS)
if err != nil {
break
}
var convertedData []byte
convertedData, err = os.ReadFile(tmpFile)
dataWriter = bytes.NewBuffer(convertedData)
default:
err = fmt.Errorf("unsupported target format %s", c.Target)
log.Error().Msg("Unknown target format")
return nil
}
if err != nil {
log.Err(err).
Str("target", c.Target).
Msg("failed to convert animated sticker to target format")
// Fallback to original data
return ConvertedSticker{DataWriter: input, MIMEType: "video/lottie+json"}
_ = os.Remove(outputFilename)
log.Err(err).Msg("Failed to convert animated sticker")
return nil
}
var outputSize int64
stat, err := os.Stat(outputFilename)
if err != nil {
log.Err(err).Msg("Failed to stat converted sticker")
} else {
outputSize = stat.Size()
}
return ConvertedSticker{
DataWriter: dataWriter,
_ = src.Close()
return &ConvertedSticker{
Success: true,
NewPath: outputFilename,
MIMEType: mimeType,
ThumbnailData: thumbnailData,
ThumbnailMIMEType: thumbnailMIMEType,
Width: c.Args.Width,
Height: c.Args.Height,
Size: dataWriter.Len(),
Size: int(outputSize),
}
}
pkg/connector/media/transfer.go
+98 -55
View File
@@ -21,13 +21,13 @@ import (
"context"
"fmt"
"io"
"net/http"
"os"
"github.com/rs/zerolog"
"go.mau.fi/util/gnuzip"
"maunium.net/go/mautrix/bridgev2"
"maunium.net/go/mautrix/event"
"maunium.net/go/mautrix/id"
"maunium.net/go/mautrix/mediaproxy"
"go.mau.fi/mautrix-telegram/pkg/connector/ids"
"go.mau.fi/mautrix-telegram/pkg/connector/store"
@@ -137,16 +137,6 @@ func (t *Transferer) WithFilename(filename string) *Transferer {
// WithStickerConfig sets the animated sticker config for the [Transferer].
func (t *Transferer) WithStickerConfig(cfg AnimatedStickerConfig) *Transferer {
t.animatedStickerConfig = &cfg
switch cfg.Target {
case "png":
t.fileInfo.MimeType = "image/png"
case "gif":
t.fileInfo.MimeType = "image/gif"
case "webp":
t.fileInfo.MimeType = "image/webp"
case "webm":
t.fileInfo.MimeType = "video/webm"
}
return t
}
@@ -278,43 +268,62 @@ func (t *ReadyTransferer) Transfer(ctx context.Context, store *store.Container,
return "", nil, nil, fmt.Errorf("downloading file failed: %w", err)
}
if t.inner.animatedStickerConfig != nil && t.inner.fileInfo.MimeType == "application/x-tgsticker" {
data, err := io.ReadAll(reader)
if err != nil {
return "", nil, nil, fmt.Errorf("reading sticker data failed: %w", err)
}
converted := t.inner.animatedStickerConfig.convert(ctx, data)
reader = converted.DataWriter
t.inner.fileInfo.MimeType = converted.MIMEType
t.inner.fileInfo.Width = converted.Width
t.inner.fileInfo.Height = converted.Height
t.inner.fileInfo.Size = converted.Size
needStickerConvert := t.inner.animatedStickerConfig != nil && (t.inner.fileInfo.MimeType == "application/x-tgsticker" ||
(t.inner.fileInfo.MimeType == "video/webm" && t.inner.animatedStickerConfig.ConvertFromWebm && t.inner.animatedStickerConfig.Target != "webm"))
if len(converted.ThumbnailData) > 0 {
thumbnailMXC, thumbnailFileInfo, err := intent.UploadMedia(ctx, t.inner.roomID, converted.ThumbnailData, t.inner.filename, converted.ThumbnailMIMEType)
var thumbnailData []byte
var thumbnailMIMEType string
mxc, encryptedFileInfo, err = intent.UploadMediaStream(ctx, t.inner.roomID, int64(t.inner.fileInfo.Size), needStickerConvert, func(file io.Writer) (*bridgev2.FileStreamResult, error) {
_, err := io.Copy(file, reader)
if err != nil {
return nil, fmt.Errorf("failed to stream download: %w", err)
}
var replacementFile string
if needStickerConvert {
osFile := file.(*os.File)
_, err = osFile.Seek(0, io.SeekStart)
if err != nil {
log.Err(err).Msg("failed to upload animated sticker thumbnail to Matrix")
return nil, fmt.Errorf("failed to seek to start of file for sticker conversion: %w", err)
}
var converted *ConvertedSticker
if t.inner.fileInfo.MimeType == "video/webm" {
converted = t.inner.animatedStickerConfig.convertWebm(ctx, osFile)
} else {
t.inner = t.inner.WithThumbnail(thumbnailMXC, thumbnailFileInfo, &event.FileInfo{
MimeType: converted.ThumbnailMIMEType,
Width: converted.Width,
Height: converted.Height,
Size: len(converted.ThumbnailData),
})
t.inner.fileInfo.MimeType = "video/lottie+json"
converted = t.inner.animatedStickerConfig.convert(ctx, osFile)
}
if converted != nil {
replacementFile = converted.NewPath
t.inner.fileInfo.MimeType = converted.MIMEType
t.inner.fileInfo.Width = converted.Width
t.inner.fileInfo.Height = converted.Height
t.inner.fileInfo.Size = converted.Size
thumbnailData = converted.ThumbnailData
thumbnailMIMEType = converted.ThumbnailMIMEType
}
}
}
mxc, encryptedFileInfo, err = intent.UploadMediaStream(ctx, t.inner.roomID, int64(t.inner.fileInfo.Size), false, func(file io.Writer) (*bridgev2.FileStreamResult, error) {
_, err := io.Copy(file, reader)
return &bridgev2.FileStreamResult{
FileName: t.inner.filename,
MimeType: t.inner.fileInfo.MimeType,
FileName: t.inner.filename,
MimeType: t.inner.fileInfo.MimeType,
ReplacementFile: replacementFile,
}, err
})
if err != nil {
return "", nil, nil, fmt.Errorf("failed to upload media to Matrix: %w", err)
}
if thumbnailData != nil {
thumbnailMXC, thumbnailFileInfo, err := intent.UploadMedia(ctx, t.inner.roomID, thumbnailData, t.inner.filename, thumbnailMIMEType)
if err != nil {
log.Err(err).Msg("failed to upload animated sticker thumbnail to Matrix")
} else {
t.inner = t.inner.WithThumbnail(thumbnailMXC, thumbnailFileInfo, &event.FileInfo{
MimeType: thumbnailMIMEType,
Width: t.inner.fileInfo.Width,
Height: t.inner.fileInfo.Height,
Size: len(thumbnailData),
})
}
}
// If it's an unencrypted file, cache the MXC URI corresponding to the
// location ID.
@@ -338,7 +347,7 @@ func (t *ReadyTransferer) Stream(ctx context.Context) (r io.Reader, mimeType str
if err != nil {
return nil, "", 0, err
}
if t.inner.fileInfo.MimeType == "" {
if t.inner.fileInfo.MimeType == "" || t.inner.fileInfo.MimeType == "application/octet-stream" {
switch storageFileTypeClass.(type) {
case *tg.StorageFileJpeg:
t.inner.fileInfo.MimeType = "image/jpeg"
@@ -349,7 +358,7 @@ func (t *ReadyTransferer) Stream(ctx context.Context) (r io.Reader, mimeType str
case *tg.StorageFilePdf:
t.inner.fileInfo.MimeType = "application/pdf"
case *tg.StorageFileMp3:
t.inner.fileInfo.MimeType = "audio/mp3"
t.inner.fileInfo.MimeType = "audio/mpeg"
case *tg.StorageFileMov:
t.inner.fileInfo.MimeType = "video/quicktime"
case *tg.StorageFileMp4:
@@ -361,24 +370,58 @@ func (t *ReadyTransferer) Stream(ctx context.Context) (r io.Reader, mimeType str
}
}
return r, t.inner.fileInfo.MimeType, t.inner.fileInfo.Size, nil
}
func (t *ReadyTransferer) ToDirectMediaResponse(ctx context.Context) (mediaproxy.GetMediaResponse, error) {
if t == nil {
return nil, fmt.Errorf("invalid direct media request")
}
log := zerolog.Ctx(ctx)
r, mimeType, size, err := t.Stream(ctx)
if err != nil {
log.Err(err).Msg("Failed to download media")
return nil, err
}
log.Debug().
Str("mime_type", mimeType).
Int("size", size).
Msg("Started downloading media successfully")
if t.inner.animatedStickerConfig != nil {
data, err := io.ReadAll(r)
if err != nil {
return nil, "", 0, fmt.Errorf("failed to read animated sticker data: %w", err)
} else if detected := http.DetectContentType(data); detected == "application/x-tgsticker" || detected == "application/x-gzip" {
if unzipped, err := gnuzip.MaybeGUnzip(data); err != nil {
zerolog.Ctx(ctx).Err(err).Msg("failed to unzip animated sticker")
} else {
converted := t.inner.animatedStickerConfig.convert(ctx, unzipped)
t.inner.fileInfo.MimeType = converted.MIMEType
t.inner.fileInfo.Size = converted.Size
return converted.DataWriter, t.inner.fileInfo.MimeType, t.inner.fileInfo.Size, nil
}
}
return bytes.NewReader(data), t.inner.fileInfo.MimeType, t.inner.fileInfo.Size, nil
return &mediaproxy.GetMediaResponseFile{
Callback: func(w *os.File) (*mediaproxy.FileMeta, error) {
_, err = io.Copy(w, r)
if err != nil {
return nil, fmt.Errorf("failed to write animated sticker data to file: %w", err)
}
_, err = w.Seek(0, io.SeekStart)
if err != nil {
return nil, fmt.Errorf("failed to seek to start of file for sticker conversion: %w", err)
}
var converted *ConvertedSticker
if t.inner.fileInfo.MimeType == "video/webm" {
converted = t.inner.animatedStickerConfig.convertWebm(ctx, w)
} else {
t.inner.fileInfo.MimeType = "video/lottie+json"
converted = t.inner.animatedStickerConfig.convert(ctx, w)
}
if converted == nil {
return &mediaproxy.FileMeta{ContentType: t.inner.fileInfo.MimeType}, nil
}
return &mediaproxy.FileMeta{
ContentType: converted.MIMEType,
ReplacementFile: converted.NewPath,
}, nil
},
}, nil
}
return r, t.inner.fileInfo.MimeType, t.inner.fileInfo.Size, nil
return &mediaproxy.GetMediaResponseData{
Reader: io.NopCloser(r),
ContentType: mimeType,
ContentLength: int64(size),
}, nil
}
// DownloadBytes downloads the media from Telegram to a byte buffer.