diff --git a/go.mod b/go.mod index ddff177c..63de0f9d 100644 --- a/go.mod +++ b/go.mod @@ -41,7 +41,7 @@ require ( golang.org/x/sync v0.18.0 golang.org/x/tools v0.39.0 gopkg.in/yaml.v3 v3.0.1 - maunium.net/go/mautrix v0.26.1-0.20251202170404-7d54edbfda13 + maunium.net/go/mautrix v0.26.1-0.20251203195941-02ce6ff91851 rsc.io/qr v0.2.0 ) diff --git a/go.sum b/go.sum index abda5d36..aaebb1ad 100644 --- a/go.sum +++ b/go.sum @@ -237,7 +237,7 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= maunium.net/go/mauflag v1.0.0 h1:YiaRc0tEI3toYtJMRIfjP+jklH45uDHtT80nUamyD4M= maunium.net/go/mauflag v1.0.0/go.mod h1:nLivPOpTpHnpzEh8jEdSL9UqO9+/KBJFmNRlwKfkPeA= -maunium.net/go/mautrix v0.26.1-0.20251202170404-7d54edbfda13 h1:zVuKNguVQCC51qDwNr9vnpvuurjZ2IqML2nDBmScx/k= -maunium.net/go/mautrix v0.26.1-0.20251202170404-7d54edbfda13/go.mod h1:NaesYcOQWFDbixVYywCVS+Twlzab9hOUpFNlCBlvciE= +maunium.net/go/mautrix v0.26.1-0.20251203195941-02ce6ff91851 h1:5dty5IkJGxpLj0SQ2+wwKIcrPfZML1uHFcGaQIA9te0= +maunium.net/go/mautrix v0.26.1-0.20251203195941-02ce6ff91851/go.mod h1:NaesYcOQWFDbixVYywCVS+Twlzab9hOUpFNlCBlvciE= rsc.io/qr v0.2.0 h1:6vBLea5/NRMVTz8V66gipeLycZMl/+UlFmk8DvqQ6WY= rsc.io/qr v0.2.0/go.mod h1:IF+uZjkb9fqyeF/4tlBoynqmQxUoPfWEKh921coOuXs= diff --git a/pkg/connector/directdownload.go b/pkg/connector/directdownload.go index 0131da9f..8802fb9a 100644 --- a/pkg/connector/directdownload.go +++ b/pkg/connector/directdownload.go @@ -19,7 +19,6 @@ package connector import ( "context" "fmt" - "io" "github.com/rs/zerolog" "maunium.net/go/mautrix" @@ -226,26 +225,7 @@ func (tc *TelegramConnector) Download(ctx context.Context, mediaID networkid.Med WithDocument(customEmojiDocuments[0], false) } - if readyTransferer == nil { - return nil, fmt.Errorf("invalid combination of direct media keys") - } - - r, mimeType, size, err := readyTransferer.Stream(ctx) - if err != nil { - log.Err(err).Msg("failed to download media") - return nil, err - } - - log.Debug(). - Str("mime_type", mimeType). - Int("size", size). - Msg("Downloaded media successfully") - - return &mediaproxy.GetMediaResponseData{ - Reader: io.NopCloser(r), - ContentType: mimeType, - ContentLength: int64(size), - }, nil + return readyTransferer.ToDirectMediaResponse(ctx) } func (tg *TelegramConnector) SetUseDirectMedia() { diff --git a/pkg/connector/media/sticker.go b/pkg/connector/media/sticker.go index b450f069..d678abc5 100644 --- a/pkg/connector/media/sticker.go +++ b/pkg/connector/media/sticker.go @@ -17,18 +17,13 @@ package media import ( - "bytes" "context" - "fmt" - "io" "os" - "path/filepath" "strconv" "github.com/rs/zerolog" "go.mau.fi/util/ffmpeg" "go.mau.fi/util/lottie" - "go.mau.fi/util/random" ) type AnimatedStickerConfig struct { @@ -42,7 +37,8 @@ type AnimatedStickerConfig struct { } type ConvertedSticker struct { - DataWriter io.Reader + Success bool + NewPath string MIMEType string ThumbnailData []byte ThumbnailMIMEType string @@ -51,23 +47,82 @@ type ConvertedSticker struct { Size int } -func (c AnimatedStickerConfig) convert(ctx context.Context, data []byte) ConvertedSticker { - input := bytes.NewBuffer(data) +func (c *AnimatedStickerConfig) convertWebm(ctx context.Context, src *os.File) *ConvertedSticker { + if !c.ConvertFromWebm || c.Target == "webm" { + return nil + } + log := zerolog.Ctx(ctx).With().Str("animated_sticker_target", c.Target).Logger() + if !ffmpeg.Supported() { + log.Warn().Msg("Not converting webm sticker as ffmpeg is not installed") + return nil + } + var newPath string + var err error + switch c.Target { + case "png": + newPath, err = ffmpeg.ConvertPath( + ctx, src.Name(), ".png", + []string{"-ss", "0", "-c:v", "libvpx-vp9"}, + []string{"-frames:v", "1"}, + false, + ) + case "gif": + newPath, err = ffmpeg.ConvertPath( + ctx, src.Name(), ".gif", + []string{"-c:v", "libvpx-vp9"}, + []string{"-vf", "split[s0][s1];[s0]palettegen[p];[s1][p]paletteuse"}, + false, + ) + case "webp": + newPath, err = ffmpeg.ConvertPath( + ctx, src.Name(), ".webp", + []string{"-c:v", "libvpx-vp9"}, + []string{"-loop", "0"}, + false, + ) + default: + log.Error().Msg("Unknown target format for webm conversion") + return nil + } + if err != nil { + log.Err(err).Msg("Failed to convert webm sticker") + return nil + } + var outputSize int64 + stat, err := os.Stat(newPath) + if err != nil { + log.Err(err).Msg("Failed to stat converted sticker") + } else { + outputSize = stat.Size() + } + + _ = src.Close() + return &ConvertedSticker{ + Success: true, + NewPath: newPath, + MIMEType: "image/" + c.Target, + Width: c.Args.Width, + Height: c.Args.Height, + Size: int(outputSize), + } +} + +func (c *AnimatedStickerConfig) convert(ctx context.Context, src *os.File) *ConvertedSticker { if c.Target == "disable" { - return ConvertedSticker{DataWriter: input, MIMEType: "video/lottie+json"} + return nil } log := zerolog.Ctx(ctx).With().Str("animated_sticker_target", c.Target).Logger() if !lottie.Supported() { - log.Warn().Msg("lottie not supported, cannot convert animated stickers") - return ConvertedSticker{DataWriter: input, MIMEType: "video/lottie+json"} + log.Warn().Msg("Not converting lottie sticker as lottieconverter is not installed") + return nil } else if (c.Target == "webp" || c.Target == "webm") && !ffmpeg.Supported() { - log.Warn().Msg("ffmpeg not supported, cannot convert animated stickers") - return ConvertedSticker{DataWriter: input, MIMEType: "video/lottie+json"} + log.Warn().Msg("Not converting lottie sticker as target is webp/webm, but ffmpeg is not installed") + return nil } + outputFilename := src.Name() + "." + c.Target - dataWriter := new(bytes.Buffer) var thumbnailData []byte var mimeType, thumbnailMIMEType string @@ -75,44 +130,47 @@ func (c AnimatedStickerConfig) convert(ctx context.Context, data []byte) Convert switch c.Target { case "png": mimeType = "image/png" - err = lottie.Convert(ctx, input, "", dataWriter, c.Target, c.Args.Width, c.Args.Height, "1") + err = lottie.Convert(ctx, src, outputFilename, nil, c.Target, c.Args.Width, c.Args.Height, "1") case "gif": mimeType = "image/gif" - err = lottie.Convert(ctx, input, "", dataWriter, c.Target, c.Args.Width, c.Args.Height, strconv.Itoa(c.Args.FPS)) + err = lottie.Convert(ctx, src, outputFilename, nil, c.Target, c.Args.Width, c.Args.Height, strconv.Itoa(c.Args.FPS)) case "webm", "webp": - tmpFile := filepath.Join(os.TempDir(), fmt.Sprintf("mautrix-telegram-lottieconverter-%s.%s", random.String(10), c.Target)) - defer func() { - _ = os.Remove(tmpFile) - }() thumbnailMIMEType = "image/png" - mimeType = "image/" + c.Target - thumbnailData, err = lottie.FFmpegConvert(ctx, input, tmpFile, c.Args.Width, c.Args.Height, c.Args.FPS) + if c.Target == "webm" { + mimeType = "video/webm" + } else { + mimeType = "image/webp" + } + thumbnailData, err = lottie.FFmpegConvert(ctx, src, outputFilename, c.Args.Width, c.Args.Height, c.Args.FPS) if err != nil { break } - var convertedData []byte - convertedData, err = os.ReadFile(tmpFile) - dataWriter = bytes.NewBuffer(convertedData) default: - err = fmt.Errorf("unsupported target format %s", c.Target) + log.Error().Msg("Unknown target format") + return nil } if err != nil { - log.Err(err). - Str("target", c.Target). - Msg("failed to convert animated sticker to target format") - - // Fallback to original data - return ConvertedSticker{DataWriter: input, MIMEType: "video/lottie+json"} + _ = os.Remove(outputFilename) + log.Err(err).Msg("Failed to convert animated sticker") + return nil + } + var outputSize int64 + stat, err := os.Stat(outputFilename) + if err != nil { + log.Err(err).Msg("Failed to stat converted sticker") + } else { + outputSize = stat.Size() } - return ConvertedSticker{ - DataWriter: dataWriter, + _ = src.Close() + return &ConvertedSticker{ + Success: true, + NewPath: outputFilename, MIMEType: mimeType, ThumbnailData: thumbnailData, ThumbnailMIMEType: thumbnailMIMEType, Width: c.Args.Width, Height: c.Args.Height, - Size: dataWriter.Len(), + Size: int(outputSize), } - } diff --git a/pkg/connector/media/transfer.go b/pkg/connector/media/transfer.go index adb53c11..e3048556 100644 --- a/pkg/connector/media/transfer.go +++ b/pkg/connector/media/transfer.go @@ -21,13 +21,13 @@ import ( "context" "fmt" "io" - "net/http" + "os" "github.com/rs/zerolog" - "go.mau.fi/util/gnuzip" "maunium.net/go/mautrix/bridgev2" "maunium.net/go/mautrix/event" "maunium.net/go/mautrix/id" + "maunium.net/go/mautrix/mediaproxy" "go.mau.fi/mautrix-telegram/pkg/connector/ids" "go.mau.fi/mautrix-telegram/pkg/connector/store" @@ -137,16 +137,6 @@ func (t *Transferer) WithFilename(filename string) *Transferer { // WithStickerConfig sets the animated sticker config for the [Transferer]. func (t *Transferer) WithStickerConfig(cfg AnimatedStickerConfig) *Transferer { t.animatedStickerConfig = &cfg - switch cfg.Target { - case "png": - t.fileInfo.MimeType = "image/png" - case "gif": - t.fileInfo.MimeType = "image/gif" - case "webp": - t.fileInfo.MimeType = "image/webp" - case "webm": - t.fileInfo.MimeType = "video/webm" - } return t } @@ -278,43 +268,62 @@ func (t *ReadyTransferer) Transfer(ctx context.Context, store *store.Container, return "", nil, nil, fmt.Errorf("downloading file failed: %w", err) } - if t.inner.animatedStickerConfig != nil && t.inner.fileInfo.MimeType == "application/x-tgsticker" { - data, err := io.ReadAll(reader) - if err != nil { - return "", nil, nil, fmt.Errorf("reading sticker data failed: %w", err) - } - converted := t.inner.animatedStickerConfig.convert(ctx, data) - reader = converted.DataWriter - t.inner.fileInfo.MimeType = converted.MIMEType - t.inner.fileInfo.Width = converted.Width - t.inner.fileInfo.Height = converted.Height - t.inner.fileInfo.Size = converted.Size + needStickerConvert := t.inner.animatedStickerConfig != nil && (t.inner.fileInfo.MimeType == "application/x-tgsticker" || + (t.inner.fileInfo.MimeType == "video/webm" && t.inner.animatedStickerConfig.ConvertFromWebm && t.inner.animatedStickerConfig.Target != "webm")) - if len(converted.ThumbnailData) > 0 { - thumbnailMXC, thumbnailFileInfo, err := intent.UploadMedia(ctx, t.inner.roomID, converted.ThumbnailData, t.inner.filename, converted.ThumbnailMIMEType) + var thumbnailData []byte + var thumbnailMIMEType string + mxc, encryptedFileInfo, err = intent.UploadMediaStream(ctx, t.inner.roomID, int64(t.inner.fileInfo.Size), needStickerConvert, func(file io.Writer) (*bridgev2.FileStreamResult, error) { + _, err := io.Copy(file, reader) + if err != nil { + return nil, fmt.Errorf("failed to stream download: %w", err) + } + var replacementFile string + if needStickerConvert { + osFile := file.(*os.File) + _, err = osFile.Seek(0, io.SeekStart) if err != nil { - log.Err(err).Msg("failed to upload animated sticker thumbnail to Matrix") + return nil, fmt.Errorf("failed to seek to start of file for sticker conversion: %w", err) + } + var converted *ConvertedSticker + if t.inner.fileInfo.MimeType == "video/webm" { + converted = t.inner.animatedStickerConfig.convertWebm(ctx, osFile) } else { - t.inner = t.inner.WithThumbnail(thumbnailMXC, thumbnailFileInfo, &event.FileInfo{ - MimeType: converted.ThumbnailMIMEType, - Width: converted.Width, - Height: converted.Height, - Size: len(converted.ThumbnailData), - }) + t.inner.fileInfo.MimeType = "video/lottie+json" + converted = t.inner.animatedStickerConfig.convert(ctx, osFile) + } + if converted != nil { + replacementFile = converted.NewPath + t.inner.fileInfo.MimeType = converted.MIMEType + t.inner.fileInfo.Width = converted.Width + t.inner.fileInfo.Height = converted.Height + t.inner.fileInfo.Size = converted.Size + thumbnailData = converted.ThumbnailData + thumbnailMIMEType = converted.ThumbnailMIMEType } } - } - - mxc, encryptedFileInfo, err = intent.UploadMediaStream(ctx, t.inner.roomID, int64(t.inner.fileInfo.Size), false, func(file io.Writer) (*bridgev2.FileStreamResult, error) { - _, err := io.Copy(file, reader) return &bridgev2.FileStreamResult{ - FileName: t.inner.filename, - MimeType: t.inner.fileInfo.MimeType, + FileName: t.inner.filename, + MimeType: t.inner.fileInfo.MimeType, + ReplacementFile: replacementFile, }, err }) if err != nil { return "", nil, nil, fmt.Errorf("failed to upload media to Matrix: %w", err) } + if thumbnailData != nil { + thumbnailMXC, thumbnailFileInfo, err := intent.UploadMedia(ctx, t.inner.roomID, thumbnailData, t.inner.filename, thumbnailMIMEType) + if err != nil { + log.Err(err).Msg("failed to upload animated sticker thumbnail to Matrix") + } else { + t.inner = t.inner.WithThumbnail(thumbnailMXC, thumbnailFileInfo, &event.FileInfo{ + MimeType: thumbnailMIMEType, + Width: t.inner.fileInfo.Width, + Height: t.inner.fileInfo.Height, + Size: len(thumbnailData), + }) + } + } // If it's an unencrypted file, cache the MXC URI corresponding to the // location ID. @@ -338,7 +347,7 @@ func (t *ReadyTransferer) Stream(ctx context.Context) (r io.Reader, mimeType str if err != nil { return nil, "", 0, err } - if t.inner.fileInfo.MimeType == "" { + if t.inner.fileInfo.MimeType == "" || t.inner.fileInfo.MimeType == "application/octet-stream" { switch storageFileTypeClass.(type) { case *tg.StorageFileJpeg: t.inner.fileInfo.MimeType = "image/jpeg" @@ -349,7 +358,7 @@ func (t *ReadyTransferer) Stream(ctx context.Context) (r io.Reader, mimeType str case *tg.StorageFilePdf: t.inner.fileInfo.MimeType = "application/pdf" case *tg.StorageFileMp3: - t.inner.fileInfo.MimeType = "audio/mp3" + t.inner.fileInfo.MimeType = "audio/mpeg" case *tg.StorageFileMov: t.inner.fileInfo.MimeType = "video/quicktime" case *tg.StorageFileMp4: @@ -361,24 +370,58 @@ func (t *ReadyTransferer) Stream(ctx context.Context) (r io.Reader, mimeType str } } + return r, t.inner.fileInfo.MimeType, t.inner.fileInfo.Size, nil +} + +func (t *ReadyTransferer) ToDirectMediaResponse(ctx context.Context) (mediaproxy.GetMediaResponse, error) { + if t == nil { + return nil, fmt.Errorf("invalid direct media request") + } + log := zerolog.Ctx(ctx) + r, mimeType, size, err := t.Stream(ctx) + if err != nil { + log.Err(err).Msg("Failed to download media") + return nil, err + } + log.Debug(). + Str("mime_type", mimeType). + Int("size", size). + Msg("Started downloading media successfully") + if t.inner.animatedStickerConfig != nil { - data, err := io.ReadAll(r) - if err != nil { - return nil, "", 0, fmt.Errorf("failed to read animated sticker data: %w", err) - } else if detected := http.DetectContentType(data); detected == "application/x-tgsticker" || detected == "application/x-gzip" { - if unzipped, err := gnuzip.MaybeGUnzip(data); err != nil { - zerolog.Ctx(ctx).Err(err).Msg("failed to unzip animated sticker") - } else { - converted := t.inner.animatedStickerConfig.convert(ctx, unzipped) - t.inner.fileInfo.MimeType = converted.MIMEType - t.inner.fileInfo.Size = converted.Size - return converted.DataWriter, t.inner.fileInfo.MimeType, t.inner.fileInfo.Size, nil - } - } - return bytes.NewReader(data), t.inner.fileInfo.MimeType, t.inner.fileInfo.Size, nil + return &mediaproxy.GetMediaResponseFile{ + Callback: func(w *os.File) (*mediaproxy.FileMeta, error) { + _, err = io.Copy(w, r) + if err != nil { + return nil, fmt.Errorf("failed to write animated sticker data to file: %w", err) + } + _, err = w.Seek(0, io.SeekStart) + if err != nil { + return nil, fmt.Errorf("failed to seek to start of file for sticker conversion: %w", err) + } + var converted *ConvertedSticker + if t.inner.fileInfo.MimeType == "video/webm" { + converted = t.inner.animatedStickerConfig.convertWebm(ctx, w) + } else { + t.inner.fileInfo.MimeType = "video/lottie+json" + converted = t.inner.animatedStickerConfig.convert(ctx, w) + } + if converted == nil { + return &mediaproxy.FileMeta{ContentType: t.inner.fileInfo.MimeType}, nil + } + return &mediaproxy.FileMeta{ + ContentType: converted.MIMEType, + ReplacementFile: converted.NewPath, + }, nil + }, + }, nil } - return r, t.inner.fileInfo.MimeType, t.inner.fileInfo.Size, nil + return &mediaproxy.GetMediaResponseData{ + Reader: io.NopCloser(r), + ContentType: mimeType, + ContentLength: int64(size), + }, nil } // DownloadBytes downloads the media from Telegram to a byte buffer.