From f430ed7169e632ac6dce8d5d34ac1952031acc3a Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Thu, 3 Oct 2019 01:28:47 +0300 Subject: [PATCH] Remove slow python converters and use asyncio subprocess --- example-config.yaml | 20 +-- mautrix_telegram/config.py | 3 +- mautrix_telegram/portal/telegram.py | 3 +- mautrix_telegram/util/file_transfer.py | 15 +-- mautrix_telegram/util/lottie2ffmpeg.sh | 26 ++++ mautrix_telegram/util/tgs_converter.py | 165 +++++++------------------ 6 files changed, 95 insertions(+), 137 deletions(-) create mode 100755 mautrix_telegram/util/lottie2ffmpeg.sh diff --git a/example-config.yaml b/example-config.yaml index fbca058b..2cba1761 100644 --- a/example-config.yaml +++ b/example-config.yaml @@ -163,16 +163,22 @@ bridge: image_as_file_size: 10 # Maximum size of Telegram documents in megabytes to bridge. max_document_size: 100 - # Format, animated sticker convert to. - # Supported values: - # png - converts to png (fastest and preferred), - # gif - converts to gif animation, requires PIL, slow, sometimes loses transparency, - # gifc - uses same utility as png, faster but without transparency at all - # mp4 - video in mp4 container (ffmpeg binary and takes a lot of time, but less than gif) - animated_sticker_target_type: gif # Whether or not created rooms should have federation enabled. # If false, created portal rooms will never be federated. federate_rooms: true + # Settings for converting animated stickers. + animated_sticker: + # Format to which animated stickers should be converted. + # disable - No conversion, send as-is (gzipped lottie) + # png - converts to non-animated png (fastest), + # gif - converts to animated gif, but loses transparency + target: gif + # Arguments for converter. All converters take width and height. + # GIF converter takes background as a hex color. + args: + width: 256 + height: 256 + background: "ffffff" # Whether to bridge Telegram bot messages as m.notices or m.texts. bot_messages_as_notices: true diff --git a/mautrix_telegram/config.py b/mautrix_telegram/config.py index 7361bfbf..9d3cd37c 100644 --- a/mautrix_telegram/config.py +++ b/mautrix_telegram/config.py @@ -101,8 +101,9 @@ class Config(BaseBridgeConfig): copy("bridge.inline_images") copy("bridge.image_as_file_size") copy("bridge.max_document_size") - copy("bridge.animated_sticker_target_type") copy("bridge.federate_rooms") + copy("bridge.animated_sticker.target") + copy("bridge.animated_sticker.args") copy("bridge.bot_messages_as_notices") if isinstance(self["bridge.bridge_notices"], bool): diff --git a/mautrix_telegram/portal/telegram.py b/mautrix_telegram/portal/telegram.py index 6cb860e0..3aa97b3a 100644 --- a/mautrix_telegram/portal/telegram.py +++ b/mautrix_telegram/portal/telegram.py @@ -183,8 +183,7 @@ class PortalTelegram(BasePortal, ABC): thumb_size = None file = await util.transfer_file_to_matrix(source.client, intent, document, thumb_loc, is_sticker=attrs.is_sticker, - tgs_convert_type= - config["bridge.animated_sticker_target_type"]) + tgs_convert=config["bridge.animated_sticker"]) if not file: return None diff --git a/mautrix_telegram/util/file_transfer.py b/mautrix_telegram/util/file_transfer.py index ead8d9d6..bcc638ce 100644 --- a/mautrix_telegram/util/file_transfer.py +++ b/mautrix_telegram/util/file_transfer.py @@ -161,8 +161,8 @@ TypeThumbnail = Optional[Union[TypeLocation, TypePhotoSize]] async def transfer_file_to_matrix(client: MautrixTelegramClient, intent: IntentAPI, location: TypeLocation, thumbnail: TypeThumbnail = None, - is_sticker: bool = False, tgs_convert_type: str = "png") \ - -> Optional[DBTelegramFile]: + is_sticker: bool = False, tgs_convert: Optional[dict] = None + ) -> Optional[DBTelegramFile]: location_id = _location_to_id(location) if not location_id: return None @@ -178,13 +178,13 @@ async def transfer_file_to_matrix(client: MautrixTelegramClient, intent: IntentA transfer_locks[location_id] = lock async with lock: return await _unlocked_transfer_file_to_matrix(client, intent, location_id, location, - thumbnail, is_sticker, tgs_convert_type) + thumbnail, is_sticker, tgs_convert) async def _unlocked_transfer_file_to_matrix(client: MautrixTelegramClient, intent: IntentAPI, loc_id: str, location: TypeLocation, thumbnail: TypeThumbnail, is_sticker: bool, - tgs_convert_type: str + tgs_convert: Optional[dict] ) -> Optional[DBTelegramFile]: db_file = DBTelegramFile.get(loc_id) if db_file: @@ -202,9 +202,10 @@ async def _unlocked_transfer_file_to_matrix(client: MautrixTelegramClient, inten mime_type = magic.from_buffer(file, mime=True) image_converted = False - if mime_type == "application/gzip" and is_sticker: - mime_type, file, width, height, thumbnail = convert_tgs_to(file, tgs_convert_type, 256, 256) - image_converted = width is not None + if mime_type == "application/gzip" and is_sticker and tgs_convert: + mime_type, file, width, height, thumbnail = await convert_tgs_to( + file, tgs_convert["target"], **tgs_convert["args"]) + image_converted = mime_type != "application/gzip" if mime_type == "image/webp": new_mime_type, file, width, height = convert_image( diff --git a/mautrix_telegram/util/lottie2ffmpeg.sh b/mautrix_telegram/util/lottie2ffmpeg.sh new file mode 100755 index 00000000..fd9a36b5 --- /dev/null +++ b/mautrix_telegram/util/lottie2ffmpeg.sh @@ -0,0 +1,26 @@ +#!/bin/bash +TMPDIR=$(mktemp -d) + +if [ ! -e $TMPDIR ]; then + >&2 echo "Failed to create temp directory" + exit 1 +fi + +trap "exit 1" HUP INT PIPE QUIT TERM +trap 'rm -rf "$TMPDIR"' EXIT + +cd $TMPDIR + +lottieconverter=$1 +resolution=$2 + +cat > input + +for i in {0..99}; do + padded="0$i" + $lottieconverter input frame-${padded: -2}.png png $resolution $((i+1)) +done + +ffmpeg -start_number 0 -framerate 30 -i frame-%02d.png -c:v libvpx-vp9 -pix_fmt yuva420p out.webm + +cat out.webm diff --git a/mautrix_telegram/util/tgs_converter.py b/mautrix_telegram/util/tgs_converter.py index 8400ff21..3e3106d1 100644 --- a/mautrix_telegram/util/tgs_converter.py +++ b/mautrix_telegram/util/tgs_converter.py @@ -14,137 +14,62 @@ # # You should have received a copy of the GNU Affero General Public License # along with this program. If not, see . +from typing import Dict, Callable, Awaitable, Optional, Tuple, Any +import asyncio.subprocess import logging -from io import BytesIO -from typing import Optional, Tuple +import shutil +import os.path + +log: logging.Logger = logging.getLogger("mau.util.tgs") +converters: Dict[str, Callable[[bytes, int, int, Any], Awaitable[Tuple[str, bytes]]]] = {} -LOG: logging.Logger = logging.getLogger("mau.util.tgs") +lottieconverter = os.path.abspath(shutil.which("lottieconverter")) +lottie2ffmpeg = os.path.abspath(shutil.which("lottie2ffmpeg.sh")) -try: - import gzip - import subprocess - - proc = subprocess.Popen(["lottieconverter"], stdout=subprocess.PIPE, stderr=subprocess.PIPE) - _, err = proc.communicate() - if err is not None and not err.decode("utf-8").startswith("Usage"): - raise ImportError(err) +if lottieconverter: + async def tgs_to_png(file: bytes, width: int, height: int, **_: Any) -> Tuple[str, bytes]: + frame = 1 + proc = await asyncio.create_subprocess_exec(lottieconverter, "-", "-", "png", + f"{width}x{height}", str(frame), + stdout=asyncio.subprocess.PIPE, + stdin=asyncio.subprocess.PIPE) + stdout, _ = await proc.communicate(file) + return "image/png", stdout - def _tgs_to_png(file: bytes, width: int, - height: int, frame: int = None) -> Tuple[bytes, Optional[bytes]]: - if not frame: - frame = 1 - p = subprocess.run(["lottieconverter", "-", "-", "png", - str.format(f"{width}x{height}"), str(frame)], stdout=subprocess.PIPE, - input=file, universal_newlines=False) - return p.stdout, None + async def tgs_to_gif(file: bytes, width: int, height: int, background: str = "202020", + **_: Any) -> Tuple[str, bytes]: + proc = await asyncio.create_subprocess_exec(lottieconverter, "-", "-", "gif", + f"{width}x{height}", "0", f"0x{background}", + stdout=asyncio.subprocess.PIPE, + stdin=asyncio.subprocess.PIPE) + stdout, _ = await proc.communicate(file) + return "image/gif", stdout - TGS_CONVERTERS = {"png": _tgs_to_png} + converters["png"] = tgs_to_png + converters["gif"] = tgs_to_gif - def _tgs_to_gif(file: bytes, width: int, height: int) -> Tuple[bytes, Optional[bytes]]: - p = subprocess.run(["lottieconverter", "-", "-", "gif", - str.format(f"{width}x{height}"), "0", "0x202020"], - stdout=subprocess.PIPE, - input=file, universal_newlines=False) - return p.stdout, None - - TGS_CONVERTERS.update({"gifc": _tgs_to_gif}) - - try: - from PIL import Image - - def _tgs_to_gif(file: bytes, width: int, height: int) \ - -> Tuple[bytes, Optional[bytes]]: - frames = [] - first_frame = None - for i in range(1, 100): - frame, _ = _tgs_to_png(file, width, height, i) - if not first_frame: - first_frame = frame - image = Image.open(BytesIO(frame)) - if image.mode not in ["RGBA", "RGBa"]: - image = image.convert("RGBA") - alpha = image.getchannel("A") - image = image.convert('P', palette=Image.ADAPTIVE, colors=255) - mask = Image.eval(alpha, lambda a: 255 if a <= 128 else 0) - image.paste(255, mask) - frames.append(image) - - duration = 100 - fo = BytesIO() - frames[0].save( - fo, - format='GIF', - append_images=frames[1:], - save_all=True, - duration=duration, - loop=0, - transparency=255, - disposal=2, - ) - return fo.getvalue(), first_frame - - TGS_CONVERTERS.update({"gif": _tgs_to_gif}) - except ImportError: - LOG.warn("Unable to create tgs to gif converter, install PIL") - - try: - import cv2 - import numpy - import tempfile - import os - - def _tgs_to_video(file: bytes, width: int, height: int) \ - -> Tuple[bytes, Optional[bytes]]: - with tempfile.NamedTemporaryFile(mode="r+b", suffix=".mp4") as tmp: - video_tmp_file = tmp.name - video = None - first_frame = None - try: - video = cv2.VideoWriter(filename=video_tmp_file, apiPreference=cv2.CAP_ANY, - fourcc=cv2.VideoWriter_fourcc(*'vp09'), - fps=10, - frameSize=(width, height)) - - for i in range(1, 100): - frame, _ = _tgs_to_png(file, width, height, i) - if not first_frame: - first_frame = frame - video.write(cv2.cvtColor(numpy.array(Image.open(BytesIO(frame))), - cv2.COLOR_RGB2BGR)) - - finally: - if video: - video.release() - with open(video_tmp_file, "rb") as video_file: - out = video_file.read() - os.remove(video_tmp_file) - return out, first_frame - """ - It seems, that riot don't wont to play converted videos... - """ - TGS_CONVERTERS.update({"mp4": _tgs_to_video}) - except ImportError: - LOG.warn("Unable to create tgs to video converter, " - "install PIL, numpy and opencv-python-headless") - -except (ImportError, OSError): - LOG.exception("Unable to init tgs converters, possibly missing lottieconverter") - TGS_CONVERTERS = {} +if lottieconverter and lottie2ffmpeg: + async def tgs_to_webm(file: bytes, width: int, height: int, **_: Any) -> Tuple[str, bytes]: + proc = await asyncio.create_subprocess_exec(lottie2ffmpeg, lottieconverter, + f"{width}x{height}", + stdout=asyncio.subprocess.PIPE, + stdin=asyncio.subprocess.PIPE) + stdout, _ = await proc.communicate(file) + return "video/webm", stdout -TYPE_TO_MIME = {"png": "image/png", "gif": "image/gif", "gifc": "image/gif", "mp4": "video/mp4"} + converters["webm"] = tgs_to_webm -def convert_tgs_to(file: bytes, convert_to: str, width: int = 200, height: int = 200) \ - -> Tuple[str, bytes, Optional[int], Optional[int], Optional[bytes]]: - if convert_to in TGS_CONVERTERS: - mime = TYPE_TO_MIME[convert_to] - converter = TGS_CONVERTERS[convert_to] - out, preview = converter(file, width, height) - return mime, out, width, height, preview - else: - LOG.warning(f"Unable to convert animated sticker, type {convert_to} not supported") +async def convert_tgs_to(file: bytes, convert_to: str, width: int, height: int, **kwargs: Any + ) -> Tuple[str, bytes, Optional[int], Optional[int], Optional[bytes]]: + if convert_to in converters: + converter = converters[convert_to] + mime, out = await converter(file, width, height, **kwargs) + return mime, out, width, height, None + elif convert_to != "disable": + log.warning(f"Unable to convert animated sticker, type {convert_to} not supported") return "application/gzip", file, None, None, None