`")
+ try:
+ await evt.sender.ensure_started(even_if_no_session=True)
+ first_name, last_name = evt.sender.command_status["full_name"]
+ user = await evt.sender.client.sign_up(evt.args[0], first_name, last_name)
+ asyncio.ensure_future(evt.sender.post_login(user), loop=evt.loop)
+ evt.sender.command_status = None
+ return await evt.reply(f"Successfully registered to Telegram.")
+ except PhoneNumberOccupiedError:
+ return await evt.reply("That phone number has already been registered. "
+ "You can log in with `$cmdprefix+sp login`.")
+ except FirstNameInvalidError:
+ return await evt.reply("Invalid name. Please set a Matrix displayname before registering.")
+ except PhoneCodeExpiredError:
+ return await evt.reply(
+ "Phone code expired. Try again with `$cmdprefix+sp register `.")
+ except PhoneCodeInvalidError:
+ return await evt.reply("Invalid phone code.")
+ except Exception:
+ evt.log.exception("Error sending phone code")
+ return await evt.reply("Unhandled exception while sending code. "
+ "Check console for more details.")
+
+
@command_handler(needs_auth=False, management_only=True)
async def login(evt):
if evt.sender.logged_in:
@@ -80,22 +124,12 @@ async def login(evt):
return await evt.reply("This bridge instance has been configured to not allow logging in.")
-@command_handler(needs_auth=False)
-async def enter_phone(evt):
- if len(evt.args) == 0:
- return await evt.reply("**Usage:** `$cmdprefix+sp enter-phone `")
- elif not evt.config.get("bridge.allow_matrix_login", True):
- return await evt.reply("This bridge instance does not allow in-Matrix login. "
- "Please use `$cmdprefix+sp login` to get login instructions")
-
- phone_number = evt.args[0]
+async def request_code(evt, phone_number, next_status):
+ ok = False
try:
await evt.sender.ensure_started(even_if_no_session=True)
await evt.sender.client.sign_in(phone_number)
- evt.sender.command_status = {
- "next": enter_code,
- "action": "Login",
- }
+ ok = True
return await evt.reply(f"Login code sent to {phone_number}. Please send the code here.")
except PhoneNumberAppSignupForbiddenError:
return await evt.reply(
@@ -109,17 +143,31 @@ async def enter_phone(evt):
"Your phone number has been temporarily blocked for flooding. "
f"Please wait for {format_duration(e.seconds)} before trying again.")
except PhoneNumberBannedError:
- return await evt.reply("Your phone number has been banned from Telegram.")
+ return await evt.reply("Your phone number has been banned from Telegram.")
except PhoneNumberUnoccupiedError:
- return await evt.reply("That phone number has not been registered. "
- "Please register with `$cmdprefix+sp register `.")
+ return await evt.reply("That phone number has not been registered. "
+ "Please register with `$cmdprefix+sp register `.")
except Exception:
evt.log.exception("Error requesting phone code")
return await evt.reply("Unhandled exception while requesting code. "
"Check console for more details.")
finally:
- if evt.sender.command_status["next"] == enter_phone:
- evt.sender.command_status = None
+ evt.sender.command_status = next_status if ok else None
+
+
+@command_handler(needs_auth=False)
+async def enter_phone(evt):
+ if len(evt.args) == 0:
+ return await evt.reply("**Usage:** `$cmdprefix+sp enter-phone `")
+ elif not evt.config.get("bridge.allow_matrix_login", True):
+ return await evt.reply("This bridge instance does not allow in-Matrix login. "
+ "Please use `$cmdprefix+sp login` to get login instructions")
+
+ phone_number = evt.args[0]
+ await request_code(evt, phone_number, {
+ "next": enter_code,
+ "action": "Login",
+ })
@command_handler(needs_auth=False)
@@ -136,8 +184,7 @@ async def enter_code(evt):
evt.sender.command_status = None
return await evt.reply(f"Successfully logged in as @{user.username}")
except PhoneCodeExpiredError:
- return await evt.reply(
- "Phone code expired. Try again with `$cmdprefix+sp login `.")
+ return await evt.reply("Phone code expired. Try again with `$cmdprefix+sp login`.")
except PhoneCodeInvalidError:
return await evt.reply("Invalid phone code.")
except SessionPasswordNeededError:
@@ -160,7 +207,6 @@ async def enter_password(evt):
elif not evt.config.get("bridge.allow_matrix_login", True):
return await evt.reply("This bridge instance does not allow in-Matrix login. "
"Please use `$cmdprefix+sp login` to get login instructions")
-
try:
await evt.sender.ensure_started(even_if_no_session=True)
user = await evt.sender.client.sign_in(password=evt.args[0])
diff --git a/mautrix_telegram/commands/clean_rooms.py b/mautrix_telegram/commands/clean_rooms.py
index 5c515645..71ef2a91 100644
--- a/mautrix_telegram/commands/clean_rooms.py
+++ b/mautrix_telegram/commands/clean_rooms.py
@@ -3,17 +3,17 @@
# Copyright (C) 2018 Tulir Asokan
#
# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
+# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
+# GNU Affero General Public License for more details.
#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see .
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see .
from mautrix_appservice import MatrixRequestError
from . import command_handler
diff --git a/mautrix_telegram/commands/handler.py b/mautrix_telegram/commands/handler.py
index fbbbfadc..77fbe9a6 100644
--- a/mautrix_telegram/commands/handler.py
+++ b/mautrix_telegram/commands/handler.py
@@ -3,17 +3,17 @@
# Copyright (C) 2018 Tulir Asokan
#
# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
+# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
+# GNU Affero General Public License for more details.
#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see .
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see .
import markdown
import logging
@@ -81,12 +81,13 @@ class CommandHandler:
async def handle(self, room, sender, command, args, is_management, is_portal):
evt = CommandEvent(self, room, sender, command, args,
is_management, is_portal)
+ orig_command = command
command = command.lower()
try:
command = command_handlers[command]
except KeyError:
if sender.command_status and "next" in sender.command_status:
- args.insert(0, command)
+ args.insert(0, orig_command)
evt.command = ""
command = sender.command_status["next"]
else:
diff --git a/mautrix_telegram/commands/meta.py b/mautrix_telegram/commands/meta.py
index 5d2f38df..f8d1fe30 100644
--- a/mautrix_telegram/commands/meta.py
+++ b/mautrix_telegram/commands/meta.py
@@ -3,17 +3,17 @@
# Copyright (C) 2018 Tulir Asokan
#
# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
+# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
+# GNU Affero General Public License for more details.
#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see .
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see .
from . import command_handler
@@ -57,7 +57,8 @@ def help(evt):
#### Miscellaneous things
**search** [_-r|--remote_] <_query_> - Search your contacts or the Telegram servers for users.
**sync** [`chats`|`contacts`|`me`] - Synchronize your chat portals, contacts and/or own info.
-**ping-bot** - Get info of the message relay Telegram bot.
+**ping-bot** - Get info of the message relay Telegram bot.
+**set-pl** <_level_> [_mxid_] - Set a temporary power level without affecting Telegram.
#### Initiating chats
**pm** <_identifier_> - Open a private chat with the given Telegram user. The identifier is either
@@ -72,7 +73,10 @@ def help(evt):
**delete-portal** - Remove all users from the current portal room and forget the portal.
Only works for group chats; to delete a private chat portal, simply
leave the room.
-**unbridge** - Remove puppets from the current portal room and forget the portal.
+**unbridge** - Remove puppets from the current portal room and forget the portal.
+**bridge** [_id_] - Bridge the current Matrix room to the Telegram chat with the given
+ ID. The ID must be the prefixed version that you get with the `/id`
+ command of the Telegram-side bot.
**group-name** <_name_|`-`> - Change the username of a supergroup/channel. To disable, use a dash
(`-`) as the name.
**clean-rooms** - Clean up unused portal/management rooms.
diff --git a/mautrix_telegram/commands/portal.py b/mautrix_telegram/commands/portal.py
index 5885147c..76a24375 100644
--- a/mautrix_telegram/commands/portal.py
+++ b/mautrix_telegram/commands/portal.py
@@ -3,17 +3,17 @@
# Copyright (C) 2018 Tulir Asokan
#
# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
+# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
+# GNU Affero General Public License for more details.
#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see .
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see .
import asyncio
from telethon_aio.errors import *
@@ -23,6 +23,24 @@ from .. import portal as po
from . import command_handler, CommandEvent
+@command_handler(needs_admin=True, needs_auth=False, name="set-pl")
+async def set_power_level(evt: CommandEvent):
+ try:
+ level = int(evt.args[0])
+ except KeyError:
+ return await evt.reply("**Usage:** `$cmdprefix+sp set-power [mxid]`")
+ except ValueError:
+ return await evt.reply("The level must be an integer.")
+ levels = await evt.az.intent.get_power_levels(evt.room_id)
+ mxid = evt.args[1] if len(evt.args) > 1 else evt.sender.mxid
+ levels["users"][mxid] = level
+ try:
+ await evt.az.intent.set_power_levels(evt.room_id, levels)
+ except MatrixRequestError:
+ evt.log.exception("Failed to set power level.")
+ return await evt.reply("Failed to set power level.")
+
+
@command_handler()
async def invite_link(evt: CommandEvent):
portal = po.Portal.get_by_mxid(evt.room_id)
diff --git a/mautrix_telegram/commands/telegram.py b/mautrix_telegram/commands/telegram.py
index 7be428e0..34e81c0a 100644
--- a/mautrix_telegram/commands/telegram.py
+++ b/mautrix_telegram/commands/telegram.py
@@ -3,17 +3,17 @@
# Copyright (C) 2018 Tulir Asokan
#
# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
+# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
+# GNU Affero General Public License for more details.
#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see .
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see .
from telethon_aio.errors import *
from telethon_aio.tl.types import User as TLUser
from telethon_aio.tl.functions.messages import ImportChatInviteRequest, CheckChatInviteRequest
diff --git a/mautrix_telegram/config.py b/mautrix_telegram/config.py
index 7bab5563..cfc00357 100644
--- a/mautrix_telegram/config.py
+++ b/mautrix_telegram/config.py
@@ -3,19 +3,21 @@
# Copyright (C) 2018 Tulir Asokan
#
# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
+# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
+# GNU Affero General Public License for more details.
#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see .
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see .
from ruamel.yaml import YAML
from ruamel.yaml.comments import CommentedMap
+from ruamel.yaml.tokens import CommentToken
+from ruamel.yaml.error import CommentMark
import random
import string
@@ -42,6 +44,9 @@ class DictWithRecursion:
def __getitem__(self, key):
return self.get(key, None)
+ def __contains__(self, key):
+ return self[key] is not None
+
def _recursive_set(self, data, key, value):
if '.' in key:
key, next_key = key.split('.', 1)
@@ -71,6 +76,7 @@ class DictWithRecursion:
return
try:
del data[key]
+ del data.ca.items[key]
except KeyError:
pass
@@ -80,6 +86,7 @@ class DictWithRecursion:
return
try:
del self._data[key]
+ del self._data.ca.items[key]
except KeyError:
pass
@@ -93,10 +100,19 @@ class DictWithRecursion:
except ValueError:
path = None
entry = self[path] if path else self._data
- c = self._data.ca.items.setdefault(key, [None, [], None, None])
+ c = entry.ca.items.setdefault(key, [None, [], None, None])
c[1] = []
entry.yaml_set_comment_before_after_key(key=key, before=message, indent=indent)
+ def comment_newline(self, key):
+ try:
+ path, key = key.rsplit(".", 1)
+ except ValueError:
+ path = None
+ entry = self[path] if path else self._data
+ c = entry.ca.items.setdefault(key, [None, [], None, None])
+ c[2] = CommentToken("\n\n", CommentMark(0), None)
+
class Config(DictWithRecursion):
def __init__(self, path, registration_path):
@@ -131,10 +147,16 @@ class Config(DictWithRecursion):
del self["bridge.whitelist"]
del self["bridge.admins"]
- self["bridge.authless_relaybot_portals"] = self.get("bridge.authless_relaybot_portals",
- True)
- self.comment("bridge.authless_relaybot_portals",
- "Whether or not to allow creating portals from Telegram.")
+ if "bridge.authless_relaybot_portals" not in self:
+ self["bridge.authless_relaybot_portals"] = True
+ self.comment("bridge.authless_relaybot_portals",
+ "Whether or not to allow creating portals from Telegram.")
+ if "bridge.max_telegram_delete" not in self:
+ self["bridge.max_telegram_delete"] = 10
+ self.comment("bridge.max_telegram_delete",
+ "The maximum number of simultaneous Telegram deletions to handle.\n"
+ "A large number of simultaneous redactions could put strain on your "
+ "homeserver.")
self.comment("bridge.permissions", "\n".join((
"",
@@ -156,13 +178,84 @@ class Config(DictWithRecursion):
"\nThe version of the config. The bridge will read this and automatically "
"update the config if\nthe schema has changed. For the latest version, "
"check the example config.")
+ return self["version"]
+
+ def update_1_2(self):
+ del self["bridge.link_in_reply"]
+ del self["bridge.native_replies"]
+ if "bridge.bridge_notices" not in self:
+ self["bridge.bridge_notices"] = False
+ self.comment("bridge.bridge_notices",
+ "Whether or not Matrix bot messages (type m.notice) should be bridged.")
+ if "bridge.allow_matrix_login" not in self:
+ self["bridge.allow_matrix_login"] = True
+ self.comment("bridge.allow_matrix_login",
+ "Allow logging in within Matrix. If false, the only way to log in is "
+ "using the out-of-Matrix login website (see appservice.public config "
+ "section)")
+ if "bridge.inline_images" not in self:
+ self["bridge.inline_images"] = False
+ self.comment("bridge.inline_images",
+ "Use inline images instead of m.image to make rich captions possible.\n"
+ "N.B. Inline images are not supported on all clients (e.g. Riot iOS).")
+ if "appservice.public" not in self:
+ self["appservice.public.enabled"] = False
+ self["appservice.public.prefix"] = "/public"
+ self["appservice.public.external"] = "https://example.com/public"
+ self.comment("appservice.public",
+ "Public part of web server for out-of-Matrix interaction with the "
+ "bridge.\nUsed for things like login if the user wants to make sure the "
+ "2FA password isn't stored in the HS database.")
+ self.comment("appservice.public.enabled",
+ "Whether or not the public-facing endpoints should be enabled.")
+ self.comment("appservice.public.prefix",
+ "The prefix to use in the public-facing endpoints.")
+ self.comment("appservice.public.external",
+ "The base URL where the public-facing endpoints are available. The "
+ "prefix is not added\nimplicitly.")
+ if "homeserver.verify_ssl" not in self:
+ self["homeserver.verify_ssl"] = True
+ self["version"] = 2
+ return self["version"]
+
+ def update_2_3(self):
+ if "bridge.plaintext_highlights" not in self:
+ self["bridge.plaintext_highlights"] = False
+ self.comment("bridge.plaintext_highlights",
+ "Whether or not to bridge plaintext highlights.\n"
+ "Only enable this if your displayname_template has some static part that "
+ "the bridge can use to\nreliably identify what is a plaintext highlight.")
+ if "bridge.highlight_edits" not in self:
+ self["bridge.highlight_edits"] = False
+ self.comment("bridge.highlight_edits",
+ "Highlight changed/added parts in edits. Requires lxml.")
+ if "bridge.relaybot" not in self:
+ self["bridge.relaybot.authless_portals"] = bool(
+ self["bridge.authless_relaybot_portals"]) or True
+ del self["bridge.authless_relaybot_portals"]
+ self["bridge.relaybot.whitelist_group_admins"] = True
+ self["bridge.relaybot.whitelist"] = []
+ self.comment("bridge.relaybot", "Options related to the message relay Telegram bot.")
+ self.comment("bridge.relaybot.authless_portals",
+ "Whether or not to allow creating portals from Telegram.")
+ self.comment("bridge.relaybot.whitelist_group_admins",
+ "Whether or not to allow Telegram group admins to use the bot commands.")
+ self.comment("bridge.relaybot.whitelist",
+ "List of usernames/user IDs who are also allowed to use the bot commands.")
+ self["version"] = 3
+ return self["version"]
def check_updates(self):
- if self.get("version", 0) == 0:
- self.update_0_1()
- else:
- return
- self.save()
+ version = self.get("version", 0)
+ new_version = version
+ if version < 1:
+ new_version = self.update_0_1()
+ if version < 2:
+ new_version = self.update_1_2()
+ if version < 3:
+ new_version = self.update_2_3()
+ if new_version != version:
+ self.save()
def _get_permissions(self, key):
level = self["bridge.permissions"].get(key, "")
diff --git a/mautrix_telegram/context.py b/mautrix_telegram/context.py
index 5b7fb8cf..530f1eed 100644
--- a/mautrix_telegram/context.py
+++ b/mautrix_telegram/context.py
@@ -3,17 +3,17 @@
# Copyright (C) 2018 Tulir Asokan
#
# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
+# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
+# GNU Affero General Public License for more details.
#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see .
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see .
class Context:
diff --git a/mautrix_telegram/db.py b/mautrix_telegram/db.py
index be849b78..a057e15f 100644
--- a/mautrix_telegram/db.py
+++ b/mautrix_telegram/db.py
@@ -3,17 +3,17 @@
# Copyright (C) 2018 Tulir Asokan
#
# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
+# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
+# GNU Affero General Public License for more details.
#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see .
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see .
from sqlalchemy import (Column, UniqueConstraint, ForeignKey, ForeignKeyConstraint, Integer,
BigInteger, String, Boolean)
from sqlalchemy.orm import relationship
@@ -81,8 +81,8 @@ class Contact(Base):
query = None
__tablename__ = "contact"
- user = Column("user", Integer, ForeignKey("user.tgid"), primary_key=True)
- contact = Column("contact", Integer, ForeignKey("puppet.id"), primary_key=True)
+ user = Column(Integer, ForeignKey("user.tgid"), primary_key=True)
+ contact = Column(Integer, ForeignKey("puppet.id"), primary_key=True)
class Puppet(Base):
@@ -112,6 +112,11 @@ class TelegramFile(Base):
mime_type = Column(String)
was_converted = Column(Boolean)
timestamp = Column(BigInteger)
+ size = Column(Integer, nullable=True)
+ width = Column(Integer, nullable=True)
+ height = Column(Integer, nullable=True)
+ thumbnail_id = Column("thumbnail", String, ForeignKey("telegram_file.id"), nullable=True)
+ thumbnail = relationship("TelegramFile", uselist=False)
def init(db_session):
diff --git a/mautrix_telegram/formatter/__init__.py b/mautrix_telegram/formatter/__init__.py
index 0428e723..7cb102f7 100644
--- a/mautrix_telegram/formatter/__init__.py
+++ b/mautrix_telegram/formatter/__init__.py
@@ -1,2 +1,9 @@
-from .from_matrix import matrix_reply_to_telegram, matrix_to_telegram, matrix_text_to_telegram
-from .from_telegram import telegram_reply_to_matrix, telegram_to_matrix
+from .from_matrix import (matrix_reply_to_telegram, matrix_to_telegram, matrix_text_to_telegram,
+ init_mx)
+from .from_telegram import (telegram_reply_to_matrix, telegram_to_matrix, init_tg)
+from ..context import Context
+
+
+def init(context: Context):
+ init_mx(context)
+ init_tg(context)
diff --git a/mautrix_telegram/formatter/from_matrix.py b/mautrix_telegram/formatter/from_matrix.py
index c74ac1ae..9ee7ead0 100644
--- a/mautrix_telegram/formatter/from_matrix.py
+++ b/mautrix_telegram/formatter/from_matrix.py
@@ -3,31 +3,48 @@
# Copyright (C) 2018 Tulir Asokan
#
# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
+# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
+# GNU Affero General Public License for more details.
#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see .
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see .
from html import unescape
from html.parser import HTMLParser
from collections import deque
+from typing import Optional, List, Tuple, Type, Callable, Dict, Union
import math
import re
import logging
-from telethon_aio.tl.types import *
+from telethon_aio.tl.types import (MessageEntityMention,
+ InputMessageEntityMentionName, MessageEntityEmail,
+ MessageEntityUrl, MessageEntityTextUrl, MessageEntityBold,
+ MessageEntityItalic, MessageEntityCode, MessageEntityPre,
+ MessageEntityBotCommand, MessageEntityHashtag,
+ MessageEntityMentionName, InputUser)
+try:
+ from telethon_aio.tl.types import TypeMessageEntity
+except ImportError:
+ TypeMessageEntity = Union[
+ MessageEntityMention, MessageEntityHashtag, MessageEntityBotCommand, MessageEntityUrl,
+ MessageEntityEmail, MessageEntityBold, MessageEntityItalic, MessageEntityCode,
+ MessageEntityPre, MessageEntityTextUrl, MessageEntityMentionName]
+
+from ..context import Context
from .. import user as u, puppet as pu, portal as po
from ..db import Message as DBMessage
-from .util import add_surrogates, remove_surrogates
+from .util import (add_surrogates, remove_surrogates, trim_reply_fallback_html,
+ trim_reply_fallback_text, html_to_unicode)
log = logging.getLogger("mau.fmt.mx")
+should_bridge_plaintext_highlights = False
class MatrixParser(HTMLParser):
@@ -35,7 +52,7 @@ class MatrixParser(HTMLParser):
room_regex = re.compile("https://matrix.to/#/(#.+:.+)")
block_tags = ("br", "p", "pre", "blockquote",
"ol", "ul", "li",
- "h1", "h2", "h3", "h4", "h5", "h6"
+ "h1", "h2", "h3", "h4", "h5", "h6",
"div", "hr", "table")
def __init__(self):
@@ -49,21 +66,20 @@ class MatrixParser(HTMLParser):
self._line_is_new = True
self._list_entry_is_new = False
- def _parse_url(self, url, args):
+ def _parse_url(self, url: str, args: Dict[str, str]
+ ) -> Tuple[Optional[Type[TypeMessageEntity]], Optional[str]]:
mention = self.mention_regex.match(url)
if mention:
mxid = mention.group(1)
- user = (pu.Puppet.get_by_mxid(mxid, create=False)
+ user = (pu.Puppet.get_by_mxid(mxid)
or u.User.get_by_mxid(mxid, create=False))
if not user:
return None, None
if user.username:
- entity_type = MessageEntityMention
- url = f"@{user.username}"
+ return MessageEntityMention, f"@{user.username}"
else:
- entity_type = MessageEntityMentionName
- args["user_id"] = user.tgid
- return entity_type, url
+ args["user_id"] = InputUser(user.tgid, 0)
+ return InputMessageEntityMentionName, user.displayname or None
room = self.room_regex.match(url)
if room:
@@ -80,7 +96,7 @@ class MatrixParser(HTMLParser):
args["url"] = url
return MessageEntityTextUrl, None
- def handle_starttag(self, tag, attrs):
+ def handle_starttag(self, tag: str, attrs: List[Tuple[str, str]]):
self._open_tags.appendleft(tag)
self._open_tags_meta.appendleft(0)
@@ -127,7 +143,7 @@ class MatrixParser(HTMLParser):
self._building_entities[tag] = entity_type(offset=offset, length=0, **args)
@property
- def _list_indent(self):
+ def _list_indent(self) -> int:
indent = 0
first_skipped = False
for index, tag in enumerate(self._open_tags):
@@ -143,24 +159,41 @@ class MatrixParser(HTMLParser):
indent += 3
return indent
- def _newline(self, allow_multi=False):
- if self._line_is_new or allow_multi:
+ def _newline(self, allow_multi: bool = False):
+ if self._line_is_new and not allow_multi:
return
self.text += "\n"
self._line_is_new = True
for entity in self._building_entities.values():
entity.length += 1
- def handle_data(self, text):
- text = unescape(text)
+ def _handle_special_previous_tags(self, text: str) -> str:
+ if "pre" not in self._open_tags and "code" not in self._open_tags:
+ text = text.replace("\n", "")
+ else:
+ text = text.strip()
+
previous_tag = self._open_tags[0] if len(self._open_tags) > 0 else ""
- extra_offset = 0
if previous_tag == "a":
url = self._open_tags_meta[0]
if url:
text = url
elif previous_tag == "command":
text = f"/{text}"
+ return text
+
+ def _html_to_unicode(self, text: str) -> str:
+ strikethrough, underline = "del" in self._open_tags, "u" in self._open_tags
+ if strikethrough and underline:
+ text = html_to_unicode(text, "\u0336\u0332")
+ elif strikethrough:
+ text = html_to_unicode(text, "\u0336")
+ elif underline:
+ text = html_to_unicode(text, "\u0332")
+ return text
+
+ def _handle_tags_for_data(self, text: str) -> Tuple[str, int]:
+ extra_offset = 0
list_entry_handled_once = False
# In order to maintain order of things like blockquotes in lists or lists in blockquotes,
# we can't just have ifs/elses and we need to actually loop through the open tags in order.
@@ -188,52 +221,77 @@ class MatrixParser(HTMLParser):
text = indent + prefix + text
self._list_entry_is_new = False
list_entry_handled_once = True
+ return text, extra_offset
+
+ def _extend_entities_in_construction(self, text: str, extra_offset: int):
for tag, entity in self._building_entities.items():
entity.length += len(text) - extra_offset
entity.offset += extra_offset
+ def handle_data(self, text: str):
+ text = unescape(text)
+ text = self._handle_special_previous_tags(text)
+ text = self._html_to_unicode(text)
+ text, extra_offset = self._handle_tags_for_data(text)
+ self._extend_entities_in_construction(text, extra_offset)
self._line_is_new = False
self.text += text
- def handle_endtag(self, tag):
+ def handle_endtag(self, tag: str):
try:
self._open_tags.popleft()
self._open_tags_meta.popleft()
except IndexError:
pass
- if tag in self.block_tags:
- self._newline(allow_multi=tag == "br")
-
entity = self._building_entities.pop(tag, None)
if entity:
self.entities.append(entity)
+ if tag in self.block_tags:
+ self._newline(allow_multi=tag == "br")
+
command_regex = re.compile("(\s|^)!([A-Za-z0-9@]+)")
+plain_mention_regex = None
-def matrix_text_to_telegram(text):
- text = command_regex.sub(r"\1/\2", text)
- return text
+def plain_mention_to_html(match):
+ puppet = pu.Puppet.find_by_displayname(match.group(2))
+ if puppet:
+ return (f"{match.group(1)}"
+ f""
+ f"{puppet.displayname}"
+ "")
+ return "".join(match.groups())
-def matrix_to_telegram(html):
+def matrix_to_telegram(html: str) -> Tuple[str, List[TypeMessageEntity]]:
try:
parser = MatrixParser()
- html = html.replace("\n", "")
html = command_regex.sub(r"\1\2 ", html)
+ if should_bridge_plaintext_highlights:
+ html = plain_mention_regex.sub(plain_mention_to_html, html)
parser.feed(add_surrogates(html))
return remove_surrogates(parser.text.strip()), parser.entities
except Exception:
log.exception("Failed to convert Matrix format:\nhtml=%s", html)
-def matrix_reply_to_telegram(content, tg_space, room_id=None):
+def matrix_reply_to_telegram(content: dict, tg_space: int, room_id: Optional[str] = None
+ ) -> Optional[int]:
try:
reply = content["m.relates_to"]["m.in_reply_to"]
room_id = room_id or reply["room_id"]
event_id = reply["event_id"]
+
+ try:
+ if content["format"] == "org.matrix.custom.html":
+ content["formatted_body"] = trim_reply_fallback_html(content["formatted_body"])
+ except KeyError:
+ pass
+ content["body"] = trim_reply_fallback_text(content["body"])
+
message = DBMessage.query.filter(DBMessage.mxid == event_id,
DBMessage.tg_space == tg_space,
DBMessage.mx_room == room_id).one_or_none()
@@ -242,3 +300,44 @@ def matrix_reply_to_telegram(content, tg_space, room_id=None):
except KeyError:
pass
return None
+
+
+def matrix_text_to_telegram(text: str) -> Tuple[str, List[TypeMessageEntity]]:
+ text = command_regex.sub(r"\1/\2", text)
+ if should_bridge_plaintext_highlights:
+ entities, pmr_replacer = plain_mention_to_text()
+ text = plain_mention_regex.sub(pmr_replacer, text)
+ else:
+ entities = []
+ return text, entities
+
+
+def plain_mention_to_text() -> Tuple[List[TypeMessageEntity], Callable[[str], str]]:
+ entities = []
+
+ def replacer(match):
+ puppet = pu.Puppet.find_by_displayname(match.group(2))
+ if puppet:
+ offset = match.start()
+ length = match.end() - offset
+ if puppet.username:
+ entity = MessageEntityMention(offset, length)
+ text = f"@{puppet.username}"
+ else:
+ entity = InputMessageEntityMentionName(offset, length,
+ user_id=InputUser(puppet.tgid, 0))
+ text = puppet.displayname
+ entities.append(entity)
+ return text
+ return "".join(match.groups())
+
+ return entities, replacer
+
+
+def init_mx(context: Context):
+ global plain_mention_regex, should_bridge_plaintext_highlights
+ config = context.config
+ dn_template = config.get("bridge.displayname_template", "{displayname} (Telegram)")
+ dn_template = re.escape(dn_template).replace(re.escape("{displayname}"), "[^>]+")
+ plain_mention_regex = re.compile(f"(\s|^)({dn_template})")
+ should_bridge_plaintext_highlights = config["bridge.plaintext_highlights"] or False
diff --git a/mautrix_telegram/formatter/from_telegram.py b/mautrix_telegram/formatter/from_telegram.py
index 82b38c10..4cd6de00 100644
--- a/mautrix_telegram/formatter/from_telegram.py
+++ b/mautrix_telegram/formatter/from_telegram.py
@@ -3,31 +3,55 @@
# Copyright (C) 2018 Tulir Asokan
#
# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
+# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
+# GNU Affero General Public License for more details.
#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see .
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see .
from html import escape
-import logging
+from typing import Optional, List, Tuple, Union
+
+try:
+ from lxml.html.diff import htmldiff
+except ImportError:
+ htmldiff = None
+import logging
+import re
+
+from telethon_aio.tl.types import (MessageEntityMention, MessageEntityMentionName,
+ MessageEntityEmail, MessageEntityUrl, MessageEntityTextUrl,
+ MessageEntityBold, MessageEntityItalic, MessageEntityCode,
+ MessageEntityPre, MessageEntityBotCommand, Message, PeerChannel,
+ MessageEntityHashtag)
+
+try:
+ from telethon_aio.tl.types import TypeMessageEntity
+except ImportError:
+ TypeMessageEntity = Union[
+ MessageEntityMention, MessageEntityHashtag, MessageEntityBotCommand, MessageEntityUrl,
+ MessageEntityEmail, MessageEntityBold, MessageEntityItalic, MessageEntityCode,
+ MessageEntityPre, MessageEntityTextUrl, MessageEntityMentionName]
-from telethon_aio.tl.types import *
from mautrix_appservice import MatrixRequestError
+from mautrix_appservice.intent_api import IntentAPI
from .. import user as u, puppet as pu, portal as po
+from ..context import Context
from ..db import Message as DBMessage
-from .util import add_surrogates, remove_surrogates
+from .util import (add_surrogates, remove_surrogates, trim_reply_fallback_html,
+ trim_reply_fallback_text, unicode_to_html)
log = logging.getLogger("mau.fmt.tg")
+should_highlight_edits = False
-def telegram_reply_to_matrix(evt, source):
+def telegram_reply_to_matrix(evt: Message, source: u.User) -> dict:
if evt.reply_to_msg_id:
space = (evt.to_id.channel_id
if isinstance(evt, Message) and isinstance(evt.to_id, PeerChannel)
@@ -43,7 +67,8 @@ def telegram_reply_to_matrix(evt, source):
return {}
-async def _add_forward_header(source, text, html, fwd_from_id):
+async def _add_forward_header(source, text: str, html: Optional[str],
+ fwd_from_id: Optional[int]) -> Tuple[str, str]:
if not html:
html = escape(text)
user = u.User.get_by_tgid(fwd_from_id)
@@ -67,8 +92,23 @@ async def _add_forward_header(source, text, html, fwd_from_id):
return text, html
-async def _add_reply_header(source, text, html, evt, relates_to,
- native_replies, message_link_in_reply, main_intent, reply_text):
+def highlight_edits(new_html: str, old_html: str) -> str:
+ # Don't include `Edit:` text in diff.
+ if old_html.startswith("Edit: "):
+ old_html = old_html[len("Edit: "):]
+
+ # Generate diff with lxml
+ new_html = htmldiff(old_html, new_html)
+
+ # Replace with since Riot doesn't allow
+ new_html = new_html.replace("", "").replace("", "")
+ # Remove s since we just want to hide deletions.
+ new_html = re.sub(".+?", "", new_html)
+ return new_html
+
+
+async def _add_reply_header(source: u.User, text: str, html: str, evt: Message, relates_to: dict,
+ main_intent: IntentAPI, is_edit: bool) -> Tuple[str, str]:
space = (evt.to_id.channel_id
if isinstance(evt, Message) and isinstance(evt.to_id, PeerChannel)
else source.tgid)
@@ -77,52 +117,71 @@ async def _add_reply_header(source, text, html, evt, relates_to,
if not msg:
return text, html
- if native_replies:
- relates_to["m.in_reply_to"] = {
- "event_id": msg.mxid,
- "room_id": msg.mx_room,
- }
- if reply_text == "Edit":
- html = f"Edit: {html or escape(text)}"
- text = f"Edit: {text}"
- return text, html
+ relates_to["m.in_reply_to"] = {
+ "event_id": msg.mxid,
+ "room_id": msg.mx_room,
+ }
- reply_displayname = "unknown user"
try:
event = await main_intent.get_event(msg.mx_room, msg.mxid)
+
content = event["content"]
- body = (content["formatted_body"]
- if "formatted_body" in content
- else content["body"])
- sender = event['sender']
- puppet = pu.Puppet.get_by_mxid(sender, create=False)
- reply_displayname = puppet.displayname if puppet else sender
- reply_to_user = f"{reply_displayname}"
- reply_to_msg = (("{reply_text}")
- if message_link_in_reply else "Reply")
- quote = f"{reply_to_msg} to {reply_to_user}{body}
"
+ r_sender = event["sender"]
+
+ r_text_body = trim_reply_fallback_text(content["body"])
+ r_html_body = trim_reply_fallback_html(content["formatted_body"]
+ if "formatted_body" in content
+ else escape(content["body"]))
+
+ puppet = pu.Puppet.get_by_mxid(r_sender, create=False)
+ r_displayname = puppet.displayname if puppet else r_sender
+ r_sender_link = f"{r_displayname}"
+
+ if is_edit and should_highlight_edits:
+ html = highlight_edits(html or escape(text), r_html_body)
except (ValueError, KeyError, MatrixRequestError):
- quote = f"{reply_text} to unknown user (Failed to fetch message):
"
- if not html:
- html = escape(text)
- html = quote + html
- text = f"{reply_text} to {reply_displayname}:\n{text}"
- return text, html
+ r_sender_link = "unknown user"
+ # r_sender = "unknown user"
+ r_text_body = "Failed to fetch message"
+ r_html_body = "Failed to fetch message"
+
+ if is_edit:
+ html = f"Edit: {html or escape(text)}"
+ text = f"Edit: {text}"
+
+ r_keyword = "In reply to" if not is_edit else "Edit to"
+ r_msg_link = f"{r_keyword}"
+ html = (f"{r_msg_link} {r_sender_link}\n{r_html_body}
"
+ + (html or escape(text)))
+
+ lines = r_text_body.strip().split("\n")
+ text_with_quote = f"> <{r_displayname}> {lines.pop(0)}"
+ for line in lines:
+ if line:
+ text_with_quote += f"\n> {line}"
+ text_with_quote += "\n\n"
+ text_with_quote += text
+ return text_with_quote, html
-async def telegram_to_matrix(evt, source, native_replies=False, message_link_in_reply=False,
- main_intent=None, reply_text="Reply"):
+async def telegram_to_matrix(evt: Message, source: u.User, main_intent: Optional[IntentAPI] = None,
+ is_edit: bool = False, prefix_text: Optional[str] = None,
+ prefix_html: Optional[str] = None) -> Tuple[str, str, dict]:
text = add_surrogates(evt.message)
html = _telegram_entities_to_matrix_catch(text, evt.entities) if evt.entities else None
relates_to = {}
+ if prefix_html:
+ html = prefix_html + (html or escape(text))
+ if prefix_text:
+ text = prefix_text + text
+
if evt.fwd_from:
text, html = await _add_forward_header(source, text, html, evt.fwd_from.from_id)
if evt.reply_to_msg_id:
- text, html = await _add_reply_header(source, text, html, evt, relates_to, native_replies,
- message_link_in_reply, main_intent, reply_text)
+ text, html = await _add_reply_header(source, text, html, evt, relates_to, main_intent,
+ is_edit)
if isinstance(evt, Message) and evt.post and evt.post_author:
if not html:
@@ -130,13 +189,16 @@ async def telegram_to_matrix(evt, source, native_replies=False, message_link_in_
text += f"\n- {evt.post_author}"
html += f"
- {evt.post_author}"
+ html = unicode_to_html(text, html, "\u0336", "del")
+ html = unicode_to_html(text, html, "\u0332", "u")
+
if html:
html = html.replace("\n", "
")
return remove_surrogates(text), remove_surrogates(html), relates_to
-def _telegram_entities_to_matrix_catch(text, entities):
+def _telegram_entities_to_matrix_catch(text: str, entities: List[TypeMessageEntity]) -> str:
try:
return _telegram_entities_to_matrix(text, entities)
except Exception:
@@ -146,7 +208,7 @@ def _telegram_entities_to_matrix_catch(text, entities):
text, entities)
-def _telegram_entities_to_matrix(text, entities):
+def _telegram_entities_to_matrix(text: str, entities: List[TypeMessageEntity]) -> str:
if not entities:
return text
html = []
@@ -190,7 +252,7 @@ def _telegram_entities_to_matrix(text, entities):
return "".join(html)
-def _parse_pre(html, entity_text, language):
+def _parse_pre(html: List[str], entity_text: str, language: str) -> bool:
if language:
html.append(""
f"{entity_text}"
@@ -200,7 +262,7 @@ def _parse_pre(html, entity_text, language):
return False
-def _parse_mention(html, entity_text):
+def _parse_mention(html: List[str], entity_text: str) -> bool:
username = entity_text[1:]
user = u.User.find_by_username(username) or pu.Puppet.find_by_username(username)
@@ -217,7 +279,7 @@ def _parse_mention(html, entity_text):
return False
-def _parse_name_mention(html, entity_text, user_id):
+def _parse_name_mention(html: List[str], entity_text: str, user_id: int) -> bool:
user = u.User.get_by_tgid(user_id)
if user:
mxid = user.mxid
@@ -231,9 +293,14 @@ def _parse_name_mention(html, entity_text, user_id):
return False
-def _parse_url(html, entity_text, url):
+def _parse_url(html: List[str], entity_text: str, url: str) -> bool:
url = escape(url) if url else entity_text
if not url.startswith(("https://", "http://", "ftp://", "magnet://")):
url = "http://" + url
html.append(f"{entity_text}")
return False
+
+
+def init_tg(context: Context):
+ global should_highlight_edits
+ should_highlight_edits = htmldiff and context.config["bridge.highlight_edits"]
diff --git a/mautrix_telegram/formatter/util.py b/mautrix_telegram/formatter/util.py
index ff35519d..f10146b4 100644
--- a/mautrix_telegram/formatter/util.py
+++ b/mautrix_telegram/formatter/util.py
@@ -1,16 +1,84 @@
-# Unicode surrogate handling
-# From https://github.com/LonamiWebs/Telethon/blob/master/telethon/extensions/markdown.py
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2018 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see .
+from html import escape
+from typing import Optional
import struct
+import re
-def add_surrogates(text):
+# add_surrogates and remove_surrogates are unicode surrogate utility functions from Telethon.
+# Licensed under the MIT license.
+# https://github.com/LonamiWebs/Telethon/blob/master/telethon/extensions/markdown.py
+def add_surrogates(text: Optional[str]) -> Optional[str]:
if text is None:
return None
return "".join("".join(chr(y) for y in struct.unpack(" Optional[str]:
if text is None:
return None
return text.encode("utf-16", "surrogatepass").decode("utf-16")
+
+
+def trim_reply_fallback_text(text: str) -> str:
+ if not text.startswith("> ") or "\n" not in text:
+ return text
+ lines = text.split("\n")
+ while len(lines) > 0 and lines[0].startswith("> "):
+ lines.pop(0)
+ return "\n".join(lines)
+
+
+html_reply_fallback_regex = re.compile(r"^[\s\S]+?
")
+
+
+def trim_reply_fallback_html(html: str) -> str:
+ return html_reply_fallback_regex.sub("", html)
+
+
+def unicode_to_html(text: str, html: str, ctrl: str, tag: str) -> str:
+ if ctrl not in text:
+ return html
+ if not html:
+ html = escape(text)
+ tag_start = f"<{tag}>"
+ tag_end = f"{tag}>"
+ characters = html.split(ctrl)
+ html = ""
+ in_tag = False
+ for char in characters:
+ if not in_tag:
+ if len(char) > 1:
+ html += char[0:-1]
+ char = char[-1]
+ html += tag_start
+ in_tag = True
+ html += char
+ else:
+ if len(char) > 1:
+ html += tag_end
+ in_tag = False
+ html += char
+ if in_tag:
+ html += tag_end
+ return html
+
+
+def html_to_unicode(text: str, ctrl: str) -> str:
+ return ctrl.join(text) + ctrl
diff --git a/mautrix_telegram/matrix.py b/mautrix_telegram/matrix.py
index fb7911d0..aca86ec7 100644
--- a/mautrix_telegram/matrix.py
+++ b/mautrix_telegram/matrix.py
@@ -3,17 +3,17 @@
# Copyright (C) 2018 Tulir Asokan
#
# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
+# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
+# GNU Affero General Public License for more details.
#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see .
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see .
import logging
from mautrix_appservice import MatrixRequestError
@@ -222,6 +222,18 @@ class MatrixHandler:
return
await handler(sender, content[content_key])
+ async def handle_room_pin(self, room, sender, new_events, old_events):
+ portal = Portal.get_by_mxid(room)
+ sender = await User.get_by_mxid(sender).ensure_started()
+ if sender.has_full_access and portal:
+ events = new_events - old_events
+ if len(events) > 0:
+ # New event pinned, set that as pinned in Telegram.
+ await portal.handle_matrix_pin(sender, events.pop())
+ elif len(new_events) == 0:
+ # All pinned events removed, remove pinned event in Telegram.
+ await portal.handle_matrix_pin(sender, None)
+
def filter_matrix_event(self, event):
return (event["sender"] == self.az.bot_mxid
or Puppet.get_id_from_mxid(event["sender"]) is not None)
@@ -250,3 +262,10 @@ class MatrixHandler:
evt["prev_content"])
elif type in ("m.room.name", "m.room.avatar", "m.room.topic"):
await self.handle_room_meta(type, evt["room_id"], evt["sender"], evt["content"])
+ elif type == "m.room.pinned_events":
+ new_events = set(evt["content"]["pinned"])
+ try:
+ old_events = set(evt["unsigned"]["prev_content"]["pinned"])
+ except KeyError:
+ old_events = set()
+ await self.handle_room_pin(evt["room_id"], evt["sender"], new_events, old_events)
diff --git a/mautrix_telegram/portal.py b/mautrix_telegram/portal.py
index 497794d2..bb29b918 100644
--- a/mautrix_telegram/portal.py
+++ b/mautrix_telegram/portal.py
@@ -3,17 +3,17 @@
# Copyright (C) 2018 Tulir Asokan
#
# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
+# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
+# GNU Affero General Public License for more details.
#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see .
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see .
from collections import deque
from datetime import datetime
import asyncio
@@ -66,6 +66,8 @@ class Portal:
self._main_intent = None
self._room_create_lock = asyncio.Lock()
+ self._temp_pinned_message_id = None
+ self._temp_pinned_message_sender = None
self._dedup = deque()
self._dedup_mxid = {}
@@ -516,12 +518,10 @@ class Portal:
try:
current_extension = body[body.rindex("."):]
if mimetypes.types_map[current_extension] == mime:
- file_name = body
- else:
- file_name = f"matrix_upload{mimetypes.guess_extension(mime)}"
+ return body
except (ValueError, KeyError):
- file_name = f"matrix_upload{mimetypes.guess_extension(mime)}"
- return file_name, None if file_name == body else body
+ pass
+ return f"matrix_upload{mimetypes.guess_extension(mime)}"
async def leave_matrix(self, user, source, event_id):
if not user.logged_in:
@@ -570,25 +570,54 @@ class Portal:
@staticmethod
def _preprocess_matrix_message(sender, message):
- if message["msgtype"] == "m.emote":
+ msgtype = message["msgtype"]
+ if msgtype == "m.emote":
if "formatted_body" in message:
message["formatted_body"] = f"* {sender.displayname} {message['formatted_body']}"
message["body"] = f"* {sender.displayname} {message['body']}"
message["msgtype"] = "m.text"
elif not sender.logged_in:
- if "formatted_body" in message:
- message["formatted_body"] = (f"<{sender.displayname}> "
- f"{message['formatted_body']}")
- message["body"] = f"<{sender.displayname}> {message['body']}"
- return type
+ html = message["formatted_body"] if "formatted_body" in message else None
+ text = message["body"]
+ if msgtype == "m.text":
+ if html:
+ html = f"<{sender.displayname}> {html}"
+ text = f"<{sender.displayname}> {text}"
+ else:
+ msgtype = msgtype[len("m."):]
+ prefix = {
+ "file": "a ",
+ "image": "an ",
+ "audio": "",
+ "video": "a ",
+ "location": "a ",
+ }.get(msgtype, "")
+ if html:
+ html = f"{sender.displayname} sent {prefix}{msgtype}: {html}"
+ text = ": " + text if text else ""
+ text = f"{sender.displayname} sent {prefix}{msgtype}{text}"
+ if html:
+ message["formatted_body"] = html
+ message["body"] = text
- def _handle_matrix_text(self, client, message, reply_to):
- if "format" in message and message["format"] == "org.matrix.custom.html":
- message, entities = formatter.matrix_to_telegram(message["formatted_body"])
- return client.send_message(self.peer, message, entities=entities, reply_to=reply_to)
- else:
- message = formatter.matrix_text_to_telegram(message["body"])
- return client.send_message(self.peer, message, reply_to=reply_to)
+ async def _matrix_event_to_entities(self, client, event):
+ try:
+ if event.get("format", None) == "org.matrix.custom.html":
+ message, entities = formatter.matrix_to_telegram(event["formatted_body"])
+
+ # TODO remove this crap
+ for entity in entities:
+ if isinstance(entity, InputMessageEntityMentionName):
+ entity.user_id = await client.get_input_entity(entity.user_id.user_id)
+ else:
+ message, entities = formatter.matrix_text_to_telegram(event["body"])
+ except KeyError:
+ message, entities = None, None
+ return message, entities
+
+ async def _handle_matrix_text(self, client, message, reply_to):
+ message, entities = await self._matrix_event_to_entities(client, message)
+ return await client.send_message(self.peer, message, entities=entities, reply_to=reply_to)
async def _handle_matrix_file(self, client, message, reply_to):
file = await self.main_intent.download_file(message["url"])
@@ -596,32 +625,53 @@ class Portal:
info = message["info"]
mime = info["mimetype"]
- file_name, caption = self._get_file_meta(message["body"], mime)
+ file_name = self._get_file_meta(message["mxtg_filename"], mime)
attributes = [DocumentAttributeFilename(file_name=file_name)]
if "w" in info and "h" in info:
attributes.append(DocumentAttributeImageSize(w=info["w"], h=info["h"]))
+ caption = message["body"] if message["body"] != file_name else None
return await client.send_file(self.peer, file, mime, caption=caption,
attributes=attributes, file_name=file_name,
reply_to=reply_to)
+ async def _handle_matrix_location(self, client, message, reply_to):
+ try:
+ lat, long = message["geo_uri"][len("geo:"):].split(",")
+ lat, long = float(lat), float(long)
+ except (KeyError, ValueError):
+ self.log.exception("Failed to parse location")
+ return None
+ message, entities = await self._matrix_event_to_entities(client, message)
+ media = MessageMediaGeo(geo=GeoPoint(lat, long))
+ return await client.send_media(self.peer, media, reply_to=reply_to, caption=message,
+ entities=entities)
+
async def handle_matrix_message(self, sender, message, event_id):
client = sender.client if sender.logged_in else self.bot.client
space = (self.tgid if self.peer_type == "channel" # Channels have their own ID space
else (sender.tgid if sender.logged_in else self.bot.tgid))
reply_to = formatter.matrix_reply_to_telegram(message, space, room_id=self.mxid)
+ message["mxtg_filename"] = message["body"]
self._preprocess_matrix_message(sender, message)
type = message["msgtype"]
if type == "m.text" or (self.bridge_notices and type == "m.notice"):
response = await self._handle_matrix_text(client, message, reply_to)
+ elif type == "m.location":
+ response = await self._handle_matrix_location(client, message, reply_to)
elif type in ("m.image", "m.file", "m.audio", "m.video"):
response = await self._handle_matrix_file(client, message, reply_to)
else:
self.log.debug("Unhandled Matrix event: %s", message)
+ response = None
+
+ if not response:
return
+
+ self.log.debug("Handled Matrix message: %s", response)
self.is_duplicate(response, (event_id, space))
self.db.add(DBMessage(
tgid=response.id,
@@ -630,6 +680,20 @@ class Portal:
mxid=event_id))
self.db.commit()
+ async def handle_matrix_pin(self, sender, pinned_message):
+ if self.peer_type != "channel":
+ return
+ try:
+ if not pinned_message:
+ await sender.client(UpdatePinnedMessageRequest(channel=self.peer, id=0))
+ else:
+ message = DBMessage.query.filter(DBMessage.mxid == pinned_message,
+ DBMessage.tg_space == self.tgid,
+ DBMessage.mx_room == self.mxid).one_or_none()
+ await sender.client(UpdatePinnedMessageRequest(channel=self.peer, id=message.tgid))
+ except ChatNotModifiedError:
+ pass
+
async def handle_matrix_deletion(self, deleter, event_id):
space = self.tgid if self.peer_type == "channel" else deleter.tgid
message = DBMessage.query.filter(DBMessage.mxid == event_id,
@@ -650,7 +714,7 @@ class Portal:
edit_messages=moderator, delete_messages=moderator,
ban_users=moderator, invite_users=moderator,
invite_link=moderator, pin_messages=moderator,
- add_admins=admin, manage_call=moderator)
+ add_admins=admin)
await sender.client(
EditAdminRequest(channel=await self.get_input_entity(sender),
user_id=user_id, admin_rights=rights))
@@ -823,12 +887,19 @@ class Portal:
if self.mxid:
await user.intent.set_typing(self.mxid, is_typing=True)
- async def handle_telegram_photo(self, source, intent, evt, relates_to=None):
+ async def handle_telegram_photo(self, source: u.User, intent, evt: Message, relates_to=None):
largest_size = self._get_largest_photo_size(evt.media.photo)
file = await util.transfer_file_to_matrix(self.db, source.client, intent,
largest_size.location)
if not file:
return None
+ if config["bridge.inline_images"] and (evt.message or evt.fwd_from or evt.reply_to_msg_id):
+ text, html, relates_to = await formatter.telegram_to_matrix(
+ evt, source, self.main_intent,
+ prefix_html=f"
",
+ prefix_text="Inline image: ")
+ await intent.set_typing(self.mxid, is_typing=False)
+ return await intent.send_text(self.mxid, text, html=html, relates_to=relates_to)
info = {
"h": largest_size.h,
"w": largest_size.w,
@@ -842,25 +913,40 @@ class Portal:
return await intent.send_image(self.mxid, file.mxc, info=info, text=name,
relates_to=relates_to)
- async def handle_telegram_document(self, source, intent, evt, relates_to=None):
+ async def handle_telegram_document(self, source, intent, evt: Message, relates_to=None):
document = evt.media.document
- file = await util.transfer_file_to_matrix(self.db, source.client, intent, document)
+ file = await util.transfer_file_to_matrix(self.db, source.client, intent, document,
+ document.thumb)
if not file:
return None
name = evt.message
+ width, height = file.width, file.height
for attr in document.attributes:
- if not name and isinstance(attr, DocumentAttributeFilename):
- name = attr.file_name
+ if isinstance(attr, DocumentAttributeFilename):
+ name = name or attr.file_name
if not file.was_converted:
(mime_from_name, _) = mimetypes.guess_type(name)
file.mime_type = mime_from_name or file.mime_type
elif isinstance(attr, DocumentAttributeSticker):
name = f"Sticker for {attr.alt}"
+ elif isinstance(attr, DocumentAttributeVideo) and (not width or not height):
+ width, height = attr.w, attr.h
mime_type = document.mime_type or file.mime_type
info = {
- "size": document.size,
+ "size": file.size,
"mimetype": mime_type,
}
+ if file.thumbnail:
+ info["thumbnail_url"] = file.thumbnail.mxc
+ info["thumbnail_info"] = {
+ "mimetype": file.thumbnail.mime_type,
+ "h": file.thumbnail.height or document.thumb.h,
+ "w": file.thumbnail.width or document.thumb.w,
+ "size": file.thumbnail.size,
+ }
+ if height and width:
+ info["h"] = height
+ info["w"] = width
type = "m.file"
if mime_type.startswith("video/"):
type = "m.video"
@@ -900,11 +986,7 @@ class Portal:
async def handle_telegram_text(self, source, intent, evt):
self.log.debug(f"Sending {evt.message} to {self.mxid} by {intent.mxid}")
- text, html, relates_to = await formatter.telegram_to_matrix(
- evt, source,
- config["bridge.native_replies"],
- config["bridge.link_in_reply"],
- self.main_intent)
+ text, html, relates_to = await formatter.telegram_to_matrix(evt, source, self.main_intent)
await intent.set_typing(self.mxid, is_typing=False)
return await intent.send_text(self.mxid, text, html=html, relates_to=relates_to)
@@ -928,11 +1010,8 @@ class Portal:
return
evt.reply_to_msg_id = evt.id
- text, html, relates_to = await formatter.telegram_to_matrix(
- evt, source,
- config["bridge.native_replies"],
- config["bridge.link_in_reply"],
- self.main_intent, reply_text="Edit")
+ text, html, relates_to = await formatter.telegram_to_matrix(evt, source, self.main_intent,
+ is_edit=True)
intent = sender.intent if sender else self.main_intent
await intent.set_typing(self.mxid, is_typing=False)
response = await intent.send_text(self.mxid, text, html=html, relates_to=relates_to)
@@ -966,7 +1045,9 @@ class Portal:
DBMessage(tgid=evt.id, mx_room=self.mxid, mxid=mxid, tg_space=tg_space))
self.db.commit()
return
- media = evt.media if hasattr(evt, "media") else None
+ allowed_media = (MessageMediaPhoto, MessageMediaDocument, MessageMediaGeo)
+ media = evt.media if hasattr(evt, "media") and isinstance(evt.media,
+ allowed_media) else None
intent = sender.intent if sender else self.main_intent
if not media and evt.message:
response = await self.handle_telegram_text(source, intent, evt)
@@ -988,7 +1069,7 @@ class Portal:
if not response:
return
-
+ self.log.debug("Handled Telegram message: %s", evt)
mxid = response["event_id"]
DBMessage.query \
.filter(DBMessage.mx_room == self.mxid,
@@ -1013,7 +1094,6 @@ class Portal:
or self.is_duplicate_action(update))
if should_ignore:
return
-
# TODO figure out how to see changes to about text / channel username
if isinstance(action, MessageActionChatEditTitle):
await self.update_title(action.title, save=True)
@@ -1031,6 +1111,8 @@ class Portal:
self.peer_type = "channel"
self.migrate_and_save(action.channel_id)
await sender.intent.send_emote(self.mxid, "upgraded this group to a supergroup.")
+ elif isinstance(action, MessageActionPinMessage):
+ await self.receive_telegram_pin_sender(sender)
else:
self.log.debug("Unhandled Telegram action in %s: %s", self.title, action)
@@ -1045,13 +1127,30 @@ class Portal:
levels["users"][puppet.mxid] = 50
await self.main_intent.set_power_levels(self.mxid, levels)
- async def update_telegram_pin(self, source, id):
- space = self.tgid if self.peer_type == "channel" else source.tgid
- message = DBMessage.query.get((id, space))
+ async def receive_telegram_pin_sender(self, sender):
+ self._temp_pinned_message_sender = sender
+ if self._temp_pinned_message_id:
+ await self.update_telegram_pin()
+
+ async def update_telegram_pin(self):
+ intent = (self._temp_pinned_message_sender.intent
+ if self._temp_pinned_message_sender else self.main_intent)
+ id = self._temp_pinned_message_id
+ self._temp_pinned_message_id = None
+ self._temp_pinned_message_sender = None
+
+ message = DBMessage.query.get((id, self.tgid))
if message:
- await self.main_intent.set_pinned_messages(self.mxid, [message.mxid])
+ await intent.set_pinned_messages(self.mxid, [message.mxid])
else:
- await self.main_intent.set_pinned_messages(self.mxid, [])
+ await intent.set_pinned_messages(self.mxid, [])
+
+ async def receive_telegram_pin_id(self, id):
+ if id == 0:
+ return await self.update_telegram_pin()
+ self._temp_pinned_message_id = id
+ if self._temp_pinned_message_sender:
+ await self.update_telegram_pin()
@staticmethod
def _get_level_from_participant(participant, _):
diff --git a/mautrix_telegram/public/__init__.py b/mautrix_telegram/public/__init__.py
index 3f57e369..8a4d9e8d 100644
--- a/mautrix_telegram/public/__init__.py
+++ b/mautrix_telegram/public/__init__.py
@@ -3,17 +3,17 @@
# Copyright (C) 2018 Tulir Asokan
#
# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
+# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
+# GNU Affero General Public License for more details.
#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see .
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see .
from aiohttp import web
from mako.template import Template
import asyncio
@@ -46,7 +46,9 @@ class PublicBridgeWebsite:
user = (User.get_by_mxid(request.rel_url.query["mxid"], create=False)
if "mxid" in request.rel_url.query else None)
if not user:
- return self.render_login(mxid=request.rel_url.query["mxid"], state="request")
+ return self.render_login(
+ mxid=request.rel_url.query["mxid"] if "mxid" in request.rel_url.query else None,
+ state="request")
elif not user.whitelisted:
return self.render_login(mxid=user.mxid, error="You are not whitelisted.", status=403)
await user.ensure_started()
@@ -144,7 +146,7 @@ class PublicBridgeWebsite:
if "mxid" not in data:
return self.render_login(error="Please enter your Matrix ID.", status=400)
- user = await User.get_by_mxid(data["mxid"]).ensure_started()
+ user = await User.get_by_mxid(data["mxid"]).ensure_started(even_if_no_session=True)
if not user.whitelisted:
return self.render_login(mxid=user.mxid, error="You are not whitelisted.", status=403)
elif user.logged_in:
@@ -153,7 +155,8 @@ class PublicBridgeWebsite:
if "phone" in data:
return await self.post_login_phone(user, data["phone"])
elif "code" in data:
- resp = await self.post_login_code(user, data["code"], password_in_data="password" in data)
+ resp = await self.post_login_code(user, data["code"],
+ password_in_data="password" in data)
if resp or "password" not in data:
return resp
elif "password" not in data:
diff --git a/mautrix_telegram/public/login.css b/mautrix_telegram/public/login.css
index 95d461b8..c7ade95b 100644
--- a/mautrix_telegram/public/login.css
+++ b/mautrix_telegram/public/login.css
@@ -1,3 +1,20 @@
+/*
+ * mautrix-telegram - A Matrix-Telegram puppeting bridge
+ * Copyright (C) 2018 Tulir Asokan
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see .
+ */
form > div {
display: none;
}
diff --git a/mautrix_telegram/public/login.html.mako b/mautrix_telegram/public/login.html.mako
index 4da8d7d5..a63e30e3 100644
--- a/mautrix_telegram/public/login.html.mako
+++ b/mautrix_telegram/public/login.html.mako
@@ -1,3 +1,20 @@
+
diff --git a/mautrix_telegram/puppet.py b/mautrix_telegram/puppet.py
index 4340498a..de00a107 100644
--- a/mautrix_telegram/puppet.py
+++ b/mautrix_telegram/puppet.py
@@ -3,17 +3,17 @@
# Copyright (C) 2018 Tulir Asokan
#
# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
+# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
+# GNU Affero General Public License for more details.
#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see .
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see .
from difflib import SequenceMatcher
import re
import logging
@@ -191,6 +191,21 @@ class Puppet:
return None
+ @classmethod
+ def find_by_displayname(cls, displayname):
+ if not displayname:
+ return None
+
+ for _, puppet in cls.cache.items():
+ if puppet.displayname and puppet.displayname == displayname:
+ return puppet
+
+ puppet = DBPuppet.query.filter(DBPuppet.displayname == displayname).one_or_none()
+ if puppet:
+ return cls.from_db(puppet)
+
+ return None
+
def init(context):
global config
diff --git a/mautrix_telegram/tgclient.py b/mautrix_telegram/tgclient.py
index d70511a7..1b7aea85 100644
--- a/mautrix_telegram/tgclient.py
+++ b/mautrix_telegram/tgclient.py
@@ -3,17 +3,17 @@
# Copyright (C) 2018 Tulir Asokan
#
# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
+# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
+# GNU Affero General Public License for more details.
#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see .
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see .
from io import BytesIO
from telethon_aio import TelegramClient
@@ -73,6 +73,11 @@ class MautrixTelegramClient(TelegramClient):
reply_to_msg_id=reply_to)
return self._get_response_message(request, await self(request))
+ async def send_media(self, entity, media, caption=None, entities=None, reply_to=None):
+ request = SendMediaRequest(entity, media, message=caption or "", entities=entities or [],
+ reply_to_msg_id=reply_to)
+ return self._get_response_message(request, await self(request))
+
async def download_file_bytes(self, location):
if isinstance(location, Document):
location = InputDocumentFileLocation(location.id, location.access_hash,
diff --git a/mautrix_telegram/user.py b/mautrix_telegram/user.py
index 26f1d79e..2af7d46a 100644
--- a/mautrix_telegram/user.py
+++ b/mautrix_telegram/user.py
@@ -3,17 +3,17 @@
# Copyright (C) 2018 Tulir Asokan
#
# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
+# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
+# GNU Affero General Public License for more details.
#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see .
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see .
import logging
import asyncio
import re
diff --git a/mautrix_telegram/util/file_transfer.py b/mautrix_telegram/util/file_transfer.py
index 8c56113d..a1424987 100644
--- a/mautrix_telegram/util/file_transfer.py
+++ b/mautrix_telegram/util/file_transfer.py
@@ -3,27 +3,39 @@
# Copyright (C) 2018 Tulir Asokan
#
# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
+# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
+# GNU Affero General Public License for more details.
#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see .
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see .
from io import BytesIO
import time
import logging
import magic
-from PIL import Image
-from sqlalchemy.exc import IntegrityError
+from sqlalchemy.exc import IntegrityError, InvalidRequestError
+from sqlalchemy.orm.exc import FlushError
+try:
+ from PIL import Image
+except ImportError:
+ Image = None
+try:
+ from moviepy.editor import VideoFileClip
+ import random
+ import string
+ import os
+ import mimetypes
+except ImportError:
+ VideoFileClip = random = string = os = mimetypes = None
from telethon_aio.tl.types import (Document, FileLocation, InputFileLocation,
- InputDocumentFileLocation)
+ InputDocumentFileLocation, PhotoSize, PhotoCachedSize)
from telethon_aio.errors import LocationInvalidError
from ..db import TelegramFile as DBTelegramFile
@@ -32,24 +44,89 @@ log = logging.getLogger("mau.util")
def _convert_webp(file, to="png"):
+ if not Image:
+ return "image/webp", file
try:
image = Image.open(BytesIO(file)).convert("RGBA")
new_file = BytesIO()
image.save(new_file, to)
- return f"image/{to}", new_file.getvalue()
+ w, h = image.size
+ return f"image/{to}", new_file.getvalue(), w, h
except Exception:
log.exception(f"Failed to convert webp to {to}")
return "image/webp", file
-async def transfer_file_to_matrix(db, client, intent, location):
+def _temp_file_name(ext):
+ return ("/tmp/mxtg-video-"
+ + "".join(random.choice(string.ascii_uppercase + string.digits) for _ in range(10))
+ + ext)
+
+
+def _read_video_thumbnail(data, video_ext="mp4", frame_ext="png", max_size=(1024, 720)):
+ # We don't have any way to read the video from memory, so save it to disk.
+ temp_file = _temp_file_name(video_ext)
+ with open(temp_file, "wb") as file:
+ file.write(data)
+
+ # Read temp file and get frame
+ clip = VideoFileClip(temp_file)
+ frame = clip.get_frame(0)
+
+ # Convert to png and save to BytesIO
+ image = Image.fromarray(frame).convert("RGBA")
+ thumbnail_file = BytesIO()
+ if max_size:
+ image.thumbnail(max_size, Image.ANTIALIAS)
+ image.save(thumbnail_file, frame_ext)
+
+ os.remove(temp_file)
+
+ w, h = image.size
+ return thumbnail_file.getvalue(), w, h
+
+
+def _location_to_id(location):
if isinstance(location, (Document, InputDocumentFileLocation)):
- id = f"{location.id}-{location.version}"
+ return f"{location.id}-{location.version}"
elif isinstance(location, (FileLocation, InputFileLocation)):
- id = f"{location.volume_id}-{location.local_id}"
+ return f"{location.volume_id}-{location.local_id}"
else:
return None
+
+async def transfer_thumbnail_to_matrix(client, intent, thumbnail_loc, video, mime):
+ if not Image or not VideoFileClip:
+ return None
+
+ id = _location_to_id(thumbnail_loc)
+ if not id:
+ return None
+
+ video_ext = mimetypes.guess_extension(mime)
+ if VideoFileClip and video_ext:
+ try:
+ file, width, height = _read_video_thumbnail(video, video_ext, frame_ext="png")
+ except OSError:
+ return None
+ mime_type = "image/png"
+ else:
+ file = await client.download_file_bytes(thumbnail_loc)
+ width, height = None, None
+ mime_type = magic.from_buffer(file, mime=True)
+
+ uploaded = await intent.upload_file(file, mime_type)
+
+ return DBTelegramFile(id=id, mxc=uploaded["content_uri"], mime_type=mime_type,
+ was_converted=False, timestamp=int(time.time()), size=len(file),
+ width=width, height=height)
+
+
+async def transfer_file_to_matrix(db, client, intent, location, thumbnail=None):
+ id = _location_to_id(location)
+ if not id:
+ return None
+
db_file = DBTelegramFile.query.get(id)
if db_file:
return db_file
@@ -58,24 +135,37 @@ async def transfer_file_to_matrix(db, client, intent, location):
file = await client.download_file_bytes(location)
except LocationInvalidError:
return None
+ width, height = None, None
mime_type = magic.from_buffer(file, mime=True)
image_converted = False
if mime_type == "image/webp":
- mime_type, file = _convert_webp(file, to="png")
+ mime_type, file, width, height = _convert_webp(file, to="png")
+ thumbnail = None
image_converted = True
uploaded = await intent.upload_file(file, mime_type)
db_file = DBTelegramFile(id=id, mxc=uploaded["content_uri"],
mime_type=mime_type, was_converted=image_converted,
- timestamp=int(time.time()))
+ timestamp=int(time.time()), size=len(file),
+ width=width, height=height)
+ if thumbnail and (mime_type.startswith("video/") or mime_type == "image/gif"):
+ if isinstance(thumbnail, (PhotoSize, PhotoCachedSize)):
+ thumbnail = thumbnail.location
+ db_file.thumbnail = await transfer_thumbnail_to_matrix(client, intent, thumbnail, file,
+ mime_type)
+
try:
db.add(db_file)
db.commit()
- except IntegrityError:
+ except FlushError as e:
+ log.exception(f"{e.__class__.__name__} while saving transferred file data. "
+ "This was probably caused by two simultaneous transfers of the same file, "
+ "and should not cause any problems.")
+ except (IntegrityError, InvalidRequestError) as e:
db.rollback()
- log.exception("Integrity error while saving transferred file data. "
+ log.exception(f"{e.__class__.__name__} while saving transferred file data. "
"This was probably caused by two simultaneous transfers of the same file, "
"and should not cause any problems.")
diff --git a/mautrix_telegram/util/format_duration.py b/mautrix_telegram/util/format_duration.py
index ffbac714..c873e9e5 100644
--- a/mautrix_telegram/util/format_duration.py
+++ b/mautrix_telegram/util/format_duration.py
@@ -3,17 +3,17 @@
# Copyright (C) 2018 Tulir Asokan
#
# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU General Public License as published by
+# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
-# GNU General Public License for more details.
+# GNU Affero General Public License for more details.
#
-# You should have received a copy of the GNU General Public License
-# along with this program. If not, see .
+# You should have received a copy of the GNU Affero General Public License
+# along with this program. If not, see .
def format_duration(seconds):
diff --git a/requirements/base.txt b/requirements/base.txt
index e16861af..6e955265 100644
--- a/requirements/base.txt
+++ b/requirements/base.txt
@@ -1,9 +1,8 @@
aiohttp
+mautrix-appservice
ruamel.yaml
python-magic
SQLAlchemy
alembic
Markdown
-Pillow
future-fstrings
-cryptg
diff --git a/requirements/optional.txt b/requirements/optional.txt
new file mode 100644
index 00000000..a4400877
--- /dev/null
+++ b/requirements/optional.txt
@@ -0,0 +1,4 @@
+lxml
+cryptg
+Pillow
+moviepy
diff --git a/setup.py b/setup.py
index dbc732ed..5166748e 100644
--- a/setup.py
+++ b/setup.py
@@ -3,6 +3,14 @@ import sys
import glob
import mautrix_telegram
+extras = {
+ "highlight_edits": ["lxml>=4.1.1,<5"],
+ "fast_crypto": ["cryptg>=0.1,<0.2"],
+ "webp_convert": ["Pillow>=5.0.0,<6"],
+ "hq_thumbnails": ["moviepy>=0.2,<0.3"],
+}
+extras["all"] = [deps[0] for deps in extras.values()]
+
setuptools.setup(
name="mautrix-telegram",
version=mautrix_telegram.__version__,
@@ -18,22 +26,22 @@ setuptools.setup(
install_requires=[
"aiohttp>=3.0.1,<4",
+ "mautrix-telegram>=0.1,<0.2",
"SQLAlchemy>=1.2.3,<2",
"alembic>=0.9.8,<0.10",
"Markdown>=2.6.11,<3",
"ruamel.yaml>=0.15.35,<0.16",
- "Pillow>=5.0.0,<6",
"future-fstrings>=0.4.2",
"python-magic>=0.4.15,<0.5",
- "cryptg>=0.1,<0.2",
"telethon-aio>=0.18,<0.19" if sys.version_info >= (3, 6) else "telethon-aio-git",
],
dependency_links=[
"https://github.com/tulir/telethon-asyncio/tarball/9b389cfb4b6d3876e9661c23507f17e96897e4b0#egg=telethon-aio-git-0.18.0+1"
],
+ extras_require=extras,
classifiers=[
- "Development Status :: 4 Beta",
+ "Development Status :: 4 - Beta",
"License :: OSI Approved :: GNU General Public License v3 or later (GPLv3+)",
"Topic :: Communications :: Chat",
"Programming Language :: Python",