Bump version to 0.6.0rc1

Allow config key override through env var

.gitignore

@@ -5,6 +5,10 @@ env/
 pip-selfcheck.json
 *.pyc
 __pycache__
+build
+dist
+*.egg-info
+.eggs
 
 config.yaml
 registration.yaml

Dockerfile

@@ -10,7 +10,6 @@ RUN apk add --no-cache \
       py3-virtualenv \
       py3-pillow \
       py3-aiohttp \
-      py3-lxml \
       py3-magic \
       py3-sqlalchemy \
       py3-markdown \

ROADMAP.md

@@ -3,6 +3,7 @@
 * Matrix → Telegram
   * [x] Message content (text, formatting, files, etc..)
   * [x] Message redactions
+  * [x] Message edits
   * [ ] ‡ Message history
   * [x] Presence
   * [x] Typing notifications

alembic/versions/17574c57f3f8_add_disable_updates_field_for_puppets.py

@@ -0,0 +1,27 @@
+"""Add disable_updates field for puppets
+
+Revision ID: 17574c57f3f8
+Revises: a9119be92164
+Create Date: 2019-05-15 00:24:46.967529
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '17574c57f3f8'
+down_revision = 'a9119be92164'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    with op.batch_alter_table("puppet") as batch_op:
+        batch_op.add_column(sa.Column("disable_updates", sa.Boolean(), nullable=False,
+                                      server_default=sa.sql.expression.false()))
+
+
+def downgrade():
+    with op.batch_alter_table("puppet") as batch_op:
+        batch_op.drop_column("disable_updates")

alembic/versions/1fa46383a9d3_add_is_bot_field_to_puppets.py

@@ -17,7 +17,8 @@ depends_on = None
 
 
 def upgrade():
-    op.add_column('puppet', sa.Column('is_bot', sa.Boolean(), nullable=True))
+    with op.batch_alter_table("puppet") as batch_op:
+        batch_op.add_column(sa.Column('is_bot', sa.Boolean(), nullable=True))
 
 
 def downgrade():

alembic/versions/30eca60587f1_add_megagroup_field_to_portals.py

@@ -16,7 +16,8 @@ depends_on = None
 
 
 def upgrade():
-    op.add_column('portal', sa.Column('megagroup', sa.Boolean()))
+    with op.batch_alter_table("portal") as batch_op:
+        batch_op.add_column(sa.Column('megagroup', sa.Boolean()))
 
 
 def downgrade():

alembic/versions/6ca3d74d51e4_move_state_store_to_main_database.py

@@ -57,7 +57,8 @@ class Puppet(Base):
 
 
 def upgrade():
-    op.add_column("puppet", sa.Column("matrix_registered", sa.Boolean(), nullable=False,
+    with op.batch_alter_table("puppet") as batch_op:
+        batch_op.add_column(sa.Column("matrix_registered", sa.Boolean(), nullable=False,
                                       server_default=sa.sql.expression.false()))
     op.create_table("mx_room_state",
                     sa.Column("room_id", sa.String(), nullable=False),

alembic/versions/9e9c89b0b877_add_edit_index_to_messages.py

@@ -0,0 +1,48 @@
+"""Add edit index to messages
+
+Revision ID: 9e9c89b0b877
+Revises: 17574c57f3f8
+Create Date: 2019-05-29 15:28:23.128377
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = '9e9c89b0b877'
+down_revision = '17574c57f3f8'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table('_message_temp',
+                    sa.Column('mxid', sa.String),
+                    sa.Column('mx_room', sa.String),
+                    sa.Column('tgid', sa.Integer),
+                    sa.Column('tg_space', sa.Integer),
+                    sa.Column('edit_index', sa.Integer),
+                    sa.PrimaryKeyConstraint('tgid', 'tg_space', 'edit_index'),
+                    sa.UniqueConstraint("mxid", "mx_room", "tg_space", name="_mx_id_room_2"))
+    c = op.get_bind()
+    c.execute("INSERT INTO _message_temp (mxid, mx_room, tgid, tg_space, edit_index) "
+              "SELECT message.mxid, message.mx_room, message.tgid, message.tg_space, 0 "
+              "FROM message")
+    c.execute("DROP TABLE message")
+    c.execute("ALTER TABLE _message_temp RENAME TO message")
+
+
+def downgrade():
+    op.create_table('_message_temp',
+                    sa.Column('mxid', sa.String),
+                    sa.Column('mx_room', sa.String),
+                    sa.Column('tgid', sa.Integer),
+                    sa.Column('tg_space', sa.Integer),
+                    sa.PrimaryKeyConstraint('tgid', 'tg_space'),
+                    sa.UniqueConstraint("mxid", "mx_room", "tg_space", name="_mx_id_room"))
+    c = op.get_bind()
+    c.execute("INSERT INTO _message_temp (mxid, mx_room, tgid, tg_space) "
+              "SELECT message.mxid, message.mx_room, message.tgid, message.tg_space "
+              "FROM message")
+    c.execute("DROP TABLE message")
+    c.execute("ALTER TABLE _message_temp RENAME TO message")

example-config.yaml

@@ -64,6 +64,11 @@ appservice:
     as_token: "This value is generated when generating the registration"
     hs_token: "This value is generated when generating the registration"
 
+# Prometheus telemetry config. Requires prometheus-aio to be installed.
+metrics:
+    enabled: false
+    listen_port: 8000
+
 # Bridge config
 bridge:
     # Localpart template of MXIDs for Telegram users.
@@ -126,15 +131,11 @@ bridge:
     # Only enable this if your displayname_template has some static part that the bridge can use to
     # reliably identify what is a plaintext highlight.
     plaintext_highlights: false
-    # Show message editing as a reply to the original message.
-    # If this is false, message edits are not shown at all, as Matrix does not support editing yet.
-    edits_as_replies: true
-    # Highlight changed/added parts in edits. Requires lxml.
-    highlight_edits: false
     # Whether or not to make portals of publicly joinable channels/supergroups publicly joinable on Matrix.
     public_portals: true
     # Whether or not to fetch and handle Telegram updates at startup from the time the bridge was down.
     # Currently only works for private chats and normal groups.
+    # WARNING: This feature seems to be broken in the telegram library.
     catch_up: false
     # Whether or not to use /sync to get presence, read receipts and typing notifications when using
     # your own Matrix account as the Matrix puppet for your Telegram account.
@@ -146,6 +147,8 @@ bridge:
     inline_images: false
     # Maximum size of image in megabytes before sending to Telegram as a document.
     image_as_file_size: 10
+    # Maximum size of Telegram documents in megabytes to bridge.
+    max_document_size: 100
 
     # Whether to bridge Telegram bot messages as m.notices or m.texts.
     bot_messages_as_notices: true
@@ -251,6 +254,40 @@ telegram:
     api_hash: tjyd5yge35lbodk1xwzw2jstp90k55qz
     # (Optional) Create your own bot at https://t.me/BotFather
     bot_token: disabled
+
+    # Telethon connection options.
+    connection:
+        # The timeout in seconds to be used when connecting.
+        timeout: 120
+        # How many times the reconnection should retry, either on the initial connection or when
+        # Telegram disconnects us. May be set to a negative or null value for infinite retries, but
+        # this is not recommended, since the program can get stuck in an infinite loop.
+        retries: 5
+        # The delay in seconds to sleep between automatic reconnections.
+        retry_delay: 1
+        # The threshold below which the library should automatically sleep on flood wait errors
+        # (inclusive). For instance, if a FloodWaitError for 17s occurs and flood_sleep_threshold
+        # is 20s, the library will sleep automatically. If the error was for 21s, it would raise
+        # the error instead. Values larger than a day (86400) will be changed to a day.
+        flood_sleep_threshold: 60
+        # How many times a request should be retried. Request are retried when Telegram is having
+        # internal issues, when there is a FloodWaitError less than flood_sleep_threshold, or when
+        # there's a migrate error. May take a negative or null value for infinite retries, but this
+        # is not recommended, since some requests can always trigger a call fail (such as searching
+        # for messages).
+        request_retries: 5
+
+    # Device info sent to Telegram.
+    device_info:
+        # "auto" = OS name+version.
+        device_model: auto
+        # "auto" = Telethon version.
+        system_version: auto
+        # "auto" = mautrix-telegram version.
+        app_version: auto
+        lang_code: en
+        system_lang_code: en
+
     # Custom server to connect to.
     server:
         # Set to true to use these server settings. If false, will automatically
@@ -262,6 +299,7 @@ telegram:
         ip: 149.154.167.40
         # The port to connect to. 443 may not work, 80 is better and both are equally secure.
         port: 80
+
     # Telethon proxy configuration.
     # You must install PySocks from pip for proxies to work.
     proxy:

mautrix_telegram/__init__.py

@@ -1,2 +1,2 @@
-__version__ = "0.5.0"
+__version__ = "0.6.0rc1"
 __author__ = "Tulir Asokan <tulir@maunium.net>"

mautrix_telegram/__main__.py

@@ -22,6 +22,7 @@ import logging.config
 import sys
 import copy
 import signal
+import os
 
 import sqlalchemy as sql
 
@@ -43,6 +44,11 @@ from .sqlstatestore import SQLStateStore
 from .user import User, init as init_user
 from . import __version__
 
+try:
+    import prometheus_client as prometheus
+except ImportError:
+    prometheus = None
+
 parser = argparse.ArgumentParser(
     description="A Matrix-Telegram puppeting bridge.",
     prog="python -m mautrix-telegram")
@@ -57,7 +63,7 @@ parser.add_argument("-r", "--registration", type=str, default="registration.yaml
                     metavar="<path>", help="the path to save the generated registration to")
 args = parser.parse_args()
 
-config = Config(args.config, args.registration, args.base_config)
+config = Config(args.config, args.registration, args.base_config, os.environ)
 config.load()
 config.update()
 
@@ -114,6 +120,12 @@ if config["appservice.provisioning.enabled"]:
 
 context.mx = MatrixHandler(context)
 
+if config["metrics.enabled"]:
+    if prometheus:
+        prometheus.start_http_server(config["metrics.listen_port"])
+    else:
+        log.warn("Metrics are enabled in the config, but prometheus-async is not installed.")
+
 with appserv.run(config["appservice.hostname"], config["appservice.port"]) as start:
     start_ts = time()
     init_db(db_engine)

mautrix_telegram/abstract_user.py

@@ -14,11 +14,12 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Tuple, Optional, List, Union, TYPE_CHECKING
+from typing import Tuple, Optional, List, Union, Dict, TYPE_CHECKING
 from abc import ABC, abstractmethod
 import asyncio
 import logging
 import platform
+import time
 
 from telethon.tl.patched import MessageService, Message
 from telethon.tl.types import (
@@ -50,6 +51,14 @@ UpdateMessage = Union[UpdateShortChatMessage, UpdateShortMessage, UpdateNewChann
                       UpdateNewMessage, UpdateEditMessage, UpdateEditChannelMessage]
 UpdateMessageContent = Union[UpdateShortMessage, UpdateShortChatMessage, Message, MessageService]
 
+try:
+    from prometheus_client import Histogram
+
+    UPDATE_TIME = Histogram("telegram_update", "Time spent processing Telegram updates",
+                            ["update_type"])
+except ImportError:
+    Histogram = None
+    UPDATE_TIME = None
 
 class AbstractUser(ABC):
     session_container = None  # type: AlchemySessionContainer
@@ -95,27 +104,43 @@ class AbstractUser(ABC):
 
     def _init_client(self) -> None:
         self.log.debug(f"Initializing client for {self.name}")
-        device = f"{platform.system()} {platform.release()}"
-        sysversion = MautrixTelegramClient.__version__
+
         self.session = self.session_container.new_session(self.name)
         if config["telegram.server.enabled"]:
             self.session.set_dc(config["telegram.server.dc"],
                                 config["telegram.server.ip"],
                                 config["telegram.server.port"])
+
         if self.is_relaybot:
             base_logger = logging.getLogger("telethon.relaybot")
         else:
             base_logger = logging.getLogger(f"telethon.{self.tgid or -hash(self.mxid)}")
-        self.client = MautrixTelegramClient(session=self.session,
-                                            api_id=config["telegram.api_id"],
-                                            api_hash=config["telegram.api_hash"],
-                                            loop=self.loop,
-                                            app_version=__version__,
-                                            system_version=sysversion,
-                                            device_model=device,
-                                            timeout=120,
-                                            base_logger=base_logger,
-                                            proxy=self._proxy_settings)
+
+        device = config["telegram.device_info.device_model"]
+        sysversion = config["telegram.device_info.system_version"]
+        appversion = config["telegram.device_info.app_version"]
+
+        self.client = MautrixTelegramClient(
+            session=self.session,
+
+            api_id=config["telegram.api_id"],
+            api_hash=config["telegram.api_hash"],
+
+            app_version=__version__ if appversion == "auto" else appversion,
+            system_version=MautrixTelegramClient.__version__ if sysversion == "auto" else sysversion,
+            device_model=f"{platform.system()} {platform.release()}" if device == "auto" else device,
+
+            timeout=config["telegram.connection.timeout"],
+            connection_retries=config["telegram.connection.retries"],
+            retry_delay=config["telegram.connection.retry_delay"],
+            flood_sleep_threshold=config["telegram.connection.flood_sleep_threshold"],
+            request_retries=config["telegram.connection.request_retries"],
+
+            proxy=self._proxy_settings,
+
+            loop=self.loop,
+            base_logger=base_logger
+        )
         self.client.add_event_handler(self._update_catch)
 
     @abstractmethod
@@ -135,11 +160,14 @@ class AbstractUser(ABC):
         raise NotImplementedError()
 
     async def _update_catch(self, update: TypeUpdate) -> None:
+        start_time = time.time()
         try:
             if not await self.update(update):
                 await self._update(update)
         except Exception:
             self.log.exception("Failed to handle Telegram update")
+        if UPDATE_TIME:
+            UPDATE_TIME.labels(update_type=type(update).__name__).observe(time.time() - start_time)
 
     async def get_dialogs(self, limit: int = None) -> List[Union[Chat, Channel]]:
         if self.is_bot:
@@ -185,6 +213,8 @@ class AbstractUser(ABC):
     # region Telegram update handling
 
     async def _update(self, update: TypeUpdate) -> None:
+        asyncio.ensure_future(self._handle_entity_updates(getattr(update, "_entities", {})),
+                              loop=self.loop)
         if isinstance(update, (UpdateShortChatMessage, UpdateShortMessage, UpdateNewChannelMessage,
                                UpdateNewMessage, UpdateEditMessage, UpdateEditChannelMessage)):
             await self.update_message(update)
@@ -234,7 +264,7 @@ class AbstractUser(ABC):
             return
 
         # We check that these are user read receipts, so tg_space is always the user ID.
-        message = DBMessage.get_by_tgid(TelegramID(update.max_id), self.tgid)
+        message = DBMessage.get_one_by_tgid(TelegramID(update.max_id), self.tgid, edit_index=-1)
         if not message:
             return
 
@@ -261,6 +291,16 @@ class AbstractUser(ABC):
         sender = pu.Puppet.get(TelegramID(update.user_id))
         await portal.handle_telegram_typing(sender, update)
 
+    async def _handle_entity_updates(self, entities: Dict[int, Union[User, Chat, Channel]]
+                                     ) -> None:
+        try:
+            users = (entity for entity in entities.values() if isinstance(entity, User))
+            puppets = ((pu.Puppet.get(TelegramID(user.id)), user) for user in users)
+            await asyncio.gather(*[puppet.update_info(self, info)
+                                   for puppet, info in puppets if puppet])
+        except Exception:
+            self.log.exception("Failed to handle entity updates")
+
     async def update_others_info(self, update: Union[UpdateUserName, UpdateUserPhoto]) -> None:
         # TODO duplication not checked
         puppet = pu.Puppet.get(TelegramID(update.user_id))
@@ -269,7 +309,7 @@ class AbstractUser(ABC):
             if await puppet.update_displayname(self, update):
                 puppet.save()
         elif isinstance(update, UpdateUserPhoto):
-            if await puppet.update_avatar(self, update.photo.photo_big):
+            if await puppet.update_avatar(self, update.photo):
                 puppet.save()
         else:
             self.log.warning("Unexpected other user info update: %s", update)
@@ -309,7 +349,8 @@ class AbstractUser(ABC):
         return update, sender, portal
 
     @staticmethod
-    async def _try_redact(portal: po.Portal, message: DBMessage) -> None:
+    async def _try_redact(message: DBMessage) -> None:
+        portal = po.Portal.get_by_mxid(message.mx_room)
         if not portal:
             return
         try:
@@ -321,30 +362,26 @@ class AbstractUser(ABC):
         if len(update.messages) > MAX_DELETIONS:
             return
 
-        for message in update.messages:
-            message = DBMessage.get_by_tgid(TelegramID(message), self.tgid)
-            if not message:
-                continue
-            message.delete()
-            number_left = DBMessage.count_spaces_by_mxid(message.mxid, message.mx_room)
-            if number_left == 0:
-                portal = po.Portal.get_by_mxid(message.mx_room)
-                await self._try_redact(portal, message)
+        for message_id in update.messages:
+            messages = DBMessage.get_all_by_tgid(TelegramID(message_id), self.tgid)
+            for message in messages:
+                message.delete()
+                number_left = DBMessage.count_spaces_by_mxid(message.mxid, message.mx_room)
+                if number_left == 0:
+                    portal = po.Portal.get_by_mxid(message.mx_room)
+                    await self._try_redact(message)
 
     async def delete_channel_message(self, update: UpdateDeleteChannelMessages) -> None:
         if len(update.messages) > MAX_DELETIONS:
             return
 
-        portal = po.Portal.get_by_tgid(TelegramID(update.channel_id))
-        if not portal:
-            return
+        channel_id = TelegramID(update.channel_id)
 
-        for message in update.messages:
-            message = DBMessage.get_by_tgid(TelegramID(message), portal.tgid)
-            if not message:
-                continue
-            message.delete()
-            await self._try_redact(portal, message)
+        for message_id in update.messages:
+            messages = DBMessage.get_all_by_tgid(TelegramID(message_id), channel_id)
+            for message in messages:
+                message.delete()
+                await self._try_redact(message)
 
     async def update_message(self, original_update: UpdateMessage) -> None:
         update, sender, portal = self.get_message_details(original_update)
@@ -370,10 +407,7 @@ class AbstractUser(ABC):
 
         user = sender.tgid if sender else "admin"
         if isinstance(original_update, (UpdateEditMessage, UpdateEditChannelMessage)):
-            if config["bridge.edits_as_replies"]:
-                self.log.debug("Handling edit %s to %s by %s", update, portal.tgid_log, user)
-                return await portal.handle_telegram_edit(self, sender, update)
-            return
+            return await portal.handle_telegram_edit(self, sender, update)
 
         self.log.debug("Handling message %s to %s by %s", update, portal.tgid_log, user)
         return await portal.handle_telegram_message(self, sender, update)

mautrix_telegram/bot.py

@@ -14,7 +14,7 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Awaitable, Callable, Dict, List, Optional, Pattern, TYPE_CHECKING
+from typing import Awaitable, Callable, Dict, List, Optional, Pattern, Tuple, TYPE_CHECKING
 import logging
 import re
 
@@ -23,7 +23,7 @@ from telethon.tl.types import (
     ChannelParticipantAdmin, ChannelParticipantCreator, ChatForbidden, ChatParticipantAdmin,
     ChatParticipantCreator, InputChannel, InputUser, MessageActionChatAddUser,
     MessageActionChatDeleteUser, MessageEntityBotCommand, PeerChannel, PeerChat, TypePeer,
-    UpdateNewChannelMessage, UpdateNewMessage, MessageActionChatMigrateTo)
+    UpdateNewChannelMessage, UpdateNewMessage, MessageActionChatMigrateTo, User)
 from telethon.tl.functions.messages import GetChatsRequest, GetFullChatRequest
 from telethon.tl.functions.channels import GetChannelsRequest, GetParticipantRequest
 from telethon.errors import ChannelInvalidError, ChannelPrivateError
@@ -60,6 +60,14 @@ class Bot(AbstractUser):
         self.tg_whitelist = []  # type: List[int]
         self.whitelist_group_admins = (config["bridge.relaybot.whitelist_group_admins"]
                                        or False)  # type: bool
+        self._me_info = None  # type: Optional[User]
+        self._me_mxid = None  # type: Optional[MatrixUserID]
+
+    async def get_me(self, use_cache: bool = True) -> Tuple[User, MatrixUserID]:
+        if not use_cache or not self._me_mxid:
+            self._me_info = await self.client.get_me()
+            self._me_mxid = pu.Puppet.get_mxid_from_id(TelegramID(self._me_info.id))
+        return self._me_info, self._me_mxid
 
     async def init_permissions(self) -> None:
         whitelist = config["bridge.relaybot.whitelist"] or []

mautrix_telegram/commands/handler.py

@@ -285,15 +285,10 @@ class CommandHandler:
 
 
 def command_handler(_func: Optional[Callable[[CommandEvent], Awaitable[Dict]]] = None, *,
-                    needs_auth: bool = True,
-                    needs_puppeting: bool = True,
-                    needs_matrix_puppeting: bool = False,
-                    needs_admin: bool = False,
-                    management_only: bool = False,
-                    name: Optional[str] = None,
-                    help_text: str = "",
-                    help_args: str = "",
-                    help_section: HelpSection = None,
+                    needs_auth: bool = True, needs_puppeting: bool = True,
+                    needs_matrix_puppeting: bool = False, needs_admin: bool = False,
+                    management_only: bool = False, name: Optional[str] = None,
+                    help_text: str = "", help_args: str = "", help_section: HelpSection = None
                     ) -> Callable[[Callable[[CommandEvent], Awaitable[Optional[Dict]]]],
                                   CommandHandler]:
     def decorator(func: Callable[[CommandEvent], Awaitable[Optional[Dict]]]) -> CommandHandler:

mautrix_telegram/commands/portal/bridge.py

@@ -36,6 +36,10 @@ async def bridge(evt: CommandEvent) -> Dict:
     if len(evt.args) == 0:
         return await evt.reply("**Usage:** "
                                "`$cmdprefix+sp bridge <Telegram chat ID> [Matrix room ID]`")
+    force_use_bot = False
+    if evt.args[0] == "--usebot" and evt.sender.is_admin:
+        force_use_bot = True
+        evt.args = evt.args[1:]
     room_id = MatrixRoomID(evt.args[1]) if len(evt.args) > 1 else evt.room_id
     that_this = "This" if room_id == evt.room_id else "That"
 
@@ -80,6 +84,7 @@ async def bridge(evt: CommandEvent) -> Dict:
             "bridge_to_mxid": room_id,
             "tgid": portal.tgid,
             "peer_type": portal.peer_type,
+            "force_use_bot": force_use_bot,
         }
         return await evt.reply(f"{has_portal_message}"
                                "However, you have the permissions to unbridge that room.\n\n"
@@ -93,6 +98,7 @@ async def bridge(evt: CommandEvent) -> Dict:
         "bridge_to_mxid": room_id,
         "tgid": portal.tgid,
         "peer_type": portal.peer_type,
+        "force_use_bot": force_use_bot,
     }
     return await evt.reply("That Telegram chat has no existing portal. To confirm bridging the "
                            "chat to this room, use `$cmdprefix+sp continue`")
@@ -149,7 +155,7 @@ async def confirm_bridge(evt: CommandEvent) -> Optional[Dict]:
                                "`$cmdprefix+sp cancel` to cancel.")
 
     evt.sender.command_status = None
-    is_logged_in = await evt.sender.is_logged_in()
+    is_logged_in = await evt.sender.is_logged_in() and not status["force_use_bot"]
     user = evt.sender if is_logged_in else evt.tgbot
     try:
         entity = await user.client.get_entity(portal.peer)

mautrix_telegram/commands/portal/config.py

@@ -77,7 +77,6 @@ def config_view(evt: CommandEvent, portal: po.Portal) -> Awaitable[Dict]:
 def config_defaults(evt: CommandEvent) -> Awaitable[Dict]:
     stream = StringIO()
     yaml.dump({
-        "edits_as_replies": evt.config["bridge.edits_as_replies"],
         "bridge_notices": {
             "default": evt.config["bridge.bridge_notices.default"],
             "exceptions": evt.config["bridge.bridge_notices.exceptions"],

mautrix_telegram/commands/telegram/account.py

@@ -17,10 +17,10 @@
 from typing import Dict, Optional
 
 from telethon.errors import (UsernameInvalidError, UsernameNotModifiedError, UsernameOccupiedError,
-                             HashInvalidError, AuthKeyError)
+                             HashInvalidError, AuthKeyError, FirstNameInvalidError)
 from telethon.tl.types import Authorization
 from telethon.tl.functions.account import (UpdateUsernameRequest, GetAuthorizationsRequest,
-                                           ResetAuthorizationRequest)
+                                           ResetAuthorizationRequest, UpdateProfileRequest)
 
 from .. import command_handler, CommandEvent, SECTION_AUTH
 
@@ -53,6 +53,25 @@ async def username(evt: CommandEvent) -> Optional[Dict]:
         await evt.reply(f"Username changed to {evt.sender.username}")
 
 
+@command_handler(needs_auth=True, help_section=SECTION_AUTH, help_args="<_new displayname_>",
+                 help_text="Change your Telegram displayname.")
+async def displayname(evt: CommandEvent) -> Optional[Dict]:
+    if len(evt.args) == 0:
+        return await evt.reply("**Usage:** `$cmdprefix+sp displayname <new displayname>`")
+    if evt.sender.is_bot:
+        return await evt.reply("Bots can't set their own displayname.")
+
+    first_name, last_name = ((evt.args[0], "")
+                             if len(evt.args) == 1
+                             else (" ".join(evt.args[:-1]), evt.args[-1]))
+    try:
+        await evt.sender.client(UpdateProfileRequest(first_name=first_name, last_name=last_name))
+    except FirstNameInvalidError:
+        return await evt.reply("Invalid first name")
+    await evt.sender.update_info()
+    await evt.reply("Displayname updated")
+
+
 def _format_session(sess: Authorization) -> str:
     return (f"**{sess.app_name} {sess.app_version}**  \n"
             f"  **Platform:** {sess.device_model} {sess.platform} {sess.system_version}  \n"
@@ -87,9 +106,7 @@ async def session(evt: CommandEvent) -> Optional[Dict]:
         try:
             session_hash = int(evt.args[1])
         except ValueError:
-            return await evt.reply("Hash must be a positive integer")
-        if session_hash <= 0:
-            return await evt.reply("Hash must be a positive integer")
+            return await evt.reply("Hash must be an integer")
         try:
             ok = await evt.sender.client(ResetAuthorizationRequest(hash=session_hash))
         except HashInvalidError:

mautrix_telegram/commands/telegram/auth.py

@@ -33,8 +33,8 @@ from ...util import format_duration, ignore_coro
                  help_text="Check if you're logged into Telegram.")
 async def ping(evt: CommandEvent) -> Optional[Dict]:
     me = await evt.sender.client.get_me() if await evt.sender.is_logged_in() else None
-    human_tg_id = f"@{me.username}" if me.username else f"+{me.phone}"
     if me:
+        human_tg_id = f"@{me.username}" if me.username else f"+{me.phone}"
         return await evt.reply(f"You're logged in as {human_tg_id}")
     else:
         return await evt.reply("You're not logged in.")
@@ -46,11 +46,9 @@ async def ping(evt: CommandEvent) -> Optional[Dict]:
 async def ping_bot(evt: CommandEvent) -> Optional[Dict]:
     if not evt.tgbot:
         return await evt.reply("Telegram message relay bot not configured.")
-    bot_info = await evt.tgbot.client.get_me()
-    mxid = pu.Puppet.get_mxid_from_id(bot_info.id)
-    displayname = bot_info.first_name
+    info, mxid = await evt.tgbot.get_me(use_cache=False)
     return await evt.reply("Telegram message relay bot is active: "
-                           f"[{displayname}](https://matrix.to/#/{mxid}) (ID {bot_info.id})\n\n"
+                           f"[{info.first_name}](https://matrix.to/#/{mxid}) (ID {info.id})\n\n"
                            "To use the bot, simply invite it to a portal room.")
 
 
@@ -126,41 +124,20 @@ async def login(evt: CommandEvent) -> Optional[Dict]:
     if evt.config["appservice.public.enabled"]:
         prefix = evt.config["appservice.public.external"]
         url = f"{prefix}/login?token={evt.public_website.make_token(evt.sender.mxid, '/login')}"
-        if allow_matrix_login:
-            if override_sender:
-                return await evt.reply(
-                    "This bridge instance allows you to log in inside or outside of Matrix, but "
-                    "logging in as another user is only possible via the web interface.\n\n"
-                    f"Please visit [the login page]({url}) to log in as "
-                    f"[{evt.sender.mxid}](https://matrix.to/#/{evt.sender.mxid}).\n\n")
-            return await evt.reply(
-                "This bridge instance allows you to log in inside or outside Matrix.\n\n"
-                "If you would like to log in within Matrix, please send your phone number or bot "
-                "auth token here.\n"
-                "If you would like to log in outside of Matrix, please visit [the login page]"
-                f"({url}).\n\n"
-                "Logging in outside of Matrix is recommended if you have two-factor authentication "
-                "enabled, because in-Matrix login would save your password in the message history."
-                f"\n\n{nb}")
         if override_sender:
-            return await evt.reply(
-                "This bridge instance does not allow logging in inside Matrix, and logging in as "
-                "another user inside Matrix isn't possible anyway.\n\n"
-                f"Please visit [the login page]({url}) to log in as "
-                f"[{evt.sender.mxid}](https://matrix.to/#/{evt.sender.mxid}).")
-        return await evt.reply(
-            "This bridge instance does not allow logging in inside Matrix.\n\n"
-            f"Please visit [the login page]({url}) to log in.\n\n"
-            f"{nb}")
+            return await evt.reply(f"[Click here to log in]({url}) as "
+                                   f"[{evt.sender.mxid}](https://matrix.to/#/{evt.sender.mxid}).")
+        elif allow_matrix_login:
+            return await evt.reply(f"[Click here to log in]({url}). Alternatively, send your phone"
+                                   f" number (or bot auth token) here to log in.\n\n{nb}")
+        return await evt.reply(f"[Click here to log in]({url}).\n\n{nb}")
     elif allow_matrix_login:
         if override_sender:
             return await evt.reply(
                 "This bridge instance does not allow you to log in outside of Matrix. "
                 "Logging in as another user inside Matrix is not currently possible.")
-        return await evt.reply(
-            "This bridge instance does not allow you to log in outside of Matrix.\n\n"
-            "Please send your phone number or bot auth token here to start the login process.\n\n"
-            f"{nb}")
+        return await evt.reply("Please send your phone number (or bot auth token) here to start "
+                               f"the login process.\n\n{nb}")
     return await evt.reply("This bridge instance has been configured to not allow logging in.")
 
 

mautrix_telegram/commands/telegram/misc.py

@@ -191,7 +191,7 @@ async def _parse_encoded_msgid(user: AbstractUser, enc_id: str, type_name: str
         raise MessageIDError(f"Invalid {type_name} ID (format)") from e
 
     if peer_type == PEER_TYPE_CHAT:
-        orig_msg = DBMessage.get_by_tgid(msg_id, space)
+        orig_msg = DBMessage.get_one_by_tgid(msg_id, space)
         if not orig_msg:
             raise MessageIDError(f"Invalid {type_name} ID (original message not found in db)")
         new_msg = DBMessage.get_by_mxid(orig_msg.mxid, orig_msg.mx_room, user.tgid)
@@ -237,11 +237,11 @@ async def play(evt: CommandEvent) -> Optional[Dict]:
 
 
 @command_handler(help_section=SECTION_MISC,
-                 help_args="<_poll ID_> <_choice ID_>",
+                 help_args="<_poll ID_> <_choice number_>",
                  help_text="Vote in a Telegram poll.")
 async def vote(evt: CommandEvent) -> Optional[Dict]:
-    if len(evt.args) < 2:
-        return await evt.reply("**Usage:** `$cmdprefix+sp vote <poll ID> <choice ID>`")
+    if len(evt.args) < 1:
+        return await evt.reply("**Usage:** `$cmdprefix+sp vote <poll ID> <choice number>`")
     elif not await evt.sender.is_logged_in():
         return await evt.reply("You must be logged in with a real account to vote in polls.")
     elif evt.sender.is_bot:
@@ -255,7 +255,25 @@ async def vote(evt: CommandEvent) -> Optional[Dict]:
     if not isinstance(msg.media, MessageMediaPoll):
         return await evt.reply("Invalid poll ID (message doesn't look like a poll)")
 
-    options = [base64.b64decode(option + (3 - (len(option) + 3) % 4) * "=")
+    options = []
+    for option in evt.args[1:]:
+        try:
+            if len(option) > 10:
+                raise ValueError("option index too long")
+            option_index = int(option) - 1
+        except ValueError:
+            option_index = None
+        if option_index is None:
+            return await evt.reply(f"Invalid option number \"{option}\"",
+                                   render_markdown=False, allow_html=False)
+        elif option_index < 0:
+            return await evt.reply(f"Invalid option number {option}. "
+                                   f"Option numbers must be positive.")
+        elif option_index >= len(msg.media.poll.answers):
+            return await evt.reply(f"Invalid option number {option}. "
+                                   f"The poll only has {len(msg.media.poll.answers)} options.")
+        options.append(msg.media.poll.answers[option_index].option)
+    options = [msg.media.poll.answers[int(option) - 1].option
                for option in evt.args[1:]]
     try:
         resp = await evt.sender.client(SendVoteRequest(peer=peer, msg_id=msg.id, options=options))

mautrix_telegram/config.py

@@ -103,12 +103,20 @@ class DictWithRecursion:
 
 
 class Config(DictWithRecursion):
-    def __init__(self, path: str, registration_path: str, base_path: str) -> None:
+    def __init__(self, path: str, registration_path: str, base_path: str,
+                 overrides: Dict[str, Any] = None) -> None:
         super().__init__()
         self.path = path  # type: str
         self.registration_path = registration_path  # type: str
         self.base_path = base_path  # type: str
         self._registration = None  # type: Optional[Dict]
+        self._overrides = overrides or {}  # type: Dict[str, Any]
+
+    def __getitem__(self, key: str) -> Any:
+        try:
+            return self._overrides[f"MAUTRIX_TELEGRAM_{key.replace('.', '_').upper()}"]
+        except KeyError:
+            return super().__getitem__(key)
 
     def load(self) -> None:
         with open(self.path, 'r') as stream:
@@ -184,6 +192,9 @@ class Config(DictWithRecursion):
         copy("appservice.as_token")
         copy("appservice.hs_token")
 
+        copy("metrics.enabled")
+        copy("metrics.listen_port")
+
         copy("bridge.username_template")
         copy("bridge.alias_template")
         copy("bridge.displayname_template")
@@ -199,14 +210,13 @@ class Config(DictWithRecursion):
         copy("bridge.sync_matrix_state")
         copy("bridge.allow_matrix_login")
         copy("bridge.plaintext_highlights")
-        copy("bridge.edits_as_replies")
-        copy("bridge.highlight_edits")
         copy("bridge.public_portals")
         copy("bridge.catch_up")
         copy("bridge.sync_with_custom_puppets")
         copy("bridge.telegram_link_preview")
         copy("bridge.inline_images")
         copy("bridge.image_as_file_size")
+        copy("bridge.max_document_size")
 
         copy("bridge.bot_messages_as_notices")
         if isinstance(self["bridge.bridge_notices"], bool):
@@ -257,10 +267,24 @@ class Config(DictWithRecursion):
         copy("telegram.api_id")
         copy("telegram.api_hash")
         copy("telegram.bot_token")
+
+        copy("telegram.connection.timeout")
+        copy("telegram.connection.retries")
+        copy("telegram.connection.retry_delay")
+        copy("telegram.connection.flood_sleep_threshold")
+        copy("telegram.connection.request_retries")
+
+        copy("telegram.device_info.device_model")
+        copy("telegram.device_info.system_version")
+        copy("telegram.device_info.app_version")
+        copy("telegram.device_info.lang_code")
+        copy("telegram.device_info.system_lang_code")
+
         copy("telegram.server.enabled")
         copy("telegram.server.dc")
         copy("telegram.server.ip")
         copy("telegram.server.port")
+
         copy("telegram.proxy.type")
         copy("telegram.proxy.address")
         copy("telegram.proxy.port")

mautrix_telegram/db/base.py

@@ -20,11 +20,10 @@ from sqlalchemy import Table
 from sqlalchemy.engine.base import Engine
 from sqlalchemy.engine.result import RowProxy
 from sqlalchemy.sql.base import ImmutableColumnCollection
-from sqlalchemy.ext.declarative import as_declarative
+from sqlalchemy.ext.declarative import declarative_base
 
 
-@as_declarative()
-class Base:
+class BaseBase:
     db = None  # type: Engine
     t = None  # type: Table
     __table__ = None  # type: Table
@@ -55,3 +54,5 @@ class Base:
     def delete(self) -> None:
         with self.db.begin() as conn:
             conn.execute(self.t.delete().where(self._edit_identity))
+
+Base = declarative_base(cls=BaseBase)

mautrix_telegram/db/message.py

@@ -14,7 +14,7 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from sqlalchemy import Column, UniqueConstraint, Integer, String, and_, func, select
+from sqlalchemy import Column, UniqueConstraint, Integer, String, and_, func, desc, select
 from sqlalchemy.engine.result import RowProxy
 from typing import Optional, List
 
@@ -29,25 +29,44 @@ class Message(Base):
     mx_room = Column(String)  # type: MatrixRoomID
     tgid = Column(Integer, primary_key=True)  # type: TelegramID
     tg_space = Column(Integer, primary_key=True)  # type: TelegramID
+    edit_index = Column(Integer, primary_key=True)  # type: int
 
     __table_args__ = (UniqueConstraint("mxid", "mx_room", "tg_space", name="_mx_id_room"),)
 
     @classmethod
     def _one_or_none(cls, rows: RowProxy) -> Optional['Message']:
         try:
-            mxid, mx_room, tgid, tg_space = next(rows)
-            return cls(mxid=mxid, mx_room=mx_room, tgid=tgid, tg_space=tg_space)
+            mxid, mx_room, tgid, tg_space, edit_index = next(rows)
+            return cls(mxid=mxid, mx_room=mx_room, tgid=tgid, tg_space=tg_space,
+                       edit_index=edit_index)
         except StopIteration:
             return None
 
     @staticmethod
     def _all(rows: RowProxy) -> List['Message']:
-        return [Message(mxid=row[0], mx_room=row[1], tgid=row[2], tg_space=row[3])
+        return [Message(mxid=row[0], mx_room=row[1], tgid=row[2], tg_space=row[3],
+                        edit_index=row[4])
                 for row in rows]
 
     @classmethod
-    def get_by_tgid(cls, tgid: TelegramID, tg_space: TelegramID) -> Optional['Message']:
-        return cls._select_one_or_none(and_(cls.c.tgid == tgid, cls.c.tg_space == tg_space))
+    def get_all_by_tgid(cls, tgid: TelegramID, tg_space: TelegramID) -> List['Message']:
+        return cls._all(cls.db.execute(cls.t.select().where(and_(cls.c.tgid == tgid,
+                                                                 cls.c.tg_space == tg_space))))
+
+    @classmethod
+    def get_one_by_tgid(cls, tgid: TelegramID, tg_space: TelegramID, edit_index: int = 0
+                        ) -> Optional['Message']:
+        query = cls.t.select()
+        if edit_index < 0:
+            query = (query
+                     .where(and_(cls.c.tgid == tgid, cls.c.tg_space == tg_space))
+                     .order_by(desc(cls.c.edit_index))
+                     .limit(1)
+                     .offset(-edit_index - 1))
+        else:
+            query = query.where(and_(cls.c.tgid == tgid, cls.c.tg_space == tg_space,
+                                     cls.c.edit_index == edit_index))
+        return cls._one_or_none(cls.db.execute(query))
 
     @classmethod
     def count_spaces_by_mxid(cls, mxid: MatrixEventID, mx_room: MatrixRoomID) -> int:
@@ -67,10 +86,12 @@ class Message(Base):
                                             cls.c.tg_space == tg_space))
 
     @classmethod
-    def update_by_tgid(cls, s_tgid: TelegramID, s_tg_space: TelegramID, **values) -> None:
+    def update_by_tgid(cls, s_tgid: TelegramID, s_tg_space: TelegramID, s_edit_index: int,
+                       **values) -> None:
         with cls.db.begin() as conn:
             conn.execute(cls.t.update()
-                         .where(and_(cls.c.tgid == s_tgid, cls.c.tg_space == s_tg_space))
+                         .where(and_(cls.c.tgid == s_tgid, cls.c.tg_space == s_tg_space,
+                                     cls.c.edit_index == s_edit_index))
                          .values(**values))
 
     @classmethod
@@ -82,9 +103,11 @@ class Message(Base):
 
     @property
     def _edit_identity(self):
-        return and_(self.c.tgid == self.tgid, self.c.tg_space == self.tg_space)
+        return and_(self.c.tgid == self.tgid, self.c.tg_space == self.tg_space,
+                    self.c.edit_index == self.edit_index)
 
     def insert(self) -> None:
         with self.db.begin() as conn:
             conn.execute(self.t.insert().values(mxid=self.mxid, mx_room=self.mx_room,
-                                                tgid=self.tgid, tg_space=self.tg_space))
+                                                tgid=self.tgid, tg_space=self.tg_space,
+                                                edit_index=self.edit_index))

mautrix_telegram/db/puppet.py

@@ -35,15 +35,16 @@ class Puppet(Base):
     photo_id = Column(String, nullable=True)
     is_bot = Column(Boolean, nullable=True)
     matrix_registered = Column(Boolean, nullable=False, server_default=expression.false())
+    disable_updates = Column(Boolean, nullable=False, server_default=expression.false())
 
     @classmethod
     def scan(cls, row) -> Optional['Puppet']:
         (id, custom_mxid, access_token, displayname, displayname_source, username, photo_id,
-         is_bot, matrix_registered) = row
+         is_bot, matrix_registered, disable_updates) = row
         return cls(id=id, custom_mxid=custom_mxid, access_token=access_token,
                    displayname=displayname, displayname_source=displayname_source,
                    username=username, photo_id=photo_id, is_bot=is_bot,
-                   matrix_registered=matrix_registered)
+                   matrix_registered=matrix_registered, disable_updates=disable_updates)
 
     @classmethod
     def _one_or_none(cls, rows: RowProxy) -> Optional['Puppet']:
@@ -84,4 +85,4 @@ class Puppet(Base):
                 id=self.id, custom_mxid=self.custom_mxid, access_token=self.access_token,
                 displayname=self.displayname, displayname_source=self.displayname_source,
                 username=self.username, photo_id=self.photo_id, is_bot=self.is_bot,
-                matrix_registered=self.matrix_registered))
+                matrix_registered=self.matrix_registered, disable_updates=self.disable_updates))

mautrix_telegram/db/user.py

@@ -106,8 +106,8 @@ class User(Base):
 
     def delete(self) -> None:
         super().delete()
-        self.portals = None
-        self.contacts = None
+        self.portals = []
+        self.contacts = []
 
 
 class UserPortal(Base):

mautrix_telegram/formatter/__init__.py

@@ -1,9 +1,8 @@
 from .from_matrix import (matrix_reply_to_telegram, matrix_to_telegram, matrix_text_to_telegram,
                           init_mx)
-from .from_telegram import (telegram_reply_to_matrix, telegram_to_matrix, init_tg)
+from .from_telegram import telegram_reply_to_matrix, telegram_to_matrix
 from .. import context as c
 
 
 def init(context: c.Context) -> None:
     init_mx(context)
-    init_tg(context)

mautrix_telegram/formatter/from_matrix/__init__.py

@@ -87,25 +87,28 @@ def matrix_to_telegram(html: str) -> ParsedMessage:
 
 def matrix_reply_to_telegram(content: Dict[str, Any], tg_space: TelegramID,
                              room_id: Optional[MatrixRoomID] = None) -> Optional[TelegramID]:
+    relates_to = content.get("m.relates_to", None) or {}
+    if not relates_to:
+        return None
+    reply = (relates_to if relates_to.get("rel_type", None) == "m.reference"
+             else relates_to.get("m.in_reply_to", None) or {})
+    if not reply:
+        return None
+    room_id = room_id or reply.get("room_id", None)
+    event_id = reply.get("event_id", None)
+    if not event_id:
+        return
+
     try:
-        reply = content.get("m.relates_to", {}).get("m.in_reply_to", {})
-        if not reply:
-            return None
-        room_id = room_id or reply["room_id"]
-        event_id = reply["event_id"]
-
-        try:
-            if content["format"] == "org.matrix.custom.html":
-                content["formatted_body"] = trim_reply_fallback_html(content["formatted_body"])
-        except KeyError:
-            pass
-        content["body"] = trim_reply_fallback_text(content["body"])
-
-        message = DBMessage.get_by_mxid(event_id, room_id, tg_space)
-        if message:
-            return message.tgid
+        if content["format"] == "org.matrix.custom.html":
+            content["formatted_body"] = trim_reply_fallback_html(content["formatted_body"])
     except KeyError:
         pass
+    content["body"] = trim_reply_fallback_text(content["body"])
+
+    message = DBMessage.get_by_mxid(event_id, room_id, tg_space)
+    if message:
+        return message.tgid
     return None
 
 

mautrix_telegram/formatter/from_matrix/html_reader.py

@@ -29,6 +29,10 @@ class HTMLNode(list):
 
 
 class NodeifyingParser(HTMLParser):
+    # From https://www.w3.org/TR/html5/syntax.html#writing-html-documents-elements
+    void_tags = ("area", "base", "br", "col", "command", "embed", "hr", "img", "input", "link",
+                 "meta", "param", "source", "track", "wbr")
+
     def __init__(self):
         super().__init__()
         self.stack = [HTMLNode("html", [])]  # type: List[HTMLNode]
@@ -36,7 +40,11 @@ class NodeifyingParser(HTMLParser):
     def handle_starttag(self, tag, attrs):
         node = HTMLNode(tag, attrs)
         self.stack[-1].append(node)
-        self.stack.append(node)
+        if tag not in self.void_tags:
+            self.stack.append(node)
+
+    def handle_startendtag(self, tag, attrs):
+        self.stack[-1].append(HTMLNode(tag, attrs))
 
     def handle_endtag(self, tag):
         if tag == self.stack[-1].tag:

mautrix_telegram/formatter/from_telegram.py

@@ -37,16 +37,8 @@ from .util import (add_surrogates, remove_surrogates, trim_reply_fallback_html,
 
 if TYPE_CHECKING:
     from ..abstract_user import AbstractUser
-    from ..context import Context
-
-try:
-    from lxml.html.diff import htmldiff
-except ImportError:
-    htmldiff = None  # type: ignore
-
 
 log = logging.getLogger("mau.fmt.tg")  # type: logging.Logger
-should_highlight_edits = False  # type: bool
 
 
 def telegram_reply_to_matrix(evt: Message, source: 'AbstractUser') -> Dict:
@@ -54,13 +46,16 @@ def telegram_reply_to_matrix(evt: Message, source: 'AbstractUser') -> Dict:
         space = (evt.to_id.channel_id
                  if isinstance(evt, Message) and isinstance(evt.to_id, PeerChannel)
                  else source.tgid)
-        msg = DBMessage.get_by_tgid(evt.reply_to_msg_id, space)
+        msg = DBMessage.get_one_by_tgid(evt.reply_to_msg_id, space)
         if msg:
             return {
                 "m.in_reply_to": {
                     "event_id": msg.mxid,
                     "room_id": msg.mx_room,
-                }
+                },
+                "rel_type": "m.reference",
+                "event_id": msg.mxid,
+                "room_id": msg.mx_room,
             }
     return {}
 
@@ -74,27 +69,30 @@ async def _add_forward_header(source, text: str, html: Optional[str],
         user = u.User.get_by_tgid(TelegramID(fwd_from.from_id))
         if user:
             fwd_from_text = user.displayname or user.mxid
-            fwd_from_html = f"<a href='https://matrix.to/#/{user.mxid}'>{fwd_from_text}</a>"
+            fwd_from_html = (f"<a href='https://matrix.to/#/{user.mxid}'>"
+                             f"{escape(fwd_from_text)}</a>")
 
         if not fwd_from_text:
             puppet = pu.Puppet.get(TelegramID(fwd_from.from_id), create=False)
             if puppet and puppet.displayname:
                 fwd_from_text = puppet.displayname or puppet.mxid
-                fwd_from_html = f"<a href='https://matrix.to/#/{puppet.mxid}'>{fwd_from_text}</a>"
+                fwd_from_html = (f"<a href='https://matrix.to/#/{puppet.mxid}'>"
+                                 f"{escape(fwd_from_text)}</a>")
 
         if not fwd_from_text:
             user = await source.client.get_entity(PeerUser(fwd_from.from_id))
             if user:
                 fwd_from_text = pu.Puppet.get_displayname(user, False)
-                fwd_from_html = f"<b>{fwd_from_text}</b>"
+                fwd_from_html = f"<b>{escape(fwd_from_text)}</b>"
     else:
         portal = po.Portal.get_by_tgid(TelegramID(fwd_from.channel_id))
         if portal:
             fwd_from_text = portal.title
             if portal.alias:
-                fwd_from_html = f"<a href='https://matrix.to/#/{portal.alias}'>{fwd_from_text}</a>"
+                fwd_from_html = (f"<a href='https://matrix.to/#/{portal.alias}'>"
+                                 f"{escape(fwd_from_text)}</a>")
             else:
-                fwd_from_html = f"<b>{fwd_from_text}</b>"
+                fwd_from_html = f"<b>{escape(fwd_from_text)}</b>"
         else:
             channel = await source.client.get_entity(PeerChannel(fwd_from.channel_id))
             if channel:
@@ -115,32 +113,19 @@ async def _add_forward_header(source, text: str, html: Optional[str],
     return text, html
 
 
-def highlight_edits(new_html: str, old_html: str) -> str:
-    # Don't include `Edit:` text in diff.
-    if old_html.startswith("<u>Edit:</u> "):
-        old_html = old_html[len("<u>Edit:</u> "):]
-
-    # Generate diff with lxml
-    new_html = htmldiff(old_html, new_html)
-
-    # Replace <ins> with <u> since Riot doesn't allow <ins>
-    new_html = new_html.replace("<ins>", "<u>").replace("</ins>", "</u>")
-    # Remove <del>s since we just want to hide deletions.
-    new_html = re.sub("<del>.+?</del>", "", new_html)
-    return new_html
-
-
 async def _add_reply_header(source: "AbstractUser", text: str, html: str, evt: Message,
-                            relates_to: Dict, main_intent: IntentAPI, is_edit: bool
-                            ) -> Tuple[str, str]:
+                            relates_to: Dict, main_intent: IntentAPI) -> Tuple[str, str]:
     space = (evt.to_id.channel_id
              if isinstance(evt, Message) and isinstance(evt.to_id, PeerChannel)
              else source.tgid)
 
-    msg = DBMessage.get_by_tgid(evt.reply_to_msg_id, space)
+    msg = DBMessage.get_one_by_tgid(evt.reply_to_msg_id, space)
     if not msg:
         return text, html
 
+    relates_to["rel_type"] = "m.reference"
+    relates_to["event_id"] = msg.mxid
+    relates_to["room_id"] = msg.mx_room
     relates_to["m.in_reply_to"] = {
         "event_id": msg.mxid,
         "room_id": msg.mx_room,
@@ -159,22 +144,14 @@ async def _add_reply_header(source: "AbstractUser", text: str, html: str, evt: M
 
         puppet = pu.Puppet.get_by_mxid(r_sender, create=False)
         r_displayname = puppet.displayname if puppet else r_sender
-        r_sender_link = f"<a href='https://matrix.to/#/{r_sender}'>{r_displayname}</a>"
-
-        if is_edit and should_highlight_edits:
-            html = highlight_edits(html or escape(text), r_html_body)
+        r_sender_link = f"<a href='https://matrix.to/#/{r_sender}'>{escape(r_displayname)}</a>"
     except (ValueError, KeyError, MatrixRequestError):
         r_sender_link = "unknown user"
         r_displayname = "unknown user"
         r_text_body = "Failed to fetch message"
         r_html_body = "<em>Failed to fetch message</em>"
 
-    if is_edit:
-        html = f"<u>Edit:</u> {html or escape(text)}"
-        text = f"Edit: {text}"
-
-    r_keyword = "In reply to" if not is_edit else "Edit to"
-    r_msg_link = f"<a href='https://matrix.to/#/{msg.mx_room}/{msg.mxid}'>{r_keyword}</a>"
+    r_msg_link = f"<a href='https://matrix.to/#/{msg.mx_room}/{msg.mxid}'>In reply to</a>"
     html = (
         f"<mx-reply><blockquote>{r_msg_link} {r_sender_link}\n{r_html_body}</blockquote></mx-reply>"
         + (html or escape(text)))
@@ -191,8 +168,8 @@ async def _add_reply_header(source: "AbstractUser", text: str, html: str, evt: M
 
 async def telegram_to_matrix(evt: Message, source: "AbstractUser",
                              main_intent: Optional[IntentAPI] = None,
-                             is_edit: bool = False, prefix_text: Optional[str] = None,
-                             prefix_html: Optional[str] = None, override_text: str = None,
+                             prefix_text: Optional[str] = None, prefix_html: Optional[str] = None,
+                             override_text: str = None,
                              override_entities: List[TypeMessageEntity] = None,
                              no_reply_fallback: bool = False) -> Tuple[str, str, Dict]:
     text = add_surrogates(override_text or evt.message)
@@ -209,8 +186,7 @@ async def telegram_to_matrix(evt: Message, source: "AbstractUser",
         text, html = await _add_forward_header(source, text, html, evt.fwd_from)
 
     if evt.reply_to_msg_id and not no_reply_fallback:
-        text, html = await _add_reply_header(source, text, html, evt, relates_to, main_intent,
-                                             is_edit)
+        text, html = await _add_reply_header(source, text, html, evt, relates_to, main_intent)
 
     if isinstance(evt, Message) and evt.post and evt.post_author:
         if not html:
@@ -258,9 +234,9 @@ def _telegram_entities_to_matrix(text: str, entities: List[TypeMessageEntity]) -
         elif entity_type == MessageEntityItalic:
             html.append(f"<em>{entity_text}</em>")
         elif entity_type == MessageEntityCode:
-            html.append(f"<pre><code>{entity_text}</code></pre>"
-                        if "\n" in entity_text
-                        else f"<code>{entity_text}</code>")
+            html.append(("<pre><code>{entity_text}</code></pre>"
+                         if "\n" in entity_text
+                         else "<code>{entity_text}</code>").format(entity_text=entity_text))
         elif entity_type == MessageEntityPre:
             skip_entity = _parse_pre(html, entity_text, entity.language)
         elif entity_type == MessageEntityMention:
@@ -341,14 +317,9 @@ def _parse_url(html: List[str], entity_text: str, url: str) -> bool:
 
         portal = po.Portal.find_by_username(group)
         if portal:
-            message = DBMessage.get_by_tgid(TelegramID(msgid), portal.tgid)
+            message = DBMessage.get_one_by_tgid(TelegramID(msgid), portal.tgid)
             if message:
                 url = f"https://matrix.to/#/{portal.mxid}/{message.mxid}"
 
     html.append(f"<a href='{url}'>{entity_text}</a>")
     return False
-
-
-def init_tg(context: "Context") -> None:
-    global should_highlight_edits
-    should_highlight_edits = htmldiff and context.config["bridge.highlight_edits"]

mautrix_telegram/matrix.py

@@ -17,6 +17,7 @@
 from typing import Dict, List, Match, Optional, Set, Tuple, TYPE_CHECKING
 import logging
 import asyncio
+import time
 import re
 
 from mautrix_appservice import MatrixRequestError, IntentError
@@ -27,6 +28,15 @@ from . import user as u, portal as po, puppet as pu, commands as com
 if TYPE_CHECKING:
     from .context import Context
 
+try:
+    from prometheus_client import Histogram
+
+    EVENT_TIME = Histogram("matrix_event", "Time spent processing Matrix events",
+                           ["event_type"])
+except ImportError:
+    Histogram = None
+    EVENT_TIME = None
+
 
 class MatrixHandler:
     log = logging.getLogger("mau.mx")  # type: logging.Logger
@@ -126,12 +136,31 @@ class MatrixHandler:
 
         if not inviter.whitelisted:
             await self.az.intent.send_notice(
-                room_id, text="",
-                html="You are not whitelisted to use this bridge.<br/><br/>"
+                room_id,
+                text="You are not whitelisted to use this bridge.\n\n"
                      "If you are the owner of this bridge, see the "
-                     "<code>bridge.permissions</code> section in your config file.")
+                     "`bridge.permissions` section in your config file.",
+                html="<p>You are not whitelisted to use this bridge.</p>"
+                     "<p>If you are the owner of this bridge, see the "
+                     "<code>bridge.permissions</code> section in your config file.</p>")
             await self.az.intent.leave_room(room_id)
 
+        try:
+            is_management = len(await self.az.intent.get_room_members(room_id)) == 2
+        except MatrixRequestError:
+            is_management = False
+        cmd_prefix = self.commands.command_prefix
+        text = html = "Hello, I'm a Telegram bridge bot. "
+        if is_management and inviter.puppet_whitelisted and not await inviter.is_logged_in():
+            text += f"Use `{cmd_prefix} help` for help or `{cmd_prefix} login` to log in."
+            html += (f"Use <code>{cmd_prefix} help</code> for help"
+                     f" or <code>{cmd_prefix} login</code> to log in.")
+            pass
+        else:
+            text += f"Use `{cmd_prefix} help` for help."
+            html += f"Use <code>{cmd_prefix} help</code> for help."
+        await self.az.intent.send_notice(room_id, text=text, html=html)
+
     async def handle_invite(self, room_id: MatrixRoomID, user_id: MatrixUserID,
                             inviter_mxid: MatrixUserID) -> None:
         self.log.debug(f"{inviter_mxid} invited {user_id} to {room_id}")
@@ -213,7 +242,7 @@ class MatrixHandler:
         prefix = self.config["bridge.command_prefix"]
         is_command = text.startswith(prefix)
         if is_command:
-            text = text[len(prefix) + 1:]
+            text = text[len(prefix) + 1:].lstrip()
         return is_command, text
 
     async def handle_message(self, room: MatrixRoomID, sender_id: MatrixUserID, message: Dict,
@@ -379,6 +408,7 @@ class MatrixHandler:
     async def handle_event(self, evt: MatrixEvent) -> None:
         if self.filter_matrix_event(evt):
             return
+        start_time = time.time()
         self.log.debug("Received event: %s", evt)
         evt_type = evt.get("type", "m.unknown")  # type: str
         room_id = evt.get("room_id", None)  # type: Optional[MatrixRoomID]
@@ -430,3 +460,7 @@ class MatrixHandler:
             await self.handle_presence(sender, content.get("presence", "offline"))
         elif evt_type == "m.typing":
             await self.handle_typing(room_id, content.get("user_ids", []))
+        else:
+            return
+        if EVENT_TIME:
+            EVENT_TIME.labels(event_type=evt_type).observe(time.time() - start_time)

mautrix_telegram/portal.py

@@ -43,31 +43,33 @@ from telethon.tl.functions.channels import (
     UpdateUsernameRequest)
 from telethon.tl.functions.messages import ReadHistoryRequest as ReadMessageHistoryRequest
 from telethon.tl.functions.channels import ReadHistoryRequest as ReadChannelHistoryRequest
-from telethon.errors import ChatAdminRequiredError, ChatNotModifiedError
+from telethon.errors import (ChatAdminRequiredError, ChatNotModifiedError, PhotoExtInvalidError,
+                             PhotoInvalidDimensionsError, PhotoSaveFileInvalidError)
 from telethon.tl.patched import Message, MessageService
 from telethon.tl.types import (
-    Channel, ChatAdminRights, ChatBannedRights, ChannelFull, ChannelParticipantAdmin,
-    ChannelParticipantCreator, ChannelParticipantsRecent, ChannelParticipantsSearch, Chat, ChatFull,
-    ChatInviteEmpty, ChatParticipantAdmin, ChatParticipantCreator, ChatPhoto, Poll, PollAnswer,
-    DocumentAttributeFilename, DocumentAttributeImageSize, DocumentAttributeSticker,
-    DocumentAttributeVideo, FileLocation, GeoPoint, InputChannel, InputChatUploadedPhoto,
-    InputPeerChannel, InputPeerChat, InputPeerUser, InputUser, InputUserSelf,
-    MessageActionChannelCreate, MessageActionChatAddUser, MessageActionChatCreate,
+    Channel, ChatAdminRights, ChatBannedRights, ChannelFull, ChannelParticipantAdmin, Document,
+    ChannelParticipantCreator, ChannelParticipantsRecent, ChannelParticipantsSearch, Chat,
+    ChatFull, ChatInviteEmpty, ChatParticipantAdmin, ChatParticipantCreator, ChatPhoto, Poll,
+    DocumentAttributeFilename, DocumentAttributeImageSize, DocumentAttributeSticker, PhotoEmpty,
+    DocumentAttributeVideo, GeoPoint, InputChannel, InputChatUploadedPhoto, InputPhotoFileLocation,
+    InputPeerChannel, InputPeerChat, InputPeerUser, InputUser, InputUserSelf, MessageMediaPoll,
+    MessageActionChannelCreate, MessageActionChatAddUser, MessageActionChatCreate, ChatPhotoEmpty,
     MessageActionChatDeletePhoto, MessageActionChatDeleteUser, MessageActionChatEditPhoto,
     MessageActionChatEditTitle, MessageActionChatJoinedByLink, MessageActionChatMigrateTo,
     MessageActionPinMessage, MessageActionGameScore, MessageMediaContact, MessageMediaDocument,
-    MessageMediaGeo, MessageMediaPhoto, MessageMediaUnsupported, MessageMediaGame, MessageMediaPoll,
+    MessageMediaGeo, MessageMediaPhoto, MessageMediaUnsupported, MessageMediaGame,
     PeerChannel, PeerChat, PeerUser, Photo, PhotoCachedSize, SendMessageCancelAction,
     SendMessageTypingAction, TypeChannelParticipant, TypeChat, TypeChatParticipant,
     TypeDocumentAttribute, TypeInputPeer, TypeMessageAction, TypeMessageEntity, TypePeer,
     TypePhotoSize, TypeUpdates, TypeUser, PhotoSize, TypeUserFull, UpdateChatUserTyping,
-    UpdateNewChannelMessage, UpdateNewMessage, UpdateUserTyping, User, UserFull, MessageEntityPre)
+    UpdateNewChannelMessage, UpdateNewMessage, UpdateUserTyping, User, UserFull, MessageEntityPre,
+    InputMediaUploadedDocument, InputPeerPhotoFileLocation)
 from mautrix_appservice import MatrixRequestError, IntentError, AppService, IntentAPI
 
 from .types import MatrixEventID, MatrixRoomID, MatrixUserID, TelegramID
 from .context import Context
 from .db import Portal as DBPortal, Message as DBMessage, TelegramFile as DBTelegramFile
-from .util import ignore_coro
+from .util import ignore_coro, sane_mimetypes
 from . import puppet as p, user as u, formatter, util
 
 if TYPE_CHECKING:
@@ -76,8 +78,6 @@ if TYPE_CHECKING:
     from .config import Config
     from .tgclient import MautrixTelegramClient
 
-mimetypes.init()
-
 config = None  # type: Config
 
 TypeMessage = Union[Message, MessageService]
@@ -573,7 +573,7 @@ class Portal:
         changed = await self.update_title(entity.title) or changed
 
         if isinstance(entity.photo, ChatPhoto):
-            changed = await self.update_avatar(user, entity.photo.photo_big) or changed
+            changed = await self.update_avatar(user, entity.photo) or changed
 
         if changed:
             self.save()
@@ -614,12 +614,23 @@ class Portal:
         return False
 
     @staticmethod
-    def _get_largest_photo_size(photo: Union[Photo, List[TypePhotoSize]]
-                                ) -> Optional[TypePhotoSize]:
+    def _get_largest_photo_size(photo: Union[Photo, Document]
+                                ) -> Tuple[Optional[InputPhotoFileLocation],
+                                           Optional[TypePhotoSize]]:
         if not photo:
-            return None
-        return max(photo.sizes if isinstance(photo, Photo) else photo, key=(lambda photo2: (
-            len(photo2.bytes) if not isinstance(photo2, PhotoSize) else photo2.size)))
+            return None, None
+        if isinstance(photo, Document) and not photo.thumbs:
+            return None, None
+        largest = max(photo.sizes if isinstance(photo, Photo) else photo.thumbs,
+                      key=(lambda photo2: (len(photo2.bytes)
+                                           if not isinstance(photo2, PhotoSize)
+                                           else photo2.size)))
+        return InputPhotoFileLocation(
+            id=photo.id,
+            access_hash=photo.access_hash,
+            file_reference=photo.file_reference,
+            thumb_size=largest.type,
+        ), largest
 
     async def remove_avatar(self, _: 'AbstractUser', save: bool = False) -> None:
         await self.main_intent.set_room_avatar(self.mxid, None)
@@ -627,11 +638,33 @@ class Portal:
         if save:
             self.save()
 
-    async def update_avatar(self, user: 'AbstractUser', photo: FileLocation,
+    async def update_avatar(self, user: 'AbstractUser',
+                            photo: Union[ChatPhoto, ChatPhotoEmpty, Photo, PhotoEmpty],
                             save: bool = False) -> bool:
-        photo_id = f"{photo.volume_id}-{photo.local_id}"
+        if isinstance(photo, ChatPhoto):
+            loc = InputPeerPhotoFileLocation(
+                peer=await self.get_input_entity(user),
+                local_id=photo.photo_big.local_id,
+                volume_id=photo.photo_big.volume_id,
+                big=True
+            )
+            photo_id = f"{loc.volume_id}-{loc.local_id}"
+        elif isinstance(photo, Photo):
+            loc, largest = self._get_largest_photo_size(photo)
+            photo_id = f"{largest.location.volume_id}-{largest.location.local_id}"
+        elif isinstance(photo, (ChatPhotoEmpty, PhotoEmpty)):
+            photo_id = ""
+            loc = None
+        else:
+            raise ValueError(f"Unknown photo type {type(photo)}")
         if self.photo_id != photo_id:
-            file = await util.transfer_file_to_matrix(user.client, self.main_intent, photo)
+            if not photo_id:
+                await self.main_intent.set_room_avatar(self.mxid, "")
+                self.photo_id = ""
+                if save:
+                    self.save()
+                return True
+            file = await util.transfer_file_to_matrix(user.client, self.main_intent, loc)
             if file:
                 await self.main_intent.set_room_avatar(self.mxid, file.mxc)
                 self.photo_id = photo_id
@@ -750,14 +783,9 @@ class Portal:
                 return body + current_extension
         except (ValueError, KeyError):
             pass
-        ext_override = {
-            "image/jpeg": ".jpg"
-        }
         if mime:
-            ext = ext_override.get(mime, mimetypes.guess_extension(mime))
-            return f"matrix_upload{ext}"
-        else:
-            return ""
+            return f"matrix_upload{sane_mimetypes.guess_extension(mime)}"
+        return ""
 
     def get_config(self, key: str) -> Any:
         local = util.recursive_get(self.local_config, key)
@@ -899,7 +927,7 @@ class Portal:
                                 ) -> None:
         if "formatted_body" not in message:
             message["format"] = "org.matrix.custom.html"
-            message["formatted_body"] = escape_html(message.get("body", ""))
+            message["formatted_body"] = escape_html(message.get("body", "")).replace("\n", "<br/>")
         body = message["formatted_body"]
 
         tpl = (self.get_config(f"message_formats.[{msgtype}]")
@@ -916,9 +944,14 @@ class Portal:
         msgtype = message.get("msgtype", "m.text")
         if msgtype == "m.emote":
             await self._apply_msg_format(sender, msgtype, message)
+            if "m.new_content" in message:
+                await self._apply_msg_format(sender, msgtype, message["m.new_content"])
+                message["m.new_content"]["msgtype"] = "m.text"
             message["msgtype"] = "m.text"
         elif use_relaybot:
             await self._apply_msg_format(sender, msgtype, message)
+            if "m.new_content" in message:
+                await self._apply_msg_format(sender, msgtype, message["m.new_content"])
 
     @staticmethod
     def _matrix_event_to_entities(event: Dict[str, Any]
@@ -950,15 +983,25 @@ class Portal:
             return None
 
     async def _handle_matrix_text(self, sender_id: TelegramID, event_id: MatrixEventID,
-                                  space: TelegramID, client: 'MautrixTelegramClient', message: Dict,
-                                  reply_to: TelegramID) -> None:
+                                  space: TelegramID, client: 'MautrixTelegramClient',
+                                  message: Dict, reply_to: TelegramID) -> None:
         lock = self.require_send_lock(sender_id)
         async with lock:
             lp = self.get_config("telegram_link_preview")
+            relates_to = message.get("m.relates_to", None) or {}
+            if relates_to.get("rel_type", None) == "m.replace":
+                orig_msg = DBMessage.get_by_mxid(relates_to.get("event_id", ""), self.mxid, space)
+                if orig_msg:
+                    response = await client.edit_message(self.peer, orig_msg.tgid,
+                                                         message.get("m.new_content", message),
+                                                         parse_mode=self._matrix_event_to_entities,
+                                                         link_preview=lp)
+                    self._add_telegram_message_to_db(event_id, space, -1, response)
+                    return
             response = await client.send_message(self.peer, message, reply_to=reply_to,
                                                  parse_mode=self._matrix_event_to_entities,
                                                  link_preview=lp)
-            self._add_telegram_message_to_db(event_id, space, response)
+            self._add_telegram_message_to_db(event_id, space, 0, response)
 
     async def _handle_matrix_file(self, msgtype: str, sender_id: TelegramID,
                                   event_id: MatrixEventID, space: TelegramID,
@@ -993,9 +1036,23 @@ class Portal:
             max_image_size=config["bridge.image_as_file_size"] * 1000 ** 2)
         lock = self.require_send_lock(sender_id)
         async with lock:
-            response = await client.send_media(self.peer, media, reply_to=reply_to,
-                                               caption=caption)
-            self._add_telegram_message_to_db(event_id, space, response)
+            relates_to = message.get("m.relates_to", None) or {}
+            if relates_to.get("rel_type", None) == "m.replace":
+                orig_msg = DBMessage.get_by_mxid(relates_to.get("event_id", ""), self.mxid, space)
+                if orig_msg:
+                    response = await client.edit_message(self.peer, orig_msg.tgid,
+                                                         caption, file=media)
+                    self._add_telegram_message_to_db(event_id, space, -1, response)
+                    return
+            try:
+                response = await client.send_media(self.peer, media, reply_to=reply_to,
+                                                   caption=caption)
+            except (PhotoInvalidDimensionsError, PhotoSaveFileInvalidError, PhotoExtInvalidError):
+                media = InputMediaUploadedDocument(file=media.file, mime_type=mime,
+                                                   attributes=attributes)
+                response = await client.send_media(self.peer, media, reply_to=reply_to,
+                                                   caption=caption)
+            self._add_telegram_message_to_db(event_id, space, 0, response)
 
     async def _handle_matrix_location(self, sender_id: TelegramID, event_id: MatrixEventID,
                                       space: TelegramID, client: 'MautrixTelegramClient',
@@ -1011,19 +1068,31 @@ class Portal:
 
         lock = self.require_send_lock(sender_id)
         async with lock:
+            relates_to = message.get("m.relates_to", None) or {}
+            if relates_to.get("rel_type", None) == "m.replace":
+                orig_msg = DBMessage.get_by_mxid(relates_to.get("event_id", ""), self.mxid, space)
+                if orig_msg:
+                    response = await client.edit_message(self.peer, orig_msg.tgid,
+                                                         caption, file=media)
+                    self._add_telegram_message_to_db(event_id, space, -1, response)
+                    return
             response = await client.send_media(self.peer, media, reply_to=reply_to,
                                                caption=caption, entities=entities)
-            self._add_telegram_message_to_db(event_id, space, response)
+            self._add_telegram_message_to_db(event_id, space, 0, response)
 
     def _add_telegram_message_to_db(self, event_id: MatrixEventID, space: TelegramID,
-                                    response: TypeMessage) -> None:
+                                    edit_index: int, response: TypeMessage) -> None:
         self.log.debug("Handled Matrix message: %s", response)
-        self.is_duplicate(response, (event_id, space))
+        self.is_duplicate(response, (event_id, space), force_hash=edit_index != 0)
+        if edit_index < 0:
+            prev_edit = DBMessage.get_one_by_tgid(TelegramID(response.id), space, -1)
+            edit_index = prev_edit.edit_index + 1
         DBMessage(
             tgid=TelegramID(response.id),
             tg_space=space,
             mx_room=self.mxid,
-            mxid=event_id).insert()
+            mxid=event_id,
+            edit_index=edit_index).insert()
 
     async def handle_matrix_message(self, sender: 'u.User', message: Dict[str, Any],
                                     event_id: MatrixEventID) -> None:
@@ -1087,7 +1156,10 @@ class Portal:
         message = DBMessage.get_by_mxid(event_id, self.mxid, space)
         if not message:
             return
-        await real_deleter.client.delete_messages(self.peer, [message.tgid])
+        if message.edit_index == 0:
+            await real_deleter.client.delete_messages(self.peer, [message.tgid])
+        else:
+            self.log.debug(f"Ignoring deletion of edit event {message.mxid} in {message.mx_room}")
 
     async def _update_telegram_power_level(self, sender: 'u.User', user_id: TelegramID,
                                            level: int) -> None:
@@ -1151,7 +1223,7 @@ class Portal:
 
         file = await self.main_intent.download_file(url)
         mime = magic.from_buffer(file, mime=True)
-        ext = mimetypes.guess_extension(mime)
+        ext = sane_mimetypes.guess_extension(mime)
         uploaded = await sender.client.upload_file(file, file_name=f"avatar{ext}", use_cache=False)
         photo = InputChatUploadedPhoto(file=uploaded)
 
@@ -1166,8 +1238,8 @@ class Portal:
                                and isinstance(update.message, MessageService)
                                and isinstance(update.message.action, MessageActionChatEditPhoto))
             if is_photo_update:
-                loc = self._get_largest_photo_size(update.message.action.photo).location
-                self.photo_id = f"{loc.volume_id}-{loc.local_id}"
+                loc, size = self._get_largest_photo_size(update.message.action.photo)
+                self.photo_id = f"{size.location.volume_id}-{size.location.local_id}"
                 self.save()
                 break
 
@@ -1258,8 +1330,13 @@ class Portal:
 
         invites = await self._get_telegram_users_in_matrix_room()
         if len(invites) < 2:
+            if self.bot is not None:
+                info, mxid = await self.bot.get_me()
+                raise ValueError("Not enough Telegram users to create a chat. "
+                                 "Invite more Telegram ghost users to the room, such as the "
+                                 f"relaybot ([{info.first_name}](https://matrix.to/#/{mxid})).")
             raise ValueError("Not enough Telegram users to create a chat. "
-                             "Invite more Telegram ghost users to the room, such as the relaybot.")
+                             "Invite more Telegram ghost users to the room.")
         if self.peer_type == "chat":
             response = await source.client(CreateChatRequest(title=self.title, users=invites))
             entity = response.chats[0]
@@ -1311,12 +1388,14 @@ class Portal:
     def get_external_url(self, evt: Message) -> Optional[str]:
         if self.peer_type == "channel" and self.username is not None:
             return f"https://t.me/{self.username}/{evt.id}"
+        elif self.peer_type != "user":
+            return f"https://t.me/c/{self.tgid}/{evt.id}"
         return None
 
     async def handle_telegram_photo(self, source: 'AbstractUser', intent: IntentAPI, evt: Message,
                                     relates_to: Dict = None) -> Optional[Dict]:
-        largest_size = self._get_largest_photo_size(evt.media.photo)
-        file = await util.transfer_file_to_matrix(source.client, intent, largest_size.location)
+        loc, largest_size = self._get_largest_photo_size(evt.media.photo)
+        file = await util.transfer_file_to_matrix(source.client, intent, loc)
         if not file:
             return None
         if self.get_config("inline_images") and (evt.message
@@ -1337,10 +1416,7 @@ class Portal:
             "orientation": 0,
             "mimetype": file.mime_type,
         }
-        ext_override = {
-            "image/jpeg": ".jpg"
-        }
-        name = "image" + ext_override.get(file.mime_type, mimetypes.guess_extension(file.mime_type))
+        name = f"image{sane_mimetypes.guess_extension(file.mime_type)}"
         await intent.set_typing(self.mxid, is_typing=False)
         result = await intent.send_image(self.mxid, file.mxc, info=info, text=name,
                                          relates_to=relates_to, timestamp=evt.date,
@@ -1375,7 +1451,7 @@ class Portal:
 
     @staticmethod
     def _parse_telegram_document_meta(evt: Message, file: DBTelegramFile, attrs: Dict,
-                                      thumb: TypePhotoSize) -> Tuple[Dict, str]:
+                                      thumb_size: TypePhotoSize) -> Tuple[Dict, str]:
         document = evt.media.document
         name = evt.message or attrs["name"]
         if attrs["is_sticker"]:
@@ -1386,7 +1462,11 @@ class Portal:
                 except ValueError:
                     name = alt
 
-        mime_type = document.mime_type or file.mime_type
+        generic_types = ("text/plain", "application/octet-stream")
+        if file.mime_type in generic_types and document.mime_type not in generic_types:
+            mime_type = document.mime_type or file.mime_type
+        else:
+            mime_type = file.mime_type or document.mime_type
         info = {
             "size": file.size,
             "mimetype": mime_type,
@@ -1403,8 +1483,8 @@ class Portal:
             info["thumbnail_url"] = file.thumbnail.mxc
             info["thumbnail_info"] = {
                 "mimetype": file.thumbnail.mime_type,
-                "h": file.thumbnail.height or thumb.h,
-                "w": file.thumbnail.width or thumb.w,
+                "h": file.thumbnail.height or thumb_size.h,
+                "w": file.thumbnail.width or thumb_size.w,
                 "size": file.thumbnail.size,
             }
 
@@ -1413,18 +1493,25 @@ class Portal:
     async def handle_telegram_document(self, source: 'AbstractUser', intent: IntentAPI,
                                        evt: Message, relates_to: dict = None) -> Optional[Dict]:
         document = evt.media.document
+
         attrs = self._parse_telegram_document_attributes(document.attributes)
 
-        thumb = self._get_largest_photo_size(document.thumbs)
-        if thumb and not isinstance(thumb, (PhotoSize, PhotoCachedSize)):
-            self.log.debug(f"Unsupported thumbnail type {type(thumb)}")
-            thumb = None
-        file = await util.transfer_file_to_matrix(source.client, intent, document, thumb,
+        if document.size > config["bridge.max_document_size"] * 1000 ** 2:
+            name = attrs["name"] or ""
+            caption = f"\n{evt.message}" if evt.message else ""
+            return await intent.send_notice(self.mxid, f"Too large file {name}{caption}")
+
+        thumb_loc, thumb_size = self._get_largest_photo_size(document)
+        if thumb_size and not isinstance(thumb_size, (PhotoSize, PhotoCachedSize)):
+            self.log.debug(f"Unsupported thumbnail type {type(thumb_size)}")
+            thumb_loc = None
+            thumb_size = None
+        file = await util.transfer_file_to_matrix(source.client, intent, document, thumb_loc,
                                                   is_sticker=attrs["is_sticker"])
         if not file:
             return None
 
-        info, name = self._parse_telegram_document_meta(evt, file, attrs, thumb)
+        info, name = self._parse_telegram_document_meta(evt, file, attrs, thumb_size)
 
         await intent.set_typing(self.mxid, is_typing=False)
 
@@ -1512,20 +1599,24 @@ class Portal:
         poll = evt.media.poll  # type: Poll
         poll_id = self._encode_msgid(source, evt)
 
-        def enc(answer: PollAnswer) -> str:
-            return base64.b64encode(answer.option).decode("utf-8").rstrip("=")
+        _n = 0
 
-        text = (f"Poll ID {poll_id}: {poll.question}\n"
-                + "\n".join(f"* {enc(answer)}: {answer.text}" for answer in poll.answers) +
+        def n() -> int:
+            nonlocal _n
+            _n += 1
+            return _n
+
+        text = (f"Poll: {poll.question}\n"
+                + "\n".join(f"{n()}. {answer.text}" for answer in poll.answers) +
                 "\n"
-                f"Vote with !tg vote <poll ID> <choice ID>")
+                f"Vote with !tg vote {poll_id} <choice number>")
 
-        html = (f"<strong>Poll</strong> ID <code>{poll_id}</code>: {poll.question}<br/>\n"
-                f"<ul>"
-                + "\n".join(f"<li><code>{enc(answer)}</code>: {answer.text}</li>"
+        html = (f"<strong>Poll</strong>: {poll.question}<br/>\n"
+                f"<ol>"
+                + "\n".join(f"<li>{answer.text}</li>"
                             for answer in poll.answers) +
-                "</ul>\n"
-                f"Vote with <code>!tg vote &lt;poll ID&gt; &lt;choice ID&gt;</code>")
+                "</ol>\n"
+                f"Vote with <code>!tg vote {poll_id} &lt;choice number&gt;</code>")
         await intent.set_typing(self.mxid, is_typing=False)
         return await intent.send_text(self.mxid, text, html=html, relates_to=relates_to,
                                       msgtype="m.text", timestamp=evt.date,
@@ -1560,7 +1651,8 @@ class Portal:
         play_id = self._encode_msgid(source, evt)
         command = f"!tg play {play_id}"
         override_text = f"Run {command} in your bridge management room to play {game.title}"
-        override_entities = [MessageEntityPre(offset=len("Run "), length=len(command), language="")]
+        override_entities = [
+            MessageEntityPre(offset=len("Run "), length=len(command), language="")]
         text, html, relates_to = await formatter.telegram_to_matrix(
             evt, source, self.main_intent,
             override_text=override_text, override_entities=override_entities)
@@ -1578,9 +1670,6 @@ class Portal:
                                    evt: Message) -> None:
         if not self.mxid:
             return
-        elif not self.get_config("edits_as_replies"):
-            self.log.debug("Edits as replies disabled, ignoring edit event...")
-            return
         elif hasattr(evt, "media") and isinstance(evt.media, (MessageMediaGame,)):
             self.log.debug("Ignoring game message edit event")
             return
@@ -1598,28 +1687,52 @@ class Portal:
         if duplicate_found:
             mxid, other_tg_space = duplicate_found
             if tg_space != other_tg_space:
-                DBMessage.update_by_tgid(TelegramID(evt.id), tg_space,
-                                         mxid=mxid,
-                                         mx_room=self.mxid)
+                prev_edit_msg = DBMessage.get_one_by_tgid(TelegramID(evt.id), tg_space, -1)
+                if not prev_edit_msg:
+                    return
+                DBMessage(mxid=mxid, mx_room=self.mxid, tg_space=tg_space, tgid=evt.id,
+                          edit_index=prev_edit_msg.edit_index + 1).insert()
             return
 
-        evt.reply_to_msg_id = evt.id
-        text, html, relates_to = await formatter.telegram_to_matrix(evt, source, self.main_intent,
-                                                                    is_edit=True)
-        intent = sender.intent if sender else self.main_intent
-        await intent.set_typing(self.mxid, is_typing=False)
-        response = await intent.send_text(self.mxid, text, html=html, relates_to=relates_to,
-                                          external_url=self.get_external_url(evt))
-
-        mxid = response["event_id"]
-
-        msg = DBMessage.get_by_tgid(TelegramID(evt.id), tg_space)
-        if not msg:
+        text, html, _ = await formatter.telegram_to_matrix(evt, source, self.main_intent,
+                                                           no_reply_fallback=True)
+        editing_msg = DBMessage.get_one_by_tgid(TelegramID(evt.id), tg_space)
+        if not editing_msg:
             self.log.info(f"Didn't find edited message {evt.id}@{tg_space} (src {source.tgid}) "
                           "in database.")
-            # Oh crap
             return
-        msg.update(mxid=mxid, mx_room=self.mxid)
+
+        msgtype = ("m.notice"
+                   if sender and sender.is_bot and self.get_config("bot_messages_as_notices")
+                   else "m.text")
+        content = {
+            "body": f"Edit: {text}",
+            "msgtype": msgtype,
+            "format": "org.matrix.custom.html",
+            "formatted_body": (f"<a href='https://matrix.to/#/{editing_msg.mx_room}/"
+                               f"{editing_msg.mxid}'>Edit</a>: "
+                               f"{html or escape_html(text)}"),
+            "external_url": self.get_external_url(evt),
+            "m.new_content": {
+                "body": text,
+                "msgtype": "m.text",
+                **({"format": "org.matrix.custom.html",
+                    "formatted_body": html} if html else {}),
+            },
+            "m.relates_to": {
+                "rel_type": "m.replace",
+                "event_id": editing_msg.mxid,
+            },
+        }
+
+        intent = sender.intent if sender else self.main_intent
+        await intent.set_typing(self.mxid, is_typing=False)
+        response = await intent.send_message(self.mxid, content)
+        mxid = response["event_id"]
+
+        prev_edit_msg = DBMessage.get_one_by_tgid(TelegramID(evt.id), tg_space, -1) or editing_msg
+        DBMessage(mxid=mxid, mx_room=self.mxid, tg_space=tg_space, tgid=evt.id,
+                  edit_index=prev_edit_msg.edit_index + 1).insert()
         DBMessage.update_by_mxid(temporary_identifier, self.mxid, mxid=mxid)
 
     async def handle_telegram_message(self, source: 'AbstractUser', sender: p.Puppet,
@@ -1643,11 +1756,11 @@ class Portal:
                            f"as it was already handled (in space {other_tg_space})")
             if tg_space != other_tg_space:
                 DBMessage(tgid=TelegramID(evt.id), mx_room=self.mxid, mxid=mxid,
-                          tg_space=tg_space).insert()
+                          tg_space=tg_space, edit_index=0).insert()
             return
 
         if self.dedup_pre_db_check and self.peer_type == "channel":
-            msg = DBMessage.get_by_tgid(TelegramID(evt.id), tg_space)
+            msg = DBMessage.get_one_by_tgid(TelegramID(evt.id), tg_space)
             if msg:
                 self.log.debug(f"Ignoring message {evt.id} (src {source.tgid}) as it was already"
                                f"handled into {msg.mxid}. This duplicate was catched in the db "
@@ -1661,8 +1774,8 @@ class Portal:
             entity = await source.client.get_entity(PeerUser(sender.tgid))
             await sender.update_info(source, entity)
 
-        allowed_media = (MessageMediaPhoto, MessageMediaDocument, MessageMediaGeo, MessageMediaGame,
-                         MessageMediaPoll, MessageMediaUnsupported)
+        allowed_media = (MessageMediaPhoto, MessageMediaDocument, MessageMediaGeo,
+                         MessageMediaGame, MessageMediaPoll, MessageMediaUnsupported)
         media = evt.media if hasattr(evt, "media") and isinstance(evt.media,
                                                                   allowed_media) else None
         intent = sender.intent if sender else self.main_intent
@@ -1702,7 +1815,7 @@ class Portal:
         self.log.debug("Handled Telegram message: %s", evt)
         try:
             DBMessage(tgid=TelegramID(evt.id), mx_room=self.mxid, mxid=mxid,
-                      tg_space=tg_space).insert()
+                      tg_space=tg_space, edit_index=0).insert()
             DBMessage.update_by_mxid(temporary_identifier, self.mxid, mxid=mxid)
         except IntegrityError as e:
             self.log.exception(f"{e.__class__.__name__} while saving message mapping. "
@@ -1735,8 +1848,7 @@ class Portal:
         if isinstance(action, MessageActionChatEditTitle):
             await self.update_title(action.title, save=True)
         elif isinstance(action, MessageActionChatEditPhoto):
-            largest_size = self._get_largest_photo_size(action.photo)
-            await self.update_avatar(source, largest_size.location, save=True)
+            await self.update_avatar(source, action.photo, save=True)
         elif isinstance(action, MessageActionChatDeletePhoto):
             await self.remove_avatar(source, save=True)
         elif isinstance(action, MessageActionChatAddUser):
@@ -1781,7 +1893,7 @@ class Portal:
         self._temp_pinned_message_id = None
         self._temp_pinned_message_sender = None
 
-        message = DBMessage.get_by_tgid(msg_id, self._temp_pinned_message_id_space)
+        message = DBMessage.get_one_by_tgid(msg_id, self._temp_pinned_message_id_space)
         if message:
             await intent.set_pinned_messages(self.mxid, [message.mxid])
         else:
@@ -1903,7 +2015,7 @@ class Portal:
             existing.delete()
         except KeyError:
             pass
-        self.db_instance.update(tgid=new_id, tg_receiver=new_id)
+        self.db_instance.update(tgid=new_id, tg_receiver=new_id, peer_type=self.peer_type)
         old_id = self.tgid
         self.tgid = new_id
         self.tg_receiver = new_id

mautrix_telegram/puppet.py

@@ -22,7 +22,8 @@ import asyncio
 import logging
 import re
 
-from telethon.tl.types import UserProfilePhoto, User, FileLocation, UpdateUserName, PeerUser
+from telethon.tl.types import (UserProfilePhoto, User, UpdateUserName, PeerUser, TypeInputPeer,
+                               InputPeerPhotoFileLocation, UserProfilePhotoEmpty)
 from mautrix_appservice import AppService, IntentAPI, IntentError, MatrixRequestError
 
 from .types import MatrixUserID, TelegramID
@@ -61,6 +62,7 @@ class Puppet:
                  photo_id: Optional[str] = None,
                  is_bot: bool = False,
                  is_registered: bool = False,
+                 disable_updates: bool = False,
                  db_instance: Optional[DBPuppet] = None) -> None:
         self.id = id  # type: TelegramID
         self.access_token = access_token  # type: Optional[str]
@@ -73,6 +75,7 @@ class Puppet:
         self.photo_id = photo_id  # type: Optional[str]
         self.is_bot = is_bot  # type: bool
         self.is_registered = is_registered  # type: bool
+        self.disable_updates = disable_updates  # type: bool
         self._db_instance = db_instance  # type: Optional[DBPuppet]
 
         self.default_mxid_intent = self.az.intent.user(self.default_mxid)
@@ -111,6 +114,9 @@ class Puppet:
         match = regex.match(self.displayname)
         return match.group(1) or self.displayname
 
+    def get_input_entity(self, user: 'AbstractUser') -> Awaitable[TypeInputPeer]:
+        return user.client.get_input_entity(PeerUser(user_id=self.tgid))
+
     # region Custom puppet management
     def _fresh_intent(self) -> IntentAPI:
         return (self.az.intent.user(self.custom_mxid, self.access_token)
@@ -281,23 +287,26 @@ class Puppet:
         return DBPuppet(id=self.id, access_token=self.access_token, custom_mxid=self.custom_mxid,
                         username=self.username, displayname=self.displayname,
                         displayname_source=self.displayname_source, photo_id=self.photo_id,
-                        is_bot=self.is_bot, matrix_registered=self.is_registered)
+                        is_bot=self.is_bot, matrix_registered=self.is_registered,
+                        disable_updates=self.disable_updates)
 
     @classmethod
     def from_db(cls, db_puppet: DBPuppet) -> 'Puppet':
         return Puppet(db_puppet.id, db_puppet.access_token, db_puppet.custom_mxid,
                       db_puppet.username, db_puppet.displayname, db_puppet.displayname_source,
                       db_puppet.photo_id, db_puppet.is_bot, db_puppet.matrix_registered,
-                      db_instance=db_puppet)
+                      db_puppet.disable_updates, db_instance=db_puppet)
 
     def save(self) -> None:
         self.db_instance.update(access_token=self.access_token, custom_mxid=self.custom_mxid,
                                 username=self.username, displayname=self.displayname,
                                 displayname_source=self.displayname_source, photo_id=self.photo_id,
-                                is_bot=self.is_bot, matrix_registered=self.is_registered)
+                                is_bot=self.is_bot, matrix_registered=self.is_registered,
+                                disable_updates=self.disable_updates)
 
     # endregion
     # region Info updating
+
     def similarity(self, query: str) -> int:
         username_similarity = (SequenceMatcher(None, self.username, query).ratio()
                                if self.username else 0)
@@ -334,6 +343,8 @@ class Puppet:
             displayname=name)
 
     async def update_info(self, source: 'AbstractUser', info: User) -> None:
+        if self.disable_updates:
+            return
         changed = False
         if self.username != info.username:
             self.username = info.username
@@ -341,7 +352,7 @@ class Puppet:
 
         changed = await self.update_displayname(source, info) or changed
         if isinstance(info.photo, UserProfilePhoto):
-            changed = await self.update_avatar(source, info.photo.photo_big) or changed
+            changed = await self.update_avatar(source, info.photo) or changed
 
         self.is_bot = info.bot
 
@@ -350,33 +361,66 @@ class Puppet:
 
     async def update_displayname(self, source: 'AbstractUser', info: Union[User, UpdateUserName]
                                  ) -> bool:
-        ignore_source = (not source.is_relaybot
-                         and self.displayname_source is not None
-                         and self.displayname_source != source.tgid)
-        if ignore_source:
+        if self.disable_updates:
             return False
-        if isinstance(info, UpdateUserName):
+        is_main_source = (source.is_relaybot or (self.displayname_source is not None
+                                                 and self.displayname_source == source.tgid))
+        # No phone -> not in contact list -> can't set custom name -> name is trustworthy
+        is_trustworthy_source = isinstance(info, User) and info.phone is None
+        if not is_main_source and not is_trustworthy_source:
+            return False
+        elif isinstance(info, UpdateUserName):
             info = await source.client.get_entity(PeerUser(self.tgid))
 
         displayname = self.get_displayname(info)
         if displayname != self.displayname:
-            await self.default_mxid_intent.set_display_name(displayname)
             self.displayname = displayname
             self.displayname_source = source.tgid
+            try:
+                await self.default_mxid_intent.set_display_name(displayname)
+            except MatrixRequestError:
+                self.log.exception("Failed to set displayname")
+                self.displayname = ""
+                self.displayname_source = None
             return True
         elif source.is_relaybot or self.displayname_source is None:
             self.displayname_source = source.tgid
             return True
         return False
 
-    async def update_avatar(self, source: 'AbstractUser', photo: FileLocation) -> bool:
-        photo_id = f"{photo.volume_id}-{photo.local_id}"
+    async def update_avatar(self, source: 'AbstractUser',
+                            photo: Union[UserProfilePhoto, UserProfilePhotoEmpty]) -> bool:
+        if self.disable_updates:
+            return False
+
+        if isinstance(photo, UserProfilePhotoEmpty):
+            photo_id = ""
+        else:
+            photo_id = str(photo.photo_id)
         if self.photo_id != photo_id:
-            file = await util.transfer_file_to_matrix(source.client, self.default_mxid_intent,
-                                                      photo)
+            if not photo_id:
+                self.photo_id = ""
+                try:
+                    await self.default_mxid_intent.set_avatar("")
+                except MatrixRequestError:
+                    self.log.exception("Failed to set avatar")
+                    self.photo_id = ""
+                return True
+
+            loc = InputPeerPhotoFileLocation(
+                peer=await self.get_input_entity(source),
+                local_id=photo.photo_big.local_id,
+                volume_id=photo.photo_big.volume_id,
+                big=True
+            )
+            file = await util.transfer_file_to_matrix(source.client, self.default_mxid_intent, loc)
             if file:
-                await self.default_mxid_intent.set_avatar(file.mxc)
                 self.photo_id = photo_id
+                try:
+                    await self.default_mxid_intent.set_avatar(file.mxc)
+                except MatrixRequestError:
+                    self.log.exception("Failed to set avatar")
+                    self.photo_id = ""
                 return True
         return False
 

mautrix_telegram/util/__init__.py

@@ -3,6 +3,5 @@ from .format_duration import format_duration
 from .signed_token import sign_token, verify_token
 from .recursive_dict import recursive_del, recursive_set, recursive_get
 
-
 def ignore_coro(coro):
     pass

mautrix_telegram/util/file_transfer.py

@@ -23,14 +23,16 @@ import asyncio
 import magic
 from sqlalchemy.exc import IntegrityError, InvalidRequestError
 
-from telethon.tl.types import (Document, FileLocation, InputFileLocation, InputDocumentFileLocation,
-                               TypePhotoSize, PhotoSize, PhotoCachedSize)
+from telethon.tl.types import (Document, InputFileLocation, InputDocumentFileLocation,
+                               TypePhotoSize, PhotoSize, PhotoCachedSize, InputPhotoFileLocation,
+                               InputPeerPhotoFileLocation)
 from telethon.errors import (AuthBytesInvalidError, AuthKeyInvalidError, LocationInvalidError,
                              SecurityError)
 from mautrix_appservice import IntentAPI
 
 from ..tgclient import MautrixTelegramClient
 from ..db import TelegramFile as DBTelegramFile
+from ..util import sane_mimetypes
 
 try:
     from PIL import Image
@@ -47,7 +49,8 @@ except ImportError:
 
 log = logging.getLogger("mau.util")  # type: logging.Logger
 
-TypeLocation = Union[Document, InputDocumentFileLocation, FileLocation, InputFileLocation]
+TypeLocation = Union[Document, InputDocumentFileLocation, InputPeerPhotoFileLocation,
+                     InputFileLocation, InputPhotoFileLocation]
 
 
 def convert_image(file: bytes, source_mime: str = "image/webp", target_type: str = "png",
@@ -99,9 +102,9 @@ def _read_video_thumbnail(data: bytes, video_ext: str = "mp4", frame_ext: str =
 
 
 def _location_to_id(location: TypeLocation) -> str:
-    if isinstance(location, (Document, InputDocumentFileLocation)):
+    if isinstance(location, (Document, InputDocumentFileLocation, InputPhotoFileLocation)):
         return f"{location.id}-{location.access_hash}"
-    elif isinstance(location, (FileLocation, InputFileLocation)):
+    elif isinstance(location, (InputFileLocation, InputPeerPhotoFileLocation)):
         return f"{location.volume_id}-{location.local_id}"
 
 
@@ -119,7 +122,7 @@ async def transfer_thumbnail_to_matrix(client: MautrixTelegramClient, intent: In
     if db_file:
         return db_file
 
-    video_ext = mimetypes.guess_extension(mime)
+    video_ext = sane_mimetypes.guess_extension(mime)
     if VideoFileClip and video_ext:
         try:
             file, width, height = _read_video_thumbnail(video, video_ext, frame_ext="png")

mautrix_telegram/util/sane_mimetypes.py

@@ -0,0 +1,38 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+import mimetypes
+
+mimetypes.init()
+
+sanity_overrides = {
+    "image/jpeg": ".jpeg",
+    "image/tiff": ".tiff",
+    "text/plain": ".txt",
+    "text/html": ".html",
+    "audio/mpeg": ".mp3",
+    "audio/ogg": ".ogg",
+    "application/xml": ".xml",
+    "application/octet-stream": "",
+    "application/x-msdos-program": ".exe",
+}
+
+
+def guess_extension(mime: str) -> str:
+    try:
+        return sanity_overrides[mime]
+    except KeyError:
+        return mimetypes.guess_extension(mime)

mautrix_telegram/web/common/auth_api.py

@@ -56,7 +56,7 @@ class AuthAPI(abc.ABC):
                                               error="You have already logged in with your Matrix "
                                                     "account.", errcode="already-logged-in")
 
-        resp = await puppet.switch_mxid(token, user.mxid)
+        resp = await puppet.switch_mxid(token.strip(), user.mxid)
         if resp == PuppetError.OnlyLoginSelf:
             return self.get_mx_login_response(status=403, errcode="only-login-self",
                                               error="You can only log in as your own Matrix user.")
@@ -72,10 +72,15 @@ class AuthAPI(abc.ABC):
                                           errcode="not-yet-implemented")
 
     async def post_login_phone(self, user: User, phone: str) -> web.Response:
+        if not phone or not phone.strip():
+            return self.get_login_response(mxid=user.mxid, state="request", status=400,
+                                           errcode="phone_number_invalid",
+                                           error="Phone number not given.")
         try:
-            await user.client.sign_in(phone or "+123")
+            await user.client.sign_in(phone.strip())
             return self.get_login_response(mxid=user.mxid, state="code", status=200,
-                                           message="Code requested successfully.")
+                                           message="Code requested successfully. Check your SMS "
+                                                   "or Telegram client and enter the code below.")
         except PhoneNumberInvalidError:
             return self.get_login_response(mxid=user.mxid, state="request", status=400,
                                            errcode="phone_number_invalid",
@@ -117,10 +122,9 @@ class AuthAPI(abc.ABC):
         if user.command_status and user.command_status["action"] == "Login":
             user.command_status = None
 
-
     async def post_login_token(self, user: User, token: str) -> web.Response:
         try:
-            user_info = await user.client.sign_in(bot_token=token)
+            user_info = await user.client.sign_in(bot_token=token.strip())
             await self.postprocess_login(user, user_info)
             return self.get_login_response(mxid=user.mxid, state="logged-in", status=200,
                                            username=user_info.username, phone=None,
@@ -174,7 +178,7 @@ class AuthAPI(abc.ABC):
 
     async def post_login_password(self, user: User, password: str) -> web.Response:
         try:
-            user_info = await user.client.sign_in(password=password)
+            user_info = await user.client.sign_in(password=password.strip())
             await self.postprocess_login(user, user_info)
             human_tg_id = f"@{user_info.username}" if user_info.username else f"+{user_info.phone}"
             return self.get_login_response(mxid=user.mxid, state="logged-in", status=200,

mautrix_telegram/web/provisioning/__init__.py

@@ -365,7 +365,7 @@ class ProvisioningAPI(AuthAPI):
 
     async def bridge_info(self, request: web.Request) -> web.Response:
         return web.json_response({
-            "relaybot_username": self.context.bot.username,
+            "relaybot_username": self.context.bot.username if self.context.bot is not None else None,
         }, status=200)
 
     @staticmethod

mautrix_telegram/web/public/__init__.py

@@ -87,7 +87,8 @@ class PublicBridgeWebsite(AuthAPI):
         return self.get_login_response(mxid=user.mxid, human_tg_id=user.human_tg_id)
 
     async def get_matrix_login(self, request: web.Request) -> web.Response:
-        mxid = self.verify_token(request.rel_url.query.get("token", None), endpoint="/matrix-login")
+        mxid = self.verify_token(request.rel_url.query.get("token", None),
+                                 endpoint="/matrix-login")
         if not mxid:
             return self.get_mx_login_response(status=401, state="invalid-token")
         user = User.get_by_mxid(mxid, create=False) if mxid else None
@@ -124,7 +125,8 @@ class PublicBridgeWebsite(AuthAPI):
                                                       error=error, message=message, mxid=mxid))
 
     async def post_matrix_login(self, request: web.Request) -> web.Response:
-        mxid = self.verify_token(request.rel_url.query.get("token", None), endpoint="/matrix-login")
+        mxid = self.verify_token(request.rel_url.query.get("token", None),
+                                 endpoint="/matrix-login")
         if not mxid:
             return self.get_mx_login_response(status=401, state="invalid-token")
 
@@ -167,7 +169,13 @@ class PublicBridgeWebsite(AuthAPI):
         elif "bot_token" in data:
             return await self.post_login_token(user, data["bot_token"])
         elif "code" in data:
-            resp = await self.post_login_code(user, data["code"],
+            try:
+                code = int(data["code"].strip())
+            except ValueError:
+                return self.get_login_response(mxid=user.mxid, state="code", status=400,
+                                               errcode="phone_code_invalid",
+                                               error="Phone code must be a number.")
+            resp = await self.post_login_code(user, code,
                                               password_in_data="password" in data)
             if resp or "password" not in data:
                 return resp

mautrix_telegram/web/public/login.html.mako

@@ -95,8 +95,8 @@ along with this program.  If not, see <https://www.gnu.org/licenses/>.
 				% if state == "request":
 					<label for="value">Phone number</label>
 					<input type="tel" id="value" name="phone" placeholder="Enter phone number"/>
-					<button type="submit">Request code</button>
-					<button class="button-clear" type="button" onclick="switchToBotLogin()">
+					<button type="submit">Start</button>
+					<button class="button-clear float-right" type="button" onclick="switchToBotLogin()">
 						Use bot token
 					</button>
 				% elif state == "bot_token":

optional-requirements.txt

@@ -1,4 +1,4 @@
-lxml
 cryptg
 Pillow
 moviepy
+prometheus-client

setup.py

@@ -3,10 +3,10 @@ import glob
 import mautrix_telegram
 
 extras = {
-    "highlight_edits": ["lxml>=4.1.1,<5"],
-    "fast_crypto": ["cryptg>=0.1,<0.2"],
-    "webp_convert": ["Pillow>=4.3.0,<6"],
+    "fast_crypto": ["cryptg>=0.1,<0.3"],
+    "webp_convert": ["Pillow>=4.3.0,<7"],
     "hq_thumbnails": ["moviepy>=1.0,<2.0"],
+    "metrics": ["prometheus-client>=0.6.0,<0.7.0"],
 }
 extras["all"] = list({dep for deps in extras.values() for dep in deps})
 
@@ -31,15 +31,15 @@ setuptools.setup(
 
     install_requires=[
         "aiohttp>=3.0.1,<4",
-        "mautrix-appservice>=0.3.8,<0.4.0",
+        "mautrix-appservice>=0.3.11,<0.4.0",
         "SQLAlchemy>=1.2.3,<2",
         "alembic>=1.0.0,<2",
         "commonmark>=0.8.1,<1",
         "ruamel.yaml>=0.15.35,<0.16",
         "future-fstrings>=0.4.2",
         "python-magic>=0.4.15,<0.5",
-        "telethon>=1.5.5,<1.7",
-        "telethon-session-sqlalchemy>=0.2.12,<0.3",
+        "telethon>=1.7,<1.9",
+        "telethon-session-sqlalchemy>=0.2.14,<0.3",
     ],
     extras_require=extras,