Bump version to 0.6.1rc2

Users who the bridge only saw via logged in users with the target user
in their contact lists wouldn't get their displayname updated due to an
invalid condition in the update_displayname function.

.gitignore

@@ -5,8 +5,13 @@ env/
 pip-selfcheck.json
 *.pyc
 __pycache__
+build
+dist
+*.egg-info
+.eggs
 
 config.yaml
 registration.yaml
 *.log*
 *.db
+*.bak

.gitlab-ci.yml

@@ -0,0 +1,38 @@
+image: docker:stable
+
+stages:
+- build
+- push
+
+default:
+  before_script:
+  - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
+
+build:
+  stage: build
+  script:
+  - docker pull $CI_REGISTRY_IMAGE:latest || true
+  - docker build --pull --cache-from $CI_REGISTRY_IMAGE:latest --tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA .
+  - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA
+
+push latest:
+  stage: push
+  only:
+  - master
+  variables:
+    GIT_STRATEGY: none
+  script:
+  - docker pull $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA
+  - docker tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA $CI_REGISTRY_IMAGE:latest
+  - docker push $CI_REGISTRY_IMAGE:latest
+
+push tag:
+  stage: push
+  variables:
+    GIT_STRATEGY: none
+  except:
+  - master
+  script:
+  - docker pull $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA
+  - docker tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME
+  - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME

Dockerfile

@@ -1,4 +1,4 @@
-FROM docker.io/alpine:3.9
+FROM docker.io/alpine:3.10
 
 ENV UID=1337 \
     GID=1337 \
@@ -10,25 +10,29 @@ RUN apk add --no-cache \
       py3-virtualenv \
       py3-pillow \
       py3-aiohttp \
-      py3-lxml \
       py3-magic \
       py3-sqlalchemy \
       py3-markdown \
       py3-psycopg2 \
+      py3-ruamel.yaml \
       # Indirect dependencies
-      py3-numpy \
-      py3-asn1crypto \
-      py3-future \
-      py3-markupsafe \
-      py3-mako \
-      py3-decorator \
-      py3-dateutil \
-      py3-idna \
-      py3-six \
-      py3-asn1 \
-      py3-rsa \
+      #commonmark
+        py3-future \
+      #alembic
+        py3-mako \
+        py3-dateutil \
+        py3-markupsafe \
+      #moviepy
+        py3-decorator \
+        #py3-tqdm \
+        py3-requests \
+        #imageio
+          py3-numpy \
+      #telethon
+        py3-rsa \
       # Other dependencies
       python3-dev \
+      libffi-dev \
       build-base \
       ffmpeg \
       ca-certificates \

ROADMAP.md

@@ -3,6 +3,7 @@
 * Matrix → Telegram
   * [x] Message content (text, formatting, files, etc..)
   * [x] Message redactions
+  * [x] Message edits
   * [ ] ‡ Message history
   * [x] Presence
   * [x] Typing notifications
@@ -21,6 +22,10 @@
     * [ ] ‡ Changes to displayname/avatar
 * Telegram → Matrix
   * [x] Message content (text, formatting, files, etc..)
+  * [ ] Advanced message content/media
+    * [x] Polls
+	* [x] Games
+	* [ ] Buttons
   * [x] Message deletions
   * [x] Message edits
   * [ ] Message history
@@ -48,6 +53,8 @@
   * [x] Option to use bot to relay messages for unauthenticated Matrix users
   * [x] Option to use own Matrix account for messages sent from other Telegram clients
   * [ ] ‡ Calls (hard, not yet supported by Telethon)
+  * [ ] ‡ Secret chats (not yet supported by Telethon)
+  * [ ] ‡ E2EE in Matrix rooms (not yet supported 
 
 † Information not automatically sent from source, i.e. implementation may not be possible  
 ‡ Maybe, i.e. this feature may or may not be implemented at some point

alembic/versions/17574c57f3f8_add_disable_updates_field_for_puppets.py

@@ -0,0 +1,27 @@
+"""Add disable_updates field for puppets
+
+Revision ID: 17574c57f3f8
+Revises: a9119be92164
+Create Date: 2019-05-15 00:24:46.967529
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '17574c57f3f8'
+down_revision = 'a9119be92164'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    with op.batch_alter_table("puppet") as batch_op:
+        batch_op.add_column(sa.Column("disable_updates", sa.Boolean(), nullable=False,
+                                      server_default=sa.sql.expression.false()))
+
+
+def downgrade():
+    with op.batch_alter_table("puppet") as batch_op:
+        batch_op.drop_column("disable_updates")

alembic/versions/1fa46383a9d3_add_is_bot_field_to_puppets.py

@@ -17,7 +17,8 @@ depends_on = None
 
 
 def upgrade():
-    op.add_column('puppet', sa.Column('is_bot', sa.Boolean(), nullable=True))
+    with op.batch_alter_table("puppet") as batch_op:
+        batch_op.add_column(sa.Column('is_bot', sa.Boolean(), nullable=True))
 
 
 def downgrade():

alembic/versions/30eca60587f1_add_megagroup_field_to_portals.py

@@ -16,7 +16,8 @@ depends_on = None
 
 
 def upgrade():
-    op.add_column('portal', sa.Column('megagroup', sa.Boolean()))
+    with op.batch_alter_table("portal") as batch_op:
+        batch_op.add_column(sa.Column('megagroup', sa.Boolean()))
 
 
 def downgrade():

alembic/versions/6ca3d74d51e4_move_state_store_to_main_database.py

@@ -57,7 +57,8 @@ class Puppet(Base):
 
 
 def upgrade():
-    op.add_column("puppet", sa.Column("matrix_registered", sa.Boolean(), nullable=False,
+    with op.batch_alter_table("puppet") as batch_op:
+        batch_op.add_column(sa.Column("matrix_registered", sa.Boolean(), nullable=False,
                                       server_default=sa.sql.expression.false()))
     op.create_table("mx_room_state",
                     sa.Column("room_id", sa.String(), nullable=False),

alembic/versions/9e9c89b0b877_add_edit_index_to_messages.py

@@ -0,0 +1,48 @@
+"""Add edit index to messages
+
+Revision ID: 9e9c89b0b877
+Revises: 17574c57f3f8
+Create Date: 2019-05-29 15:28:23.128377
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = '9e9c89b0b877'
+down_revision = '17574c57f3f8'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table('_message_temp',
+                    sa.Column('mxid', sa.String),
+                    sa.Column('mx_room', sa.String),
+                    sa.Column('tgid', sa.Integer),
+                    sa.Column('tg_space', sa.Integer),
+                    sa.Column('edit_index', sa.Integer),
+                    sa.PrimaryKeyConstraint('tgid', 'tg_space', 'edit_index'),
+                    sa.UniqueConstraint("mxid", "mx_room", "tg_space", name="_mx_id_room_2"))
+    c = op.get_bind()
+    c.execute("INSERT INTO _message_temp (mxid, mx_room, tgid, tg_space, edit_index) "
+              "SELECT message.mxid, message.mx_room, message.tgid, message.tg_space, 0 "
+              "FROM message")
+    c.execute("DROP TABLE message")
+    c.execute("ALTER TABLE _message_temp RENAME TO message")
+
+
+def downgrade():
+    op.create_table('_message_temp',
+                    sa.Column('mxid', sa.String),
+                    sa.Column('mx_room', sa.String),
+                    sa.Column('tgid', sa.Integer),
+                    sa.Column('tg_space', sa.Integer),
+                    sa.PrimaryKeyConstraint('tgid', 'tg_space'),
+                    sa.UniqueConstraint("mxid", "mx_room", "tg_space", name="_mx_id_room"))
+    c = op.get_bind()
+    c.execute("INSERT INTO _message_temp (mxid, mx_room, tgid, tg_space) "
+              "SELECT message.mxid, message.mx_room, message.tgid, message.tg_space "
+              "FROM message")
+    c.execute("DROP TABLE message")
+    c.execute("ALTER TABLE _message_temp RENAME TO message")

example-config.yaml

@@ -60,10 +60,19 @@ appservice:
     bot_displayname: Telegram bridge bot
     bot_avatar: mxc://maunium.net/tJCRmUyJDsgRNgqhOgoiHWbX
 
+    # Community ID for bridged users (changes registration file) and rooms.
+    # Must be created manually.
+    community_id: false
+
     # Authentication tokens for AS <-> HS communication. Autogenerated; do not modify.
     as_token: "This value is generated when generating the registration"
     hs_token: "This value is generated when generating the registration"
 
+# Prometheus telemetry config. Requires prometheus-client to be installed.
+metrics:
+    enabled: false
+    listen_port: 8000
+
 # Bridge config
 bridge:
     # Localpart template of MXIDs for Telegram users.
@@ -126,15 +135,11 @@ bridge:
     # Only enable this if your displayname_template has some static part that the bridge can use to
     # reliably identify what is a plaintext highlight.
     plaintext_highlights: false
-    # Show message editing as a reply to the original message.
-    # If this is false, message edits are not shown at all, as Matrix does not support editing yet.
-    edits_as_replies: true
-    # Highlight changed/added parts in edits. Requires lxml.
-    highlight_edits: false
     # Whether or not to make portals of publicly joinable channels/supergroups publicly joinable on Matrix.
     public_portals: true
     # Whether or not to fetch and handle Telegram updates at startup from the time the bridge was down.
     # Currently only works for private chats and normal groups.
+    # WARNING: This feature seems to be broken in the telegram library.
     catch_up: false
     # Whether or not to use /sync to get presence, read receipts and typing notifications when using
     # your own Matrix account as the Matrix puppet for your Telegram account.
@@ -146,6 +151,8 @@ bridge:
     inline_images: false
     # Maximum size of image in megabytes before sending to Telegram as a document.
     image_as_file_size: 10
+    # Maximum size of Telegram documents in megabytes to bridge.
+    max_document_size: 100
 
     # Whether to bridge Telegram bot messages as m.notices or m.texts.
     bot_messages_as_notices: true
@@ -251,6 +258,40 @@ telegram:
     api_hash: tjyd5yge35lbodk1xwzw2jstp90k55qz
     # (Optional) Create your own bot at https://t.me/BotFather
     bot_token: disabled
+
+    # Telethon connection options.
+    connection:
+        # The timeout in seconds to be used when connecting.
+        timeout: 120
+        # How many times the reconnection should retry, either on the initial connection or when
+        # Telegram disconnects us. May be set to a negative or null value for infinite retries, but
+        # this is not recommended, since the program can get stuck in an infinite loop.
+        retries: 5
+        # The delay in seconds to sleep between automatic reconnections.
+        retry_delay: 1
+        # The threshold below which the library should automatically sleep on flood wait errors
+        # (inclusive). For instance, if a FloodWaitError for 17s occurs and flood_sleep_threshold
+        # is 20s, the library will sleep automatically. If the error was for 21s, it would raise
+        # the error instead. Values larger than a day (86400) will be changed to a day.
+        flood_sleep_threshold: 60
+        # How many times a request should be retried. Request are retried when Telegram is having
+        # internal issues, when there is a FloodWaitError less than flood_sleep_threshold, or when
+        # there's a migrate error. May take a negative or null value for infinite retries, but this
+        # is not recommended, since some requests can always trigger a call fail (such as searching
+        # for messages).
+        request_retries: 5
+
+    # Device info sent to Telegram.
+    device_info:
+        # "auto" = OS name+version.
+        device_model: auto
+        # "auto" = Telethon version.
+        system_version: auto
+        # "auto" = mautrix-telegram version.
+        app_version: auto
+        lang_code: en
+        system_lang_code: en
+
     # Custom server to connect to.
     server:
         # Set to true to use these server settings. If false, will automatically
@@ -262,6 +303,7 @@ telegram:
         ip: 149.154.167.40
         # The port to connect to. 443 may not work, 80 is better and both are equally secure.
         port: 80
+
     # Telethon proxy configuration.
     # You must install PySocks from pip for proxies to work.
     proxy:

mautrix_telegram/__init__.py

@@ -1,2 +1,2 @@
-__version__ = "0.5.0rc4"
+__version__ = "0.6.1rc2"
 __author__ = "Tulir Asokan <tulir@maunium.net>"

mautrix_telegram/__main__.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -22,6 +22,7 @@ import logging.config
 import sys
 import copy
 import signal
+import os
 
 import sqlalchemy as sql
 
@@ -43,6 +44,11 @@ from .sqlstatestore import SQLStateStore
 from .user import User, init as init_user
 from . import __version__
 
+try:
+    import prometheus_client as prometheus
+except ImportError:
+    prometheus = None
+
 parser = argparse.ArgumentParser(
     description="A Matrix-Telegram puppeting bridge.",
     prog="python -m mautrix-telegram")
@@ -57,7 +63,7 @@ parser.add_argument("-r", "--registration", type=str, default="registration.yaml
                     metavar="<path>", help="the path to save the generated registration to")
 args = parser.parse_args()
 
-config = Config(args.config, args.registration, args.base_config)
+config = Config(args.config, args.registration, args.base_config, os.environ)
 config.load()
 config.update()
 
@@ -100,7 +106,6 @@ appserv = AppService(config["homeserver.address"], config["homeserver.domain"],
                      })
 bot = init_bot(config)
 context = Context(appserv, config, loop, session_container, bot)
-context.mx = MatrixHandler(context)
 
 if config["appservice.public.enabled"]:
     public_website = PublicBridgeWebsite(loop)
@@ -113,6 +118,14 @@ if config["appservice.provisioning.enabled"]:
                            provisioning_api.app)
     context.provisioning_api = provisioning_api
 
+context.mx = MatrixHandler(context)
+
+if config["metrics.enabled"]:
+    if prometheus:
+        prometheus.start_http_server(config["metrics.listen_port"])
+    else:
+        log.warn("Metrics are enabled in the config, but prometheus-async is not installed.")
+
 with appserv.run(config["appservice.hostname"], config["appservice.port"]) as start:
     start_ts = time()
     init_db(db_engine)

mautrix_telegram/abstract_user.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -14,11 +14,12 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Tuple, Optional, List, Union, TYPE_CHECKING
+from typing import Tuple, Optional, List, Union, Dict, TYPE_CHECKING
 from abc import ABC, abstractmethod
 import asyncio
 import logging
 import platform
+import time
 
 from telethon.tl.patched import MessageService, Message
 from telethon.tl.types import (
@@ -50,6 +51,14 @@ UpdateMessage = Union[UpdateShortChatMessage, UpdateShortMessage, UpdateNewChann
                       UpdateNewMessage, UpdateEditMessage, UpdateEditChannelMessage]
 UpdateMessageContent = Union[UpdateShortMessage, UpdateShortChatMessage, Message, MessageService]
 
+try:
+    from prometheus_client import Histogram
+
+    UPDATE_TIME = Histogram("telegram_update", "Time spent processing Telegram updates",
+                            ["update_type"])
+except ImportError:
+    Histogram = None
+    UPDATE_TIME = None
 
 class AbstractUser(ABC):
     session_container = None  # type: AlchemySessionContainer
@@ -95,27 +104,43 @@ class AbstractUser(ABC):
 
     def _init_client(self) -> None:
         self.log.debug(f"Initializing client for {self.name}")
-        device = f"{platform.system()} {platform.release()}"
-        sysversion = MautrixTelegramClient.__version__
+
         self.session = self.session_container.new_session(self.name)
         if config["telegram.server.enabled"]:
             self.session.set_dc(config["telegram.server.dc"],
                                 config["telegram.server.ip"],
                                 config["telegram.server.port"])
+
         if self.is_relaybot:
             base_logger = logging.getLogger("telethon.relaybot")
         else:
             base_logger = logging.getLogger(f"telethon.{self.tgid or -hash(self.mxid)}")
-        self.client = MautrixTelegramClient(session=self.session,
-                                            api_id=config["telegram.api_id"],
-                                            api_hash=config["telegram.api_hash"],
-                                            loop=self.loop,
-                                            app_version=__version__,
-                                            system_version=sysversion,
-                                            device_model=device,
-                                            timeout=120,
-                                            base_logger=base_logger,
-                                            proxy=self._proxy_settings)
+
+        device = config["telegram.device_info.device_model"]
+        sysversion = config["telegram.device_info.system_version"]
+        appversion = config["telegram.device_info.app_version"]
+
+        self.client = MautrixTelegramClient(
+            session=self.session,
+
+            api_id=config["telegram.api_id"],
+            api_hash=config["telegram.api_hash"],
+
+            app_version=__version__ if appversion == "auto" else appversion,
+            system_version=MautrixTelegramClient.__version__ if sysversion == "auto" else sysversion,
+            device_model=f"{platform.system()} {platform.release()}" if device == "auto" else device,
+
+            timeout=config["telegram.connection.timeout"],
+            connection_retries=config["telegram.connection.retries"],
+            retry_delay=config["telegram.connection.retry_delay"],
+            flood_sleep_threshold=config["telegram.connection.flood_sleep_threshold"],
+            request_retries=config["telegram.connection.request_retries"],
+
+            proxy=self._proxy_settings,
+
+            loop=self.loop,
+            base_logger=base_logger
+        )
         self.client.add_event_handler(self._update_catch)
 
     @abstractmethod
@@ -135,11 +160,14 @@ class AbstractUser(ABC):
         raise NotImplementedError()
 
     async def _update_catch(self, update: TypeUpdate) -> None:
+        start_time = time.time()
         try:
             if not await self.update(update):
                 await self._update(update)
         except Exception:
             self.log.exception("Failed to handle Telegram update")
+        if UPDATE_TIME:
+            UPDATE_TIME.labels(update_type=type(update).__name__).observe(time.time() - start_time)
 
     async def get_dialogs(self, limit: int = None) -> List[Union[Chat, Channel]]:
         if self.is_bot:
@@ -185,6 +213,8 @@ class AbstractUser(ABC):
     # region Telegram update handling
 
     async def _update(self, update: TypeUpdate) -> None:
+        asyncio.ensure_future(self._handle_entity_updates(getattr(update, "_entities", {})),
+                              loop=self.loop)
         if isinstance(update, (UpdateShortChatMessage, UpdateShortMessage, UpdateNewChannelMessage,
                                UpdateNewMessage, UpdateEditMessage, UpdateEditChannelMessage)):
             await self.update_message(update)
@@ -234,7 +264,7 @@ class AbstractUser(ABC):
             return
 
         # We check that these are user read receipts, so tg_space is always the user ID.
-        message = DBMessage.get_by_tgid(TelegramID(update.max_id), self.tgid)
+        message = DBMessage.get_one_by_tgid(TelegramID(update.max_id), self.tgid, edit_index=-1)
         if not message:
             return
 
@@ -261,6 +291,16 @@ class AbstractUser(ABC):
         sender = pu.Puppet.get(TelegramID(update.user_id))
         await portal.handle_telegram_typing(sender, update)
 
+    async def _handle_entity_updates(self, entities: Dict[int, Union[User, Chat, Channel]]
+                                     ) -> None:
+        try:
+            users = (entity for entity in entities.values() if isinstance(entity, User))
+            puppets = ((pu.Puppet.get(TelegramID(user.id)), user) for user in users)
+            await asyncio.gather(*[puppet.update_info(self, info)
+                                   for puppet, info in puppets if puppet])
+        except Exception:
+            self.log.exception("Failed to handle entity updates")
+
     async def update_others_info(self, update: Union[UpdateUserName, UpdateUserPhoto]) -> None:
         # TODO duplication not checked
         puppet = pu.Puppet.get(TelegramID(update.user_id))
@@ -269,7 +309,7 @@ class AbstractUser(ABC):
             if await puppet.update_displayname(self, update):
                 puppet.save()
         elif isinstance(update, UpdateUserPhoto):
-            if await puppet.update_avatar(self, update.photo.photo_big):
+            if await puppet.update_avatar(self, update.photo):
                 puppet.save()
         else:
             self.log.warning("Unexpected other user info update: %s", update)
@@ -309,7 +349,8 @@ class AbstractUser(ABC):
         return update, sender, portal
 
     @staticmethod
-    async def _try_redact(portal: po.Portal, message: DBMessage) -> None:
+    async def _try_redact(message: DBMessage) -> None:
+        portal = po.Portal.get_by_mxid(message.mx_room)
         if not portal:
             return
         try:
@@ -321,30 +362,26 @@ class AbstractUser(ABC):
         if len(update.messages) > MAX_DELETIONS:
             return
 
-        for message in update.messages:
-            message = DBMessage.get_by_tgid(TelegramID(message), self.tgid)
-            if not message:
-                continue
-            message.delete()
-            number_left = DBMessage.count_spaces_by_mxid(message.mxid, message.mx_room)
-            if number_left == 0:
-                portal = po.Portal.get_by_mxid(message.mx_room)
-                await self._try_redact(portal, message)
+        for message_id in update.messages:
+            messages = DBMessage.get_all_by_tgid(TelegramID(message_id), self.tgid)
+            for message in messages:
+                message.delete()
+                number_left = DBMessage.count_spaces_by_mxid(message.mxid, message.mx_room)
+                if number_left == 0:
+                    portal = po.Portal.get_by_mxid(message.mx_room)
+                    await self._try_redact(message)
 
     async def delete_channel_message(self, update: UpdateDeleteChannelMessages) -> None:
         if len(update.messages) > MAX_DELETIONS:
             return
 
-        portal = po.Portal.get_by_tgid(TelegramID(update.channel_id))
-        if not portal:
-            return
+        channel_id = TelegramID(update.channel_id)
 
-        for message in update.messages:
-            message = DBMessage.get_by_tgid(TelegramID(message), portal.tgid)
-            if not message:
-                continue
-            message.delete()
-            await self._try_redact(portal, message)
+        for message_id in update.messages:
+            messages = DBMessage.get_all_by_tgid(TelegramID(message_id), channel_id)
+            for message in messages:
+                message.delete()
+                await self._try_redact(message)
 
     async def update_message(self, original_update: UpdateMessage) -> None:
         update, sender, portal = self.get_message_details(original_update)
@@ -370,10 +407,7 @@ class AbstractUser(ABC):
 
         user = sender.tgid if sender else "admin"
         if isinstance(original_update, (UpdateEditMessage, UpdateEditChannelMessage)):
-            if config["bridge.edits_as_replies"]:
-                self.log.debug("Handling edit %s to %s by %s", update, portal.tgid_log, user)
-                return await portal.handle_telegram_edit(self, sender, update)
-            return
+            return await portal.handle_telegram_edit(self, sender, update)
 
         self.log.debug("Handling message %s to %s by %s", update, portal.tgid_log, user)
         return await portal.handle_telegram_message(self, sender, update)

mautrix_telegram/bot.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -14,7 +14,7 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Awaitable, Callable, Dict, List, Optional, Pattern, TYPE_CHECKING
+from typing import Awaitable, Callable, Dict, List, Optional, Pattern, Tuple, TYPE_CHECKING
 import logging
 import re
 
@@ -23,7 +23,7 @@ from telethon.tl.types import (
     ChannelParticipantAdmin, ChannelParticipantCreator, ChatForbidden, ChatParticipantAdmin,
     ChatParticipantCreator, InputChannel, InputUser, MessageActionChatAddUser,
     MessageActionChatDeleteUser, MessageEntityBotCommand, PeerChannel, PeerChat, TypePeer,
-    UpdateNewChannelMessage, UpdateNewMessage, MessageActionChatMigrateTo)
+    UpdateNewChannelMessage, UpdateNewMessage, MessageActionChatMigrateTo, User)
 from telethon.tl.functions.messages import GetChatsRequest, GetFullChatRequest
 from telethon.tl.functions.channels import GetChannelsRequest, GetParticipantRequest
 from telethon.errors import ChannelInvalidError, ChannelPrivateError
@@ -60,6 +60,14 @@ class Bot(AbstractUser):
         self.tg_whitelist = []  # type: List[int]
         self.whitelist_group_admins = (config["bridge.relaybot.whitelist_group_admins"]
                                        or False)  # type: bool
+        self._me_info = None  # type: Optional[User]
+        self._me_mxid = None  # type: Optional[MatrixUserID]
+
+    async def get_me(self, use_cache: bool = True) -> Tuple[User, MatrixUserID]:
+        if not use_cache or not self._me_mxid:
+            self._me_info = await self.client.get_me()
+            self._me_mxid = pu.Puppet.get_mxid_from_id(TelegramID(self._me_info.id))
+        return self._me_info, self._me_mxid
 
     async def init_permissions(self) -> None:
         whitelist = config["bridge.relaybot.whitelist"] or []

mautrix_telegram/commands/clean_rooms.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by

mautrix_telegram/commands/handler.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -15,18 +15,7 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 """This module contains classes handling commands issued by Matrix users."""
-from typing import (
-    Any,
-    Awaitable,
-    Callable,
-    Coroutine,
-    Dict,
-    List,
-    NamedTuple,
-    Optional,
-    Union,
-    NewType,
-)
+from typing import Awaitable, Callable, Dict, List, NamedTuple, Optional
 import logging
 import traceback
 
@@ -93,6 +82,7 @@ class CommandEvent:
         is_portal: Determines whether the room in which the command was issued
             is a portal.
     """
+
     def __init__(self, processor: 'CommandProcessor', room: MatrixRoomID, event: MatrixEventID,
                  sender: u.User, command: str, args: List[str], is_management: bool,
                  is_portal: bool) -> None:
@@ -111,12 +101,8 @@ class CommandEvent:
         self.is_management = is_management
         self.is_portal = is_portal
 
-    def reply(
-            self,
-            message: str,
-            allow_html: bool = False,
-            render_markdown: bool = True,
-        ) -> Awaitable[Dict]:
+    def reply(self, message: str, allow_html: bool = False, render_markdown: bool = True
+              ) -> Awaitable[Dict]:
         """Write a reply to the room in which the command was issued.
 
         Replaces occurences of "$cmdprefix" in the message with the command
@@ -136,9 +122,8 @@ class CommandEvent:
             Handler for the message sending function.
         """
         message_cmd = self._replace_command_prefix(message)
-        html = self._render_message(
-            message_cmd, allow_html=allow_html, render_markdown=render_markdown
-        )
+        html = self._render_message(message_cmd, allow_html=allow_html,
+                                    render_markdown=render_markdown)
 
         return self.az.intent.send_notice(self.room_id, message_cmd, html=html)
 
@@ -153,9 +138,8 @@ class CommandEvent:
         )
         return message.replace("$cmdprefix", self.command_prefix)
 
-    def _render_message(
-        self, message: str, allow_html: bool, render_markdown: bool
-    ) -> Optional[str]:
+    @staticmethod
+    def _render_message(message: str, allow_html: bool, render_markdown: bool) -> Optional[str]:
         """Renders the message as HTML.
 
         Args:
@@ -194,6 +178,7 @@ class CommandHandler:
         name: The name of this command.
         help_section: Section of the help in which this command will appear.
     """
+
     def __init__(self, handler: Callable[[CommandEvent], Awaitable[Dict]], needs_auth: bool,
                  needs_puppeting: bool, needs_matrix_puppeting: bool, needs_admin: bool,
                  management_only: bool, name: str, help_text: str, help_args: str,
@@ -254,7 +239,7 @@ class CommandHandler:
         Args:
             is_management: If the room in which the command will be issued is a
                 management room.
-            puppet_whitelited: If the connected Telegram account puppet is
+            puppet_whitelisted: If the connected Telegram account puppet is
                 allowed to issue the command.
             matrix_puppet_whitelisted: If the connected Matrix account puppet is
                 allowed to issue the command.
@@ -300,18 +285,12 @@ class CommandHandler:
 
 
 def command_handler(_func: Optional[Callable[[CommandEvent], Awaitable[Dict]]] = None, *,
-                    needs_auth: bool = True,
-                    needs_puppeting: bool = True,
-                    needs_matrix_puppeting: bool = False,
-                    needs_admin: bool = False,
-                    management_only: bool = False,
-                    name: Optional[str] = None,
-                    help_text: str = "",
-                    help_args: str = "",
-                    help_section: HelpSection = None
+                    needs_auth: bool = True, needs_puppeting: bool = True,
+                    needs_matrix_puppeting: bool = False, needs_admin: bool = False,
+                    management_only: bool = False, name: Optional[str] = None,
+                    help_text: str = "", help_args: str = "", help_section: HelpSection = None
                     ) -> Callable[[Callable[[CommandEvent], Awaitable[Optional[Dict]]]],
                                   CommandHandler]:
-
     def decorator(func: Callable[[CommandEvent], Awaitable[Optional[Dict]]]) -> CommandHandler:
         actual_name = name or func.__name__.replace("_", "-")
         handler = CommandHandler(func, needs_auth, needs_puppeting, needs_matrix_puppeting,

mautrix_telegram/commands/matrix_auth.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by

mautrix_telegram/commands/meta.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -65,8 +65,8 @@ def _get_management_status(evt: CommandEvent) -> str:
     return "**This is not a management room**: you must prefix commands with `$cmdprefix`."
 
 
-@command_handler(needs_auth=False, needs_puppeting=False,
+@command_handler(name="help", needs_auth=False, needs_puppeting=False,
                  help_section=SECTION_GENERAL,
                  help_text="Show this help message.")
-async def help(evt: CommandEvent) -> Optional[Dict]:
+async def help_cmd(evt: CommandEvent) -> Optional[Dict]:
     return await evt.reply(_get_management_status(evt) + "\n" + await _get_help_text(evt))

mautrix_telegram/commands/portal/admin.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by

mautrix_telegram/commands/portal/bridge.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -36,6 +36,10 @@ async def bridge(evt: CommandEvent) -> Dict:
     if len(evt.args) == 0:
         return await evt.reply("**Usage:** "
                                "`$cmdprefix+sp bridge <Telegram chat ID> [Matrix room ID]`")
+    force_use_bot = False
+    if evt.args[0] == "--usebot" and evt.sender.is_admin:
+        force_use_bot = True
+        evt.args = evt.args[1:]
     room_id = MatrixRoomID(evt.args[1]) if len(evt.args) > 1 else evt.room_id
     that_this = "This" if room_id == evt.room_id else "That"
 
@@ -80,6 +84,7 @@ async def bridge(evt: CommandEvent) -> Dict:
             "bridge_to_mxid": room_id,
             "tgid": portal.tgid,
             "peer_type": portal.peer_type,
+            "force_use_bot": force_use_bot,
         }
         return await evt.reply(f"{has_portal_message}"
                                "However, you have the permissions to unbridge that room.\n\n"
@@ -93,6 +98,7 @@ async def bridge(evt: CommandEvent) -> Dict:
         "bridge_to_mxid": room_id,
         "tgid": portal.tgid,
         "peer_type": portal.peer_type,
+        "force_use_bot": force_use_bot,
     }
     return await evt.reply("That Telegram chat has no existing portal. To confirm bridging the "
                            "chat to this room, use `$cmdprefix+sp continue`")
@@ -149,7 +155,7 @@ async def confirm_bridge(evt: CommandEvent) -> Optional[Dict]:
                                "`$cmdprefix+sp cancel` to cancel.")
 
     evt.sender.command_status = None
-    is_logged_in = await evt.sender.is_logged_in()
+    is_logged_in = await evt.sender.is_logged_in() and not status["force_use_bot"]
     user = evt.sender if is_logged_in else evt.tgbot
     try:
         entity = await user.client.get_entity(portal.peer)

mautrix_telegram/commands/portal/config.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -18,9 +18,10 @@ from typing import Dict, Awaitable
 from io import StringIO
 
 from ...config import yaml
-from ... import portal as po, user as u, util
+from ... import portal as po, util
 from .. import command_handler, CommandEvent, SECTION_PORTAL_MANAGEMENT
 
+
 @command_handler(help_section=SECTION_PORTAL_MANAGEMENT,
                  help_text="View or change per-portal settings.",
                  help_args="<`help`|_subcommand_> [...]")
@@ -76,7 +77,6 @@ def config_view(evt: CommandEvent, portal: po.Portal) -> Awaitable[Dict]:
 def config_defaults(evt: CommandEvent) -> Awaitable[Dict]:
     stream = StringIO()
     yaml.dump({
-        "edits_as_replies": evt.config["bridge.edits_as_replies"],
         "bridge_notices": {
             "default": evt.config["bridge.bridge_notices.default"],
             "exceptions": evt.config["bridge.bridge_notices.exceptions"],

mautrix_telegram/commands/portal/create_chat.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -17,6 +17,7 @@
 from typing import Dict
 
 from ... import portal as po
+from ...types import TelegramID
 from .. import command_handler, CommandEvent, SECTION_CREATING_PORTALS
 from .util import user_has_power_level, get_initial_state
 
@@ -50,7 +51,8 @@ async def create(evt: CommandEvent) -> Dict:
         "group": "chat",
     }[type]
 
-    portal = po.Portal(tgid=None, mxid=evt.room_id, title=title, about=about, peer_type=type)
+    portal = po.Portal(tgid=TelegramID(0), peer_type=type,
+                       mxid=evt.room_id, title=title, about=about)
     try:
         await portal.create_telegram_chat(evt.sender, supergroup=supergroup)
     except ValueError as e:

mautrix_telegram/commands/portal/filter.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -46,11 +46,11 @@ async def filter_mode(evt: CommandEvent) -> Dict:
                                "`!filter blacklist <chat ID>`.")
 
 
-@command_handler(needs_admin=True,
+@command_handler(name="filter", needs_admin=True,
                  help_section=SECTION_ADMIN,
                  help_args="<`whitelist`|`blacklist`> <_chat ID_>",
                  help_text="Allow or disallow bridging a specific chat.")
-async def filter(evt: CommandEvent) -> Optional[Dict]:
+async def edit_filter(evt: CommandEvent) -> Optional[Dict]:
     try:
         action = evt.args[0]
         if action not in ("whitelist", "blacklist", "add", "remove"):
@@ -58,11 +58,11 @@ async def filter(evt: CommandEvent) -> Optional[Dict]:
 
         id_str = evt.args[1]
         if id_str.startswith("-100"):
-            id = int(id_str[4:])
+            filter_id = int(id_str[4:])
         elif id_str.startswith("-"):
-            id = int(id_str[1:])
+            filter_id = int(id_str[1:])
         else:
-            id = int(id_str)
+            filter_id = int(id_str)
     except (IndexError, ValueError):
         return await evt.reply("**Usage:** `$cmdprefix+sp filter <whitelist/blacklist> <chat ID>`")
 
@@ -70,26 +70,26 @@ async def filter(evt: CommandEvent) -> Optional[Dict]:
     if mode not in ("blacklist", "whitelist"):
         return await evt.reply(f"Unknown filter mode \"{mode}\". Please fix the bridge config.")
 
-    list = evt.config["bridge.filter.list"]
+    filter_id_list = evt.config["bridge.filter.list"]
 
     if action in ("blacklist", "whitelist"):
         action = "add" if mode == action else "remove"
 
     def save() -> None:
-        evt.config["bridge.filter.list"] = list
+        evt.config["bridge.filter.list"] = filter_id_list
         evt.config.save()
-        po.Portal.filter_list = list
+        po.Portal.filter_list = filter_id_list
 
     if action == "add":
-        if id in list:
+        if filter_id in filter_id_list:
             return await evt.reply(f"That chat is already {mode}ed.")
-        list.append(id)
+        filter_id_list.append(filter_id)
         save()
         return await evt.reply(f"Chat ID added to {mode}.")
     elif action == "remove":
-        if id not in list:
+        if filter_id not in filter_id_list:
             return await evt.reply(f"That chat is not {mode}ed.")
-        list.remove(id)
+        filter_id_list.remove(filter_id)
         save()
         return await evt.reply(f"Chat ID removed from {mode}.")
     return None

mautrix_telegram/commands/portal/misc.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -38,10 +38,10 @@ async def sync_state(evt: CommandEvent) -> Dict:
     await evt.reply("Synchronization complete")
 
 
-@command_handler(needs_admin=False, needs_puppeting=False, needs_auth=False,
+@command_handler(name="id", needs_admin=False, needs_puppeting=False, needs_auth=False,
                  help_section=SECTION_MISC,
                  help_text="Get the ID of the Telegram chat where this room is bridged.")
-async def id(evt: CommandEvent) -> Dict:
+async def get_id(evt: CommandEvent) -> Dict:
     portal = po.Portal.get_by_mxid(evt.room_id)
     if not portal:
         return await evt.reply("This is not a portal room.")

mautrix_telegram/commands/portal/unbridge.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by

mautrix_telegram/commands/portal/util.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by

mautrix_telegram/commands/telegram/account.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -17,10 +17,10 @@
 from typing import Dict, Optional
 
 from telethon.errors import (UsernameInvalidError, UsernameNotModifiedError, UsernameOccupiedError,
-                             HashInvalidError, AuthKeyError)
+                             HashInvalidError, AuthKeyError, FirstNameInvalidError)
 from telethon.tl.types import Authorization
 from telethon.tl.functions.account import (UpdateUsernameRequest, GetAuthorizationsRequest,
-                                           ResetAuthorizationRequest)
+                                           ResetAuthorizationRequest, UpdateProfileRequest)
 
 from .. import command_handler, CommandEvent, SECTION_AUTH
 
@@ -53,6 +53,25 @@ async def username(evt: CommandEvent) -> Optional[Dict]:
         await evt.reply(f"Username changed to {evt.sender.username}")
 
 
+@command_handler(needs_auth=True, help_section=SECTION_AUTH, help_args="<_new displayname_>",
+                 help_text="Change your Telegram displayname.")
+async def displayname(evt: CommandEvent) -> Optional[Dict]:
+    if len(evt.args) == 0:
+        return await evt.reply("**Usage:** `$cmdprefix+sp displayname <new displayname>`")
+    if evt.sender.is_bot:
+        return await evt.reply("Bots can't set their own displayname.")
+
+    first_name, last_name = ((evt.args[0], "")
+                             if len(evt.args) == 1
+                             else (" ".join(evt.args[:-1]), evt.args[-1]))
+    try:
+        await evt.sender.client(UpdateProfileRequest(first_name=first_name, last_name=last_name))
+    except FirstNameInvalidError:
+        return await evt.reply("Invalid first name")
+    await evt.sender.update_info()
+    await evt.reply("Displayname updated")
+
+
 def _format_session(sess: Authorization) -> str:
     return (f"**{sess.app_name} {sess.app_version}**  \n"
             f"  **Platform:** {sess.device_model} {sess.platform} {sess.system_version}  \n"
@@ -87,9 +106,7 @@ async def session(evt: CommandEvent) -> Optional[Dict]:
         try:
             session_hash = int(evt.args[1])
         except ValueError:
-            return await evt.reply("Hash must be a positive integer")
-        if session_hash <= 0:
-            return await evt.reply("Hash must be a positive integer")
+            return await evt.reply("Hash must be an integer")
         try:
             ok = await evt.sender.client(ResetAuthorizationRequest(hash=session_hash))
         except HashInvalidError:

mautrix_telegram/commands/telegram/auth.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -33,8 +33,8 @@ from ...util import format_duration, ignore_coro
                  help_text="Check if you're logged into Telegram.")
 async def ping(evt: CommandEvent) -> Optional[Dict]:
     me = await evt.sender.client.get_me() if await evt.sender.is_logged_in() else None
-    human_tg_id = f"@{me.username}" if me.username else f"+{me.phone}"
     if me:
+        human_tg_id = f"@{me.username}" if me.username else f"+{me.phone}"
         return await evt.reply(f"You're logged in as {human_tg_id}")
     else:
         return await evt.reply("You're not logged in.")
@@ -46,11 +46,9 @@ async def ping(evt: CommandEvent) -> Optional[Dict]:
 async def ping_bot(evt: CommandEvent) -> Optional[Dict]:
     if not evt.tgbot:
         return await evt.reply("Telegram message relay bot not configured.")
-    bot_info = await evt.tgbot.client.get_me()
-    mxid = pu.Puppet.get_mxid_from_id(bot_info.id)
-    displayname = bot_info.first_name
+    info, mxid = await evt.tgbot.get_me(use_cache=False)
     return await evt.reply("Telegram message relay bot is active: "
-                           f"[{displayname}](https://matrix.to/#/{mxid}) (ID {bot_info.id})\n\n"
+                           f"[{info.first_name}](https://matrix.to/#/{mxid}) (ID {info.id})\n\n"
                            "To use the bot, simply invite it to a portal room.")
 
 
@@ -126,41 +124,20 @@ async def login(evt: CommandEvent) -> Optional[Dict]:
     if evt.config["appservice.public.enabled"]:
         prefix = evt.config["appservice.public.external"]
         url = f"{prefix}/login?token={evt.public_website.make_token(evt.sender.mxid, '/login')}"
-        if allow_matrix_login:
-            if override_sender:
-                return await evt.reply(
-                    "This bridge instance allows you to log in inside or outside of Matrix, but "
-                    "logging in as another user is only possible via the web interface.\n\n"
-                    f"Please visit [the login page]({url}) to log in as "
-                    f"[{evt.sender.mxid}](https://matrix.to/#/{evt.sender.mxid}).\n\n")
-            return await evt.reply(
-                "This bridge instance allows you to log in inside or outside Matrix.\n\n"
-                "If you would like to log in within Matrix, please send your phone number or bot "
-                "auth token here.\n"
-                "If you would like to log in outside of Matrix, please visit [the login page]"
-                f"({url}).\n\n"
-                "Logging in outside of Matrix is recommended if you have two-factor authentication "
-                "enabled, because in-Matrix login would save your password in the message history."
-                f"\n\n{nb}")
         if override_sender:
-            return await evt.reply(
-                "This bridge instance does not allow logging in inside Matrix, and logging in as "
-                "another user inside Matrix isn't possible anyway.\n\n"
-                f"Please visit [the login page]({url}) to log in as "
-                f"[{evt.sender.mxid}](https://matrix.to/#/{evt.sender.mxid}).")
-        return await evt.reply(
-            "This bridge instance does not allow logging in inside Matrix.\n\n"
-            f"Please visit [the login page]({url}) to log in.\n\n"
-            f"{nb}")
+            return await evt.reply(f"[Click here to log in]({url}) as "
+                                   f"[{evt.sender.mxid}](https://matrix.to/#/{evt.sender.mxid}).")
+        elif allow_matrix_login:
+            return await evt.reply(f"[Click here to log in]({url}). Alternatively, send your phone"
+                                   f" number (or bot auth token) here to log in.\n\n{nb}")
+        return await evt.reply(f"[Click here to log in]({url}).\n\n{nb}")
     elif allow_matrix_login:
         if override_sender:
             return await evt.reply(
                 "This bridge instance does not allow you to log in outside of Matrix. "
                 "Logging in as another user inside Matrix is not currently possible.")
-        return await evt.reply(
-            "This bridge instance does not allow you to log in outside of Matrix.\n\n"
-            "Please send your phone number or bot auth token here to start the login process.\n\n"
-            f"{nb}")
+        return await evt.reply("Please send your phone number (or bot auth token) here to start "
+                               f"the login process.\n\n{nb}")
     return await evt.reply("This bridge instance has been configured to not allow logging in.")
 
 

mautrix_telegram/commands/telegram/misc.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -191,7 +191,7 @@ async def _parse_encoded_msgid(user: AbstractUser, enc_id: str, type_name: str
         raise MessageIDError(f"Invalid {type_name} ID (format)") from e
 
     if peer_type == PEER_TYPE_CHAT:
-        orig_msg = DBMessage.get_by_tgid(msg_id, space)
+        orig_msg = DBMessage.get_one_by_tgid(msg_id, space)
         if not orig_msg:
             raise MessageIDError(f"Invalid {type_name} ID (original message not found in db)")
         new_msg = DBMessage.get_by_mxid(orig_msg.mxid, orig_msg.mx_room, user.tgid)
@@ -237,11 +237,11 @@ async def play(evt: CommandEvent) -> Optional[Dict]:
 
 
 @command_handler(help_section=SECTION_MISC,
-                 help_args="<_poll ID_> <_choice ID_>",
+                 help_args="<_poll ID_> <_choice number_>",
                  help_text="Vote in a Telegram poll.")
 async def vote(evt: CommandEvent) -> Optional[Dict]:
-    if len(evt.args) < 2:
-        return await evt.reply("**Usage:** `$cmdprefix+sp vote <poll ID> <choice ID>`")
+    if len(evt.args) < 1:
+        return await evt.reply("**Usage:** `$cmdprefix+sp vote <poll ID> <choice number>`")
     elif not await evt.sender.is_logged_in():
         return await evt.reply("You must be logged in with a real account to vote in polls.")
     elif evt.sender.is_bot:
@@ -255,7 +255,25 @@ async def vote(evt: CommandEvent) -> Optional[Dict]:
     if not isinstance(msg.media, MessageMediaPoll):
         return await evt.reply("Invalid poll ID (message doesn't look like a poll)")
 
-    options = [base64.b64decode(option + (3 - (len(option) + 3) % 4) * "=")
+    options = []
+    for option in evt.args[1:]:
+        try:
+            if len(option) > 10:
+                raise ValueError("option index too long")
+            option_index = int(option) - 1
+        except ValueError:
+            option_index = None
+        if option_index is None:
+            return await evt.reply(f"Invalid option number \"{option}\"",
+                                   render_markdown=False, allow_html=False)
+        elif option_index < 0:
+            return await evt.reply(f"Invalid option number {option}. "
+                                   f"Option numbers must be positive.")
+        elif option_index >= len(msg.media.poll.answers):
+            return await evt.reply(f"Invalid option number {option}. "
+                                   f"The poll only has {len(msg.media.poll.answers)} options.")
+        options.append(msg.media.poll.answers[option_index].option)
+    options = [msg.media.poll.answers[int(option) - 1].option
                for option in evt.args[1:]]
     try:
         resp = await evt.sender.client(SendVoteRequest(peer=peer, msg_id=msg.id, options=options))

mautrix_telegram/config.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -103,12 +103,20 @@ class DictWithRecursion:
 
 
 class Config(DictWithRecursion):
-    def __init__(self, path: str, registration_path: str, base_path: str) -> None:
+    def __init__(self, path: str, registration_path: str, base_path: str,
+                 overrides: Dict[str, Any] = None) -> None:
         super().__init__()
         self.path = path  # type: str
         self.registration_path = registration_path  # type: str
         self.base_path = base_path  # type: str
         self._registration = None  # type: Optional[Dict]
+        self._overrides = overrides or {}  # type: Dict[str, Any]
+
+    def __getitem__(self, key: str) -> Any:
+        try:
+            return self._overrides[f"MAUTRIX_TELEGRAM_{key.replace('.', '_').upper()}"]
+        except KeyError:
+            return super().__getitem__(key)
 
     def load(self) -> None:
         with open(self.path, 'r') as stream:
@@ -181,9 +189,14 @@ class Config(DictWithRecursion):
         copy("appservice.bot_displayname")
         copy("appservice.bot_avatar")
 
+        copy("appservice.community_id")
+
         copy("appservice.as_token")
         copy("appservice.hs_token")
 
+        copy("metrics.enabled")
+        copy("metrics.listen_port")
+
         copy("bridge.username_template")
         copy("bridge.alias_template")
         copy("bridge.displayname_template")
@@ -199,14 +212,13 @@ class Config(DictWithRecursion):
         copy("bridge.sync_matrix_state")
         copy("bridge.allow_matrix_login")
         copy("bridge.plaintext_highlights")
-        copy("bridge.edits_as_replies")
-        copy("bridge.highlight_edits")
         copy("bridge.public_portals")
         copy("bridge.catch_up")
         copy("bridge.sync_with_custom_puppets")
         copy("bridge.telegram_link_preview")
         copy("bridge.inline_images")
         copy("bridge.image_as_file_size")
+        copy("bridge.max_document_size")
 
         copy("bridge.bot_messages_as_notices")
         if isinstance(self["bridge.bridge_notices"], bool):
@@ -257,10 +269,24 @@ class Config(DictWithRecursion):
         copy("telegram.api_id")
         copy("telegram.api_hash")
         copy("telegram.bot_token")
+
+        copy("telegram.connection.timeout")
+        copy("telegram.connection.retries")
+        copy("telegram.connection.retry_delay")
+        copy("telegram.connection.flood_sleep_threshold")
+        copy("telegram.connection.request_retries")
+
+        copy("telegram.device_info.device_model")
+        copy("telegram.device_info.system_version")
+        copy("telegram.device_info.app_version")
+        copy("telegram.device_info.lang_code")
+        copy("telegram.device_info.system_lang_code")
+
         copy("telegram.server.enabled")
         copy("telegram.server.dc")
         copy("telegram.server.ip")
         copy("telegram.server.port")
+
         copy("telegram.proxy.type")
         copy("telegram.proxy.address")
         copy("telegram.proxy.port")
@@ -328,3 +354,6 @@ class Config(DictWithRecursion):
             "sender_localpart": self["appservice.bot_username"],
             "rate_limited": False
         }
+        if self["appservice.community_id"]:
+            self._registration["namespaces"]["users"][0]["group_id"] \
+                = self["appservice.community_id"]

mautrix_telegram/context.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by

mautrix_telegram/db/__init__.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by

mautrix_telegram/db/base.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -55,5 +55,4 @@ class BaseBase:
         with self.db.begin() as conn:
             conn.execute(self.t.delete().where(self._edit_identity))
 
-
 Base = declarative_base(cls=BaseBase)

mautrix_telegram/db/bot_chat.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -29,16 +29,16 @@ class BotChat(Base):
     type = Column(String, nullable=False)
 
     @classmethod
-    def delete(cls, id: TelegramID) -> None:
+    def delete(cls, chat_id: TelegramID) -> None:
         with cls.db.begin() as conn:
-            conn.execute(cls.t.delete().where(cls.c.id == id))
+            conn.execute(cls.t.delete().where(cls.c.id == chat_id))
 
     @classmethod
     def all(cls) -> Iterable['BotChat']:
         rows = cls.db.execute(cls.t.select())
         for row in rows:
-            id, type = row
-            yield cls(id=id, type=type)
+            chat_id, chat_type = row
+            yield cls(id=chat_id, type=chat_type)
 
     def insert(self) -> None:
         with self.db.begin() as conn:

mautrix_telegram/db/message.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -14,7 +14,7 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from sqlalchemy import Column, UniqueConstraint, Integer, String, and_, func, select
+from sqlalchemy import Column, UniqueConstraint, Integer, String, and_, func, desc, select
 from sqlalchemy.engine.result import RowProxy
 from typing import Optional, List
 
@@ -29,25 +29,44 @@ class Message(Base):
     mx_room = Column(String)  # type: MatrixRoomID
     tgid = Column(Integer, primary_key=True)  # type: TelegramID
     tg_space = Column(Integer, primary_key=True)  # type: TelegramID
+    edit_index = Column(Integer, primary_key=True)  # type: int
 
     __table_args__ = (UniqueConstraint("mxid", "mx_room", "tg_space", name="_mx_id_room"),)
 
     @classmethod
     def _one_or_none(cls, rows: RowProxy) -> Optional['Message']:
         try:
-            mxid, mx_room, tgid, tg_space = next(rows)
-            return cls(mxid=mxid, mx_room=mx_room, tgid=tgid, tg_space=tg_space)
+            mxid, mx_room, tgid, tg_space, edit_index = next(rows)
+            return cls(mxid=mxid, mx_room=mx_room, tgid=tgid, tg_space=tg_space,
+                       edit_index=edit_index)
         except StopIteration:
             return None
 
     @staticmethod
     def _all(rows: RowProxy) -> List['Message']:
-        return [Message(mxid=row[0], mx_room=row[1], tgid=row[2], tg_space=row[3])
+        return [Message(mxid=row[0], mx_room=row[1], tgid=row[2], tg_space=row[3],
+                        edit_index=row[4])
                 for row in rows]
 
     @classmethod
-    def get_by_tgid(cls, tgid: TelegramID, tg_space: TelegramID) -> Optional['Message']:
-        return cls._select_one_or_none(and_(cls.c.tgid == tgid, cls.c.tg_space == tg_space))
+    def get_all_by_tgid(cls, tgid: TelegramID, tg_space: TelegramID) -> List['Message']:
+        return cls._all(cls.db.execute(cls.t.select().where(and_(cls.c.tgid == tgid,
+                                                                 cls.c.tg_space == tg_space))))
+
+    @classmethod
+    def get_one_by_tgid(cls, tgid: TelegramID, tg_space: TelegramID, edit_index: int = 0
+                        ) -> Optional['Message']:
+        query = cls.t.select()
+        if edit_index < 0:
+            query = (query
+                     .where(and_(cls.c.tgid == tgid, cls.c.tg_space == tg_space))
+                     .order_by(desc(cls.c.edit_index))
+                     .limit(1)
+                     .offset(-edit_index - 1))
+        else:
+            query = query.where(and_(cls.c.tgid == tgid, cls.c.tg_space == tg_space,
+                                     cls.c.edit_index == edit_index))
+        return cls._one_or_none(cls.db.execute(query))
 
     @classmethod
     def count_spaces_by_mxid(cls, mxid: MatrixEventID, mx_room: MatrixRoomID) -> int:
@@ -67,10 +86,12 @@ class Message(Base):
                                             cls.c.tg_space == tg_space))
 
     @classmethod
-    def update_by_tgid(cls, s_tgid: TelegramID, s_tg_space: TelegramID, **values) -> None:
+    def update_by_tgid(cls, s_tgid: TelegramID, s_tg_space: TelegramID, s_edit_index: int,
+                       **values) -> None:
         with cls.db.begin() as conn:
             conn.execute(cls.t.update()
-                         .where(and_(cls.c.tgid == s_tgid, cls.c.tg_space == s_tg_space))
+                         .where(and_(cls.c.tgid == s_tgid, cls.c.tg_space == s_tg_space,
+                                     cls.c.edit_index == s_edit_index))
                          .values(**values))
 
     @classmethod
@@ -82,9 +103,11 @@ class Message(Base):
 
     @property
     def _edit_identity(self):
-        return and_(self.c.tgid == self.tgid, self.c.tg_space == self.tg_space)
+        return and_(self.c.tgid == self.tgid, self.c.tg_space == self.tg_space,
+                    self.c.edit_index == self.edit_index)
 
     def insert(self) -> None:
         with self.db.begin() as conn:
             conn.execute(self.t.insert().values(mxid=self.mxid, mx_room=self.mx_room,
-                                                tgid=self.tgid, tg_space=self.tg_space))
+                                                tgid=self.tgid, tg_space=self.tg_space,
+                                                edit_index=self.edit_index))

mautrix_telegram/db/portal.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by

mautrix_telegram/db/puppet.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -35,15 +35,16 @@ class Puppet(Base):
     photo_id = Column(String, nullable=True)
     is_bot = Column(Boolean, nullable=True)
     matrix_registered = Column(Boolean, nullable=False, server_default=expression.false())
+    disable_updates = Column(Boolean, nullable=False, server_default=expression.false())
 
     @classmethod
     def scan(cls, row) -> Optional['Puppet']:
         (id, custom_mxid, access_token, displayname, displayname_source, username, photo_id,
-         is_bot, matrix_registered) = row
+         is_bot, matrix_registered, disable_updates) = row
         return cls(id=id, custom_mxid=custom_mxid, access_token=access_token,
                    displayname=displayname, displayname_source=displayname_source,
                    username=username, photo_id=photo_id, is_bot=is_bot,
-                   matrix_registered=matrix_registered)
+                   matrix_registered=matrix_registered, disable_updates=disable_updates)
 
     @classmethod
     def _one_or_none(cls, rows: RowProxy) -> Optional['Puppet']:
@@ -84,4 +85,4 @@ class Puppet(Base):
                 id=self.id, custom_mxid=self.custom_mxid, access_token=self.access_token,
                 displayname=self.displayname, displayname_source=self.displayname_source,
                 username=self.username, photo_id=self.photo_id, is_bot=self.is_bot,
-                matrix_registered=self.matrix_registered))
+                matrix_registered=self.matrix_registered, disable_updates=self.disable_updates))

mautrix_telegram/db/room_state.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by

mautrix_telegram/db/telegram_file.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -35,14 +35,14 @@ class TelegramFile(Base):
     thumbnail = None  # type: Optional[TelegramFile]
 
     @classmethod
-    def get(cls, id: str) -> Optional['TelegramFile']:
-        rows = cls.db.execute(cls.t.select().where(cls.c.id == id))
+    def get(cls, loc_id: str) -> Optional['TelegramFile']:
+        rows = cls.db.execute(cls.t.select().where(cls.c.id == loc_id))
         try:
-            id, mxc, mime, conv, ts, s, w, h, thumb_id = next(rows)
+            loc_id, mxc, mime, conv, ts, s, w, h, thumb_id = next(rows)
             thumb = None
             if thumb_id:
                 thumb = cls.get(thumb_id)
-            return cls(id=id, mxc=mxc, mime_type=mime, was_converted=conv, timestamp=ts,
+            return cls(id=loc_id, mxc=mxc, mime_type=mime, was_converted=conv, timestamp=ts,
                        size=s, width=w, height=h, thumbnail_id=thumb_id, thumbnail=thumb)
         except StopIteration:
             return None

mautrix_telegram/db/user.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -106,8 +106,8 @@ class User(Base):
 
     def delete(self) -> None:
         super().delete()
-        self.portals = None
-        self.contacts = None
+        self.portals = []
+        self.contacts = []
 
 
 class UserPortal(Base):

mautrix_telegram/db/user_profile.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by

mautrix_telegram/formatter/__init__.py

@@ -1,9 +1,8 @@
 from .from_matrix import (matrix_reply_to_telegram, matrix_to_telegram, matrix_text_to_telegram,
                           init_mx)
-from .from_telegram import (telegram_reply_to_matrix, telegram_to_matrix, init_tg)
+from .from_telegram import telegram_reply_to_matrix, telegram_to_matrix
 from .. import context as c
 
 
 def init(context: c.Context) -> None:
     init_mx(context)
-    init_tg(context)

mautrix_telegram/formatter/from_matrix/__init__.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -36,7 +36,7 @@ should_bridge_plaintext_highlights = False  # type: bool
 
 command_regex = re.compile(r"^!([A-Za-z0-9@]+)")  # type: Pattern
 not_command_regex = re.compile(r"^\\(![A-Za-z0-9@]+)")  # type: Pattern
-plain_mention_regex = None  # type: Pattern
+plain_mention_regex = None  # type: Optional[Pattern]
 
 
 def plain_mention_to_html(match: Match) -> str:
@@ -87,25 +87,28 @@ def matrix_to_telegram(html: str) -> ParsedMessage:
 
 def matrix_reply_to_telegram(content: Dict[str, Any], tg_space: TelegramID,
                              room_id: Optional[MatrixRoomID] = None) -> Optional[TelegramID]:
+    relates_to = content.get("m.relates_to", None) or {}
+    if not relates_to:
+        return None
+    reply = (relates_to if relates_to.get("rel_type", None) == "m.reference"
+             else relates_to.get("m.in_reply_to", None) or {})
+    if not reply:
+        return None
+    room_id = room_id or reply.get("room_id", None)
+    event_id = reply.get("event_id", None)
+    if not event_id:
+        return
+
     try:
-        reply = content.get("m.relates_to", {}).get("m.in_reply_to", {})
-        if not reply:
-            return None
-        room_id = room_id or reply["room_id"]
-        event_id = reply["event_id"]
-
-        try:
-            if content["format"] == "org.matrix.custom.html":
-                content["formatted_body"] = trim_reply_fallback_html(content["formatted_body"])
-        except KeyError:
-            pass
-        content["body"] = trim_reply_fallback_text(content["body"])
-
-        message = DBMessage.get_by_mxid(event_id, room_id, tg_space)
-        if message:
-            return message.tgid
+        if content["format"] == "org.matrix.custom.html":
+            content["formatted_body"] = trim_reply_fallback_html(content["formatted_body"])
     except KeyError:
         pass
+    content["body"] = trim_reply_fallback_text(content["body"])
+
+    message = DBMessage.get_by_mxid(event_id, room_id, tg_space)
+    if message:
+        return message.tgid
     return None
 
 
@@ -147,5 +150,5 @@ def init_mx(context: "Context") -> None:
     config = context.config
     dn_template = config.get("bridge.displayname_template", "{displayname} (Telegram)")
     dn_template = re.escape(dn_template).replace(re.escape("{displayname}"), "[^>]+")
-    plain_mention_regex = re.compile(f"(\s|^)({dn_template})")
+    plain_mention_regex = re.compile(f"^({dn_template})")
     should_bridge_plaintext_highlights = config["bridge.plaintext_highlights"] or False

mautrix_telegram/formatter/from_matrix/html_reader.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -29,6 +29,10 @@ class HTMLNode(list):
 
 
 class NodeifyingParser(HTMLParser):
+    # From https://www.w3.org/TR/html5/syntax.html#writing-html-documents-elements
+    void_tags = ("area", "base", "br", "col", "command", "embed", "hr", "img", "input", "link",
+                 "meta", "param", "source", "track", "wbr")
+
     def __init__(self):
         super().__init__()
         self.stack = [HTMLNode("html", [])]  # type: List[HTMLNode]
@@ -36,7 +40,11 @@ class NodeifyingParser(HTMLParser):
     def handle_starttag(self, tag, attrs):
         node = HTMLNode(tag, attrs)
         self.stack[-1].append(node)
-        self.stack.append(node)
+        if tag not in self.void_tags:
+            self.stack.append(node)
+
+    def handle_startendtag(self, tag, attrs):
+        self.stack[-1].append(HTMLNode(tag, attrs))
 
     def handle_endtag(self, tag):
         if tag == self.stack[-1].tag:

mautrix_telegram/formatter/from_matrix/parser.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -18,15 +18,15 @@ from typing import List, Tuple, Pattern
 import re
 
 from telethon.tl.types import (MessageEntityMention as Mention, MessageEntityBotCommand as Command,
-                               MessageEntityMentionName as MentionName, MessageEntityEmail as Email,
-                               MessageEntityUrl as URL, MessageEntityTextUrl as TextURL,
+                               MessageEntityMentionName as MentionName, MessageEntityUrl as URL,
+                               MessageEntityEmail as Email, MessageEntityTextUrl as TextURL,
                                MessageEntityBold as Bold, MessageEntityItalic as Italic,
                                MessageEntityCode as Code, MessageEntityPre as Pre,
-                               TypeMessageEntity)
+                               MessageEntityStrike as Strike, MessageEntityUnderline as Underline,
+                               MessageEntityBlockquote as Blockquote, TypeMessageEntity)
 
 from ... import user as u, puppet as pu, portal as po
 from ...types import MatrixUserID
-from ..util import html_to_unicode
 from .telegram_message import TelegramMessage, Entity, offset_length_multiply
 
 from .html_reader import HTMLNode, read_html
@@ -101,13 +101,6 @@ class MatrixParser:
             children.append(child)
         return TelegramMessage.join(children, "\n")
 
-    @classmethod
-    def blockquote_to_tmessage(cls, node: HTMLNode, ctx: RecursionContext) -> TelegramMessage:
-        msg = cls.tag_aware_parse_node(node, ctx)
-        children = msg.trim().split("\n")
-        children = [child.prepend("> ") for child in children]
-        return TelegramMessage.join(children, "\n")
-
     @classmethod
     def header_to_tmessage(cls, node: HTMLNode, ctx: RecursionContext) -> TelegramMessage:
         children = cls.node_to_tmessages(node, ctx)
@@ -122,15 +115,14 @@ class MatrixParser:
             msg.format(Bold)
         elif node.tag in ("i", "em"):
             msg.format(Italic)
+        elif node.tag in ("s", "strike", "del"):
+            msg.format(Strike)
+        elif node.tag in ("u", "ins"):
+            msg.format(Underline)
+        elif node == "blockquote":
+            msg.format(Blockquote)
         elif node.tag == "command":
             msg.format(Command)
-        elif node.tag in ("s", "strike", "del"):
-            msg.text = html_to_unicode(msg.text, "\u0336")
-        elif node.tag in ("u", "ins"):
-            msg.text = html_to_unicode(msg.text, "\u0332")
-
-        if node.tag in ("s", "strike", "del", "u", "ins"):
-            msg.entities = Entity.adjust(msg.entities, offset_length_multiply(2))
 
         return msg
 
@@ -169,10 +161,17 @@ class MatrixParser:
                 if msg.text == href
                 else msg.format(TextURL, url=href))
 
+    @classmethod
+    def blockquote_to_tmessage(cls, node: HTMLNode, ctx: RecursionContext) -> TelegramMessage:
+        msg = cls.tag_aware_parse_node(node, ctx)
+        children = msg.trim().split("\n")
+        children = [child.prepend("> ") for child in children]
+        return TelegramMessage.join(children, "\n")
+
     @classmethod
     def node_to_tmessage(cls, node: HTMLNode, ctx: RecursionContext) -> TelegramMessage:
-        if node.tag == "blockquote":
-            return cls.blockquote_to_tmessage(node, ctx)
+        if node.tag == "mx-reply":
+            return TelegramMessage("")
         elif node.tag == "ol":
             return cls.list_to_tmessage(node, ctx)
         elif node.tag == "ul":
@@ -183,6 +182,11 @@ class MatrixParser:
             return TelegramMessage("\n")
         elif node.tag in ("b", "strong", "i", "em", "s", "del", "u", "ins", "command"):
             return cls.basic_format_to_tmessage(node, ctx)
+        elif node.tag == "blockquote":
+            # Telegram already has blockquote entities in the protocol schema, but it strips them
+            # server-side and none of the official clients support them.
+            # TODO once Telegram changes that, use the above if block for blockquotes too.
+            return cls.blockquote_to_tmessage(node, ctx)
         elif node.tag == "a":
             return cls.link_to_tstring(node, ctx)
         elif node.tag == "p":

mautrix_telegram/formatter/from_matrix/telegram_message.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by

mautrix_telegram/formatter/from_telegram.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -24,7 +24,8 @@ from telethon.tl.types import (MessageEntityMention, MessageEntityMentionName, M
                                MessageEntityItalic, MessageEntityCode, MessageEntityPre,
                                MessageEntityBotCommand, MessageEntityHashtag, MessageEntityCashtag,
                                MessageEntityPhone, TypeMessageEntity, Message, PeerChannel,
-                               MessageFwdHeader, PeerUser)
+                               MessageEntityBlockquote, MessageEntityStrike, MessageFwdHeader,
+                               MessageEntityUnderline, PeerUser)
 
 from mautrix_appservice import MatrixRequestError
 from mautrix_appservice.intent_api import IntentAPI
@@ -33,20 +34,12 @@ from .. import user as u, puppet as pu, portal as po
 from ..types import TelegramID
 from ..db import Message as DBMessage
 from .util import (add_surrogates, remove_surrogates, trim_reply_fallback_html,
-                   trim_reply_fallback_text, unicode_to_html)
+                   trim_reply_fallback_text)
 
 if TYPE_CHECKING:
     from ..abstract_user import AbstractUser
-    from ..context import Context
-
-try:
-    from lxml.html.diff import htmldiff
-except ImportError:
-    htmldiff = None  # type: ignore
-
 
 log = logging.getLogger("mau.fmt.tg")  # type: logging.Logger
-should_highlight_edits = False  # type: bool
 
 
 def telegram_reply_to_matrix(evt: Message, source: 'AbstractUser') -> Dict:
@@ -54,13 +47,16 @@ def telegram_reply_to_matrix(evt: Message, source: 'AbstractUser') -> Dict:
         space = (evt.to_id.channel_id
                  if isinstance(evt, Message) and isinstance(evt.to_id, PeerChannel)
                  else source.tgid)
-        msg = DBMessage.get_by_tgid(evt.reply_to_msg_id, space)
+        msg = DBMessage.get_one_by_tgid(evt.reply_to_msg_id, space)
         if msg:
             return {
                 "m.in_reply_to": {
                     "event_id": msg.mxid,
                     "room_id": msg.mx_room,
-                }
+                },
+                "rel_type": "m.reference",
+                "event_id": msg.mxid,
+                "room_id": msg.mx_room,
             }
     return {}
 
@@ -71,30 +67,33 @@ async def _add_forward_header(source, text: str, html: Optional[str],
         html = escape(text)
     fwd_from_html, fwd_from_text = None, None
     if fwd_from.from_id:
-        user = u.User.get_by_tgid(fwd_from.from_id)
+        user = u.User.get_by_tgid(TelegramID(fwd_from.from_id))
         if user:
             fwd_from_text = user.displayname or user.mxid
-            fwd_from_html = f"<a href='https://matrix.to/#/{user.mxid}'>{fwd_from_text}</a>"
+            fwd_from_html = (f"<a href='https://matrix.to/#/{user.mxid}'>"
+                             f"{escape(fwd_from_text)}</a>")
 
         if not fwd_from_text:
             puppet = pu.Puppet.get(TelegramID(fwd_from.from_id), create=False)
             if puppet and puppet.displayname:
                 fwd_from_text = puppet.displayname or puppet.mxid
-                fwd_from_html = f"<a href='https://matrix.to/#/{puppet.mxid}'>{fwd_from_text}</a>"
+                fwd_from_html = (f"<a href='https://matrix.to/#/{puppet.mxid}'>"
+                                 f"{escape(fwd_from_text)}</a>")
 
         if not fwd_from_text:
             user = await source.client.get_entity(PeerUser(fwd_from.from_id))
             if user:
                 fwd_from_text = pu.Puppet.get_displayname(user, False)
-                fwd_from_html = f"<b>{fwd_from_text}</b>"
+                fwd_from_html = f"<b>{escape(fwd_from_text)}</b>"
     else:
         portal = po.Portal.get_by_tgid(TelegramID(fwd_from.channel_id))
         if portal:
             fwd_from_text = portal.title
             if portal.alias:
-                fwd_from_html = f"<a href='https://matrix.to/#/{portal.alias}'>{fwd_from_text}</a>"
+                fwd_from_html = (f"<a href='https://matrix.to/#/{portal.alias}'>"
+                                 f"{escape(fwd_from_text)}</a>")
             else:
-                fwd_from_html = f"<b>{fwd_from_text}</b>"
+                fwd_from_html = f"<b>{escape(fwd_from_text)}</b>"
         else:
             channel = await source.client.get_entity(PeerChannel(fwd_from.channel_id))
             if channel:
@@ -115,32 +114,19 @@ async def _add_forward_header(source, text: str, html: Optional[str],
     return text, html
 
 
-def highlight_edits(new_html: str, old_html: str) -> str:
-    # Don't include `Edit:` text in diff.
-    if old_html.startswith("<u>Edit:</u> "):
-        old_html = old_html[len("<u>Edit:</u> "):]
-
-    # Generate diff with lxml
-    new_html = htmldiff(old_html, new_html)
-
-    # Replace <ins> with <u> since Riot doesn't allow <ins>
-    new_html = new_html.replace("<ins>", "<u>").replace("</ins>", "</u>")
-    # Remove <del>s since we just want to hide deletions.
-    new_html = re.sub("<del>.+?</del>", "", new_html)
-    return new_html
-
-
 async def _add_reply_header(source: "AbstractUser", text: str, html: str, evt: Message,
-                            relates_to: Dict, main_intent: IntentAPI, is_edit: bool
-                            ) -> Tuple[str, str]:
+                            relates_to: Dict, main_intent: IntentAPI) -> Tuple[str, str]:
     space = (evt.to_id.channel_id
              if isinstance(evt, Message) and isinstance(evt.to_id, PeerChannel)
              else source.tgid)
 
-    msg = DBMessage.get_by_tgid(evt.reply_to_msg_id, space)
+    msg = DBMessage.get_one_by_tgid(evt.reply_to_msg_id, space)
     if not msg:
         return text, html
 
+    relates_to["rel_type"] = "m.reference"
+    relates_to["event_id"] = msg.mxid
+    relates_to["room_id"] = msg.mx_room
     relates_to["m.in_reply_to"] = {
         "event_id": msg.mxid,
         "room_id": msg.mx_room,
@@ -159,22 +145,14 @@ async def _add_reply_header(source: "AbstractUser", text: str, html: str, evt: M
 
         puppet = pu.Puppet.get_by_mxid(r_sender, create=False)
         r_displayname = puppet.displayname if puppet else r_sender
-        r_sender_link = f"<a href='https://matrix.to/#/{r_sender}'>{r_displayname}</a>"
-
-        if is_edit and should_highlight_edits:
-            html = highlight_edits(html or escape(text), r_html_body)
+        r_sender_link = f"<a href='https://matrix.to/#/{r_sender}'>{escape(r_displayname)}</a>"
     except (ValueError, KeyError, MatrixRequestError):
         r_sender_link = "unknown user"
         r_displayname = "unknown user"
         r_text_body = "Failed to fetch message"
         r_html_body = "<em>Failed to fetch message</em>"
 
-    if is_edit:
-        html = f"<u>Edit:</u> {html or escape(text)}"
-        text = f"Edit: {text}"
-
-    r_keyword = "In reply to" if not is_edit else "Edit to"
-    r_msg_link = f"<a href='https://matrix.to/#/{msg.mx_room}/{msg.mxid}'>{r_keyword}</a>"
+    r_msg_link = f"<a href='https://matrix.to/#/{msg.mx_room}/{msg.mxid}'>In reply to</a>"
     html = (
         f"<mx-reply><blockquote>{r_msg_link} {r_sender_link}\n{r_html_body}</blockquote></mx-reply>"
         + (html or escape(text)))
@@ -191,10 +169,10 @@ async def _add_reply_header(source: "AbstractUser", text: str, html: str, evt: M
 
 async def telegram_to_matrix(evt: Message, source: "AbstractUser",
                              main_intent: Optional[IntentAPI] = None,
-                             is_edit: bool = False, prefix_text: Optional[str] = None,
-                             prefix_html: Optional[str] = None, override_text: str = None,
-                             override_entities: List[TypeMessageEntity] = None
-                             ) -> Tuple[str, str, Dict]:
+                             prefix_text: Optional[str] = None, prefix_html: Optional[str] = None,
+                             override_text: str = None,
+                             override_entities: List[TypeMessageEntity] = None,
+                             no_reply_fallback: bool = False) -> Tuple[str, str, Dict]:
     text = add_surrogates(override_text or evt.message)
     entities = override_entities or evt.entities
     html = _telegram_entities_to_matrix_catch(text, entities) if entities else None
@@ -208,9 +186,8 @@ async def telegram_to_matrix(evt: Message, source: "AbstractUser",
     if evt.fwd_from:
         text, html = await _add_forward_header(source, text, html, evt.fwd_from)
 
-    if evt.reply_to_msg_id:
-        text, html = await _add_reply_header(source, text, html, evt, relates_to, main_intent,
-                                             is_edit)
+    if evt.reply_to_msg_id and not no_reply_fallback:
+        text, html = await _add_reply_header(source, text, html, evt, relates_to, main_intent)
 
     if isinstance(evt, Message) and evt.post and evt.post_author:
         if not html:
@@ -218,9 +195,6 @@ async def telegram_to_matrix(evt: Message, source: "AbstractUser",
         text += f"\n- {evt.post_author}"
         html += f"<br/><i>- <u>{evt.post_author}</u></i>"
 
-    html = unicode_to_html(text, html, "\u0336", "del")
-    html = unicode_to_html(text, html, "\u0332", "u")
-
     if html:
         html = html.replace("\n", "<br/>")
 
@@ -238,25 +212,39 @@ def _telegram_entities_to_matrix_catch(text: str, entities: List[TypeMessageEnti
     return "[failed conversion in _telegram_entities_to_matrix]"
 
 
-def _telegram_entities_to_matrix(text: str, entities: List[TypeMessageEntity]) -> str:
+def _telegram_entities_to_matrix(text: str, entities: List[TypeMessageEntity],
+                                 offset: int = 0, length: int = None) -> str:
     if not entities:
-        return text
+        return escape(text)
+    if length is None:
+        length = len(text)
     html = []
     last_offset = 0
-    for entity in entities:
-        if entity.offset > last_offset:
-            html.append(escape(text[last_offset:entity.offset]))
-        elif entity.offset < last_offset:
+    for i, entity in enumerate(entities):
+        if entity.offset > offset + length:
+            break
+        relative_offset = entity.offset - offset
+        if relative_offset > last_offset:
+            html.append(escape(text[last_offset:relative_offset]))
+        elif relative_offset < last_offset:
             continue
 
         skip_entity = False
-        entity_text = escape(text[entity.offset:entity.offset + entity.length])
+        entity_text = _telegram_entities_to_matrix(
+            text=text[relative_offset:relative_offset + entity.length],
+            entities=entities[i + 1:], offset=entity.offset, length=entity.length)
         entity_type = type(entity)
 
         if entity_type == MessageEntityBold:
             html.append(f"<strong>{entity_text}</strong>")
         elif entity_type == MessageEntityItalic:
             html.append(f"<em>{entity_text}</em>")
+        elif entity_type == MessageEntityUnderline:
+            html.append(f"<u>{entity_text}</u>")
+        elif entity_type == MessageEntityStrike:
+            html.append(f"<del>{entity_text}</del>")
+        elif entity_type == MessageEntityBlockquote:
+            html.append(f"<blockquote>{entity_text}</blockquote>")
         elif entity_type == MessageEntityCode:
             html.append(f"<pre><code>{entity_text}</code></pre>"
                         if "\n" in entity_text
@@ -278,8 +266,8 @@ def _telegram_entities_to_matrix(text: str, entities: List[TypeMessageEntity]) -
             html.append(f"<font color='blue'>{entity_text}</font>")
         else:
             skip_entity = True
-        last_offset = entity.offset + (0 if skip_entity else entity.length)
-    html.append(text[last_offset:])
+        last_offset = relative_offset + (0 if skip_entity else entity.length)
+    html.append(escape(text[last_offset:]))
 
     return "".join(html)
 
@@ -341,14 +329,9 @@ def _parse_url(html: List[str], entity_text: str, url: str) -> bool:
 
         portal = po.Portal.find_by_username(group)
         if portal:
-            message = DBMessage.get_by_tgid(TelegramID(msgid), portal.tgid)
+            message = DBMessage.get_one_by_tgid(TelegramID(msgid), portal.tgid)
             if message:
                 url = f"https://matrix.to/#/{portal.mxid}/{message.mxid}"
 
     html.append(f"<a href='{url}'>{entity_text}</a>")
     return False
-
-
-def init_tg(context: "Context") -> None:
-    global should_highlight_edits
-    should_highlight_edits = htmldiff and context.config["bridge.highlight_edits"]

mautrix_telegram/formatter/util.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -20,38 +20,6 @@ import struct
 import re
 
 
-def unicode_to_html(text: str, html: str, ctrl: str, tag: str) -> str:
-    if ctrl not in text:
-        return html
-    if not html:
-        html = escape(text)
-    tag_start = f"<{tag}>"
-    tag_end = f"</{tag}>"
-    characters = html.split(ctrl)
-    html = ""
-    in_tag = False
-    for char in characters:
-        if not in_tag:
-            if len(char) > 1:
-                html += char[0:-1]
-                char = char[-1]
-            html += tag_start
-            in_tag = True
-            html += char
-        else:
-            if len(char) > 1:
-                html += tag_end
-                in_tag = False
-            html += char
-    if in_tag:
-        html += tag_end
-    return html
-
-
-def html_to_unicode(text: str, ctrl: str) -> str:
-    return ctrl.join(text) + ctrl
-
-
 # add_surrogates and remove_surrogates are unicode surrogate utility functions from Telethon.
 # Licensed under the MIT license.
 # https://github.com/LonamiWebs/Telethon/blob/7cce7aa3e4c6c7019a55530391b1761d33e5a04e/telethon/helpers.py

mautrix_telegram/matrix.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -17,6 +17,7 @@
 from typing import Dict, List, Match, Optional, Set, Tuple, TYPE_CHECKING
 import logging
 import asyncio
+import time
 import re
 
 from mautrix_appservice import MatrixRequestError, IntentError
@@ -27,6 +28,15 @@ from . import user as u, portal as po, puppet as pu, commands as com
 if TYPE_CHECKING:
     from .context import Context
 
+try:
+    from prometheus_client import Histogram
+
+    EVENT_TIME = Histogram("matrix_event", "Time spent processing Matrix events",
+                           ["event_type"])
+except ImportError:
+    Histogram = None
+    EVENT_TIME = None
+
 
 class MatrixHandler:
     log = logging.getLogger("mau.mx")  # type: logging.Logger
@@ -126,12 +136,31 @@ class MatrixHandler:
 
         if not inviter.whitelisted:
             await self.az.intent.send_notice(
-                room_id, text="",
-                html="You are not whitelisted to use this bridge.<br/><br/>"
+                room_id,
+                text="You are not whitelisted to use this bridge.\n\n"
                      "If you are the owner of this bridge, see the "
-                     "<code>bridge.permissions</code> section in your config file.")
+                     "`bridge.permissions` section in your config file.",
+                html="<p>You are not whitelisted to use this bridge.</p>"
+                     "<p>If you are the owner of this bridge, see the "
+                     "<code>bridge.permissions</code> section in your config file.</p>")
             await self.az.intent.leave_room(room_id)
 
+        try:
+            is_management = len(await self.az.intent.get_room_members(room_id)) == 2
+        except MatrixRequestError:
+            is_management = False
+        cmd_prefix = self.commands.command_prefix
+        text = html = "Hello, I'm a Telegram bridge bot. "
+        if is_management and inviter.puppet_whitelisted and not await inviter.is_logged_in():
+            text += f"Use `{cmd_prefix} help` for help or `{cmd_prefix} login` to log in."
+            html += (f"Use <code>{cmd_prefix} help</code> for help"
+                     f" or <code>{cmd_prefix} login</code> to log in.")
+            pass
+        else:
+            text += f"Use `{cmd_prefix} help` for help."
+            html += f"Use <code>{cmd_prefix} help</code> for help."
+        await self.az.intent.send_notice(room_id, text=text, html=html)
+
     async def handle_invite(self, room_id: MatrixRoomID, user_id: MatrixUserID,
                             inviter_mxid: MatrixUserID) -> None:
         self.log.debug(f"{inviter_mxid} invited {user_id} to {room_id}")
@@ -213,7 +242,7 @@ class MatrixHandler:
         prefix = self.config["bridge.command_prefix"]
         is_command = text.startswith(prefix)
         if is_command:
-            text = text[len(prefix) + 1:]
+            text = text[len(prefix) + 1:].lstrip()
         return is_command, text
 
     async def handle_message(self, room: MatrixRoomID, sender_id: MatrixUserID, message: Dict,
@@ -370,63 +399,79 @@ class MatrixHandler:
         return (sender == self.az.bot_mxid
                 or pu.Puppet.get_id_from_mxid(sender) is not None)
 
-    async def try_handle_event(self, evt: MatrixEvent) -> None:
+    async def try_handle_ephemeral_event(self, evt: MatrixEvent) -> None:
         try:
-            await self.handle_event(evt)
+            await self.handle_ephemeral_event(evt)
         except Exception:
             self.log.exception("Error handling manually received Matrix event")
 
-    async def handle_event(self, evt: MatrixEvent) -> None:
-        if self.filter_matrix_event(evt):
-            return
-        self.log.debug("Received event: %s", evt)
+    async def handle_ephemeral_event(self, evt: MatrixEvent) -> None:
         evt_type = evt.get("type", "m.unknown")  # type: str
         room_id = evt.get("room_id", None)  # type: Optional[MatrixRoomID]
-        event_id = evt.get("event_id", None)  # type: Optional[MatrixEventID]
         sender = evt.get("sender", None)  # type: Optional[MatrixUserID]
         content = evt.get("content", {})  # type: Dict
-        if evt_type == "m.room.member":
-            state_key = evt["state_key"]  # type: MatrixUserID
-            prev_content = evt.get("unsigned", {}).get("prev_content", {})  # type: Dict
-            membership = content.get("membership", "")  # type: str
-            prev_membership = prev_content.get("membership", "leave")  # type: str
-            if membership == prev_membership:
-                match = re.compile("@(.+):(.+)").match(state_key)  # type: Match
-                mxid = match.group(0)  # type: str
-                displayname = content.get("displayname", None) or mxid  # type: str
-                prev_displayname = prev_content.get("displayname", None) or mxid  # type: str
-                if displayname != prev_displayname:
-                    await self.handle_name_change(room_id, state_key, displayname,
-                                                  prev_displayname, event_id)
-            elif membership == "invite":
-                await self.handle_invite(room_id, state_key, sender)
-            elif prev_membership == "join" and membership == "leave":
-                await self.handle_part(room_id, state_key, sender, event_id)
-            elif membership == "join":
-                await self.handle_join(room_id, state_key, event_id)
-        elif evt_type in ("m.room.message", "m.sticker"):
-            if evt_type != "m.room.message":
-                content["msgtype"] = evt_type
-            await self.handle_message(room_id, sender, content, event_id)
-        elif evt_type == "m.room.redaction":
-            await self.handle_redaction(room_id, sender, evt["redacts"])
-        elif evt_type == "m.room.power_levels":
-            prev_content = evt.get("unsigned", {}).get("prev_content", {})
-            await self.handle_power_levels(room_id, sender, evt["content"], prev_content)
-        elif evt_type in ("m.room.name", "m.room.avatar", "m.room.topic"):
-            await self.handle_room_meta(evt_type, room_id, sender, evt["content"])
-        elif evt_type == "m.room.pinned_events":
-            new_events = set(evt["content"]["pinned"])
-            try:
-                old_events = set(evt["unsigned"]["prev_content"]["pinned"])
-            except KeyError:
-                old_events = set()
-            await self.handle_room_pin(room_id, sender, new_events, old_events)
-        elif evt_type == "m.room.tombstone":
-            await self.handle_room_upgrade(room_id, evt["content"]["replacement_room"])
-        elif evt_type == "m.receipt":
+        if evt_type == "m.receipt":
             await self.handle_read_receipts(room_id, self.parse_read_receipts(content))
         elif evt_type == "m.presence":
             await self.handle_presence(sender, content.get("presence", "offline"))
         elif evt_type == "m.typing":
             await self.handle_typing(room_id, content.get("user_ids", []))
+
+    async def handle_event(self, evt: MatrixEvent) -> None:
+        if self.filter_matrix_event(evt):
+            return
+        start_time = time.time()
+        self.log.debug("Received event: %s", evt)
+        evt_type = evt.get("type", "m.unknown")  # type: str
+        room_id = evt.get("room_id", None)  # type: Optional[MatrixRoomID]
+        event_id = evt.get("event_id", None)  # type: Optional[MatrixEventID]
+        sender = evt.get("sender", None)  # type: Optional[MatrixUserID]
+        state_key = evt.get("state_key", None)
+        content = evt.get("content", {})  # type: Dict
+        if state_key is not None:
+            if evt_type == "m.room.member":
+                prev_content = evt.get("unsigned", {}).get("prev_content", {})  # type: Dict
+                membership = content.get("membership", "")  # type: str
+                prev_membership = prev_content.get("membership", "leave")  # type: str
+                if membership == prev_membership:
+                    match = re.compile("@(.+):(.+)").match(state_key)  # type: Match
+                    mxid = match.group(0)  # type: str
+                    displayname = content.get("displayname", None) or mxid  # type: str
+                    prev_displayname = prev_content.get("displayname", None) or mxid  # type: str
+                    if displayname != prev_displayname:
+                        await self.handle_name_change(room_id, state_key, displayname,
+                                                      prev_displayname, event_id)
+                elif membership == "invite":
+                    await self.handle_invite(room_id, state_key, sender)
+                elif prev_membership == "join" and membership == "leave":
+                    await self.handle_part(room_id, state_key, sender, event_id)
+                elif membership == "join":
+                    await self.handle_join(room_id, state_key, event_id)
+            elif evt_type == "m.room.power_levels":
+                prev_content = evt.get("unsigned", {}).get("prev_content", {})
+                await self.handle_power_levels(room_id, sender, evt["content"], prev_content)
+            elif evt_type in ("m.room.name", "m.room.avatar", "m.room.topic"):
+                await self.handle_room_meta(evt_type, room_id, sender, evt["content"])
+            elif evt_type == "m.room.pinned_events":
+                new_events = set(evt["content"]["pinned"])
+                try:
+                    old_events = set(evt["unsigned"]["prev_content"]["pinned"])
+                except KeyError:
+                    old_events = set()
+                await self.handle_room_pin(room_id, sender, new_events, old_events)
+            elif evt_type == "m.room.tombstone":
+                await self.handle_room_upgrade(room_id, evt["content"]["replacement_room"])
+            else:
+                return
+        else:
+            if evt_type in ("m.room.message", "m.sticker"):
+                if evt_type != "m.room.message":
+                    content["msgtype"] = evt_type
+                await self.handle_message(room_id, sender, content, event_id)
+            elif evt_type == "m.room.redaction":
+                await self.handle_redaction(room_id, sender, evt["redacts"])
+            else:
+                return
+
+        if EVENT_TIME:
+            EVENT_TIME.labels(event_type=evt_type).observe(time.time() - start_time)

mautrix_telegram/portal.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -43,31 +43,33 @@ from telethon.tl.functions.channels import (
     UpdateUsernameRequest)
 from telethon.tl.functions.messages import ReadHistoryRequest as ReadMessageHistoryRequest
 from telethon.tl.functions.channels import ReadHistoryRequest as ReadChannelHistoryRequest
-from telethon.errors import ChatAdminRequiredError, ChatNotModifiedError
+from telethon.errors import (ChatAdminRequiredError, ChatNotModifiedError, PhotoExtInvalidError,
+                             PhotoInvalidDimensionsError, PhotoSaveFileInvalidError)
 from telethon.tl.patched import Message, MessageService
 from telethon.tl.types import (
-    Channel, ChatAdminRights, ChatBannedRights, ChannelFull, ChannelParticipantAdmin,
-    ChannelParticipantCreator, ChannelParticipantsRecent, ChannelParticipantsSearch, Chat, ChatFull,
-    ChatInviteEmpty, ChatParticipantAdmin, ChatParticipantCreator, ChatPhoto, Poll, PollAnswer,
-    DocumentAttributeFilename, DocumentAttributeImageSize, DocumentAttributeSticker,
-    DocumentAttributeVideo, FileLocation, GeoPoint, InputChannel, InputChatUploadedPhoto,
-    InputPeerChannel, InputPeerChat, InputPeerUser, InputUser, InputUserSelf,
-    MessageActionChannelCreate, MessageActionChatAddUser, MessageActionChatCreate,
+    Channel, ChatAdminRights, ChatBannedRights, ChannelFull, ChannelParticipantAdmin, Document,
+    ChannelParticipantCreator, ChannelParticipantsRecent, ChannelParticipantsSearch, Chat,
+    ChatFull, ChatInviteEmpty, ChatParticipantAdmin, ChatParticipantCreator, ChatPhoto, Poll,
+    DocumentAttributeFilename, DocumentAttributeImageSize, DocumentAttributeSticker, PhotoEmpty,
+    DocumentAttributeVideo, GeoPoint, InputChannel, InputChatUploadedPhoto, InputPhotoFileLocation,
+    InputPeerChannel, InputPeerChat, InputPeerUser, InputUser, InputUserSelf, MessageMediaPoll,
+    MessageActionChannelCreate, MessageActionChatAddUser, MessageActionChatCreate, ChatPhotoEmpty,
     MessageActionChatDeletePhoto, MessageActionChatDeleteUser, MessageActionChatEditPhoto,
     MessageActionChatEditTitle, MessageActionChatJoinedByLink, MessageActionChatMigrateTo,
     MessageActionPinMessage, MessageActionGameScore, MessageMediaContact, MessageMediaDocument,
-    MessageMediaGeo, MessageMediaPhoto, MessageMediaUnsupported, MessageMediaGame, MessageMediaPoll,
+    MessageMediaGeo, MessageMediaPhoto, MessageMediaUnsupported, MessageMediaGame,
     PeerChannel, PeerChat, PeerUser, Photo, PhotoCachedSize, SendMessageCancelAction,
     SendMessageTypingAction, TypeChannelParticipant, TypeChat, TypeChatParticipant,
     TypeDocumentAttribute, TypeInputPeer, TypeMessageAction, TypeMessageEntity, TypePeer,
     TypePhotoSize, TypeUpdates, TypeUser, PhotoSize, TypeUserFull, UpdateChatUserTyping,
-    UpdateNewChannelMessage, UpdateNewMessage, UpdateUserTyping, User, UserFull, MessageEntityPre)
+    UpdateNewChannelMessage, UpdateNewMessage, UpdateUserTyping, User, UserFull, MessageEntityPre,
+    InputMediaUploadedDocument, InputPeerPhotoFileLocation)
 from mautrix_appservice import MatrixRequestError, IntentError, AppService, IntentAPI
 
 from .types import MatrixEventID, MatrixRoomID, MatrixUserID, TelegramID
 from .context import Context
 from .db import Portal as DBPortal, Message as DBMessage, TelegramFile as DBTelegramFile
-from .util import ignore_coro
+from .util import ignore_coro, sane_mimetypes
 from . import puppet as p, user as u, formatter, util
 
 if TYPE_CHECKING:
@@ -76,8 +78,6 @@ if TYPE_CHECKING:
     from .config import Config
     from .tgclient import MautrixTelegramClient
 
-mimetypes.init()
-
 config = None  # type: Config
 
 TypeMessage = Union[Message, MessageService]
@@ -346,7 +346,10 @@ class Portal:
                 await self.invite_to_matrix(invites or [])
             return self.mxid
         async with self._room_create_lock:
-            return await self._create_matrix_room(user, entity, invites)
+            try:
+                return await self._create_matrix_room(user, entity, invites)
+            except Exception:
+                self.log.exception("Fatal error creating Matrix room")
 
     async def _create_matrix_room(self, user: 'AbstractUser', entity: TypeChat, invites: InviteList
                                   ) -> Optional[MatrixRoomID]:
@@ -397,6 +400,11 @@ class Portal:
             "type": "m.room.power_levels",
             "content": power_levels,
         }]
+        if config["appservice.community_id"]:
+            initial_state.append({
+                "type": "m.room.related_groups",
+                "content": {"groups": [config["appservice.community_id"]]},
+            })
 
         room_id = await self.main_intent.create_room(alias=alias, is_public=public,
                                                      is_direct=direct, invitees=invites or [],
@@ -573,7 +581,7 @@ class Portal:
         changed = await self.update_title(entity.title) or changed
 
         if isinstance(entity.photo, ChatPhoto):
-            changed = await self.update_avatar(user, entity.photo.photo_big) or changed
+            changed = await self.update_avatar(user, entity.photo) or changed
 
         if changed:
             self.save()
@@ -614,12 +622,23 @@ class Portal:
         return False
 
     @staticmethod
-    def _get_largest_photo_size(photo: Union[Photo, List[TypePhotoSize]]
-                                ) -> Optional[TypePhotoSize]:
+    def _get_largest_photo_size(photo: Union[Photo, Document]
+                                ) -> Tuple[Optional[InputPhotoFileLocation],
+                                           Optional[TypePhotoSize]]:
         if not photo:
-            return None
-        return max(photo.sizes if isinstance(photo, Photo) else photo, key=(lambda photo2: (
-            len(photo2.bytes) if not isinstance(photo2, PhotoSize) else photo2.size)))
+            return None, None
+        if isinstance(photo, Document) and not photo.thumbs:
+            return None, None
+        largest = max(photo.sizes if isinstance(photo, Photo) else photo.thumbs,
+                      key=(lambda photo2: (len(photo2.bytes)
+                                           if not isinstance(photo2, PhotoSize)
+                                           else photo2.size)))
+        return InputPhotoFileLocation(
+            id=photo.id,
+            access_hash=photo.access_hash,
+            file_reference=photo.file_reference,
+            thumb_size=largest.type,
+        ), largest
 
     async def remove_avatar(self, _: 'AbstractUser', save: bool = False) -> None:
         await self.main_intent.set_room_avatar(self.mxid, None)
@@ -627,11 +646,33 @@ class Portal:
         if save:
             self.save()
 
-    async def update_avatar(self, user: 'AbstractUser', photo: FileLocation,
+    async def update_avatar(self, user: 'AbstractUser',
+                            photo: Union[ChatPhoto, ChatPhotoEmpty, Photo, PhotoEmpty],
                             save: bool = False) -> bool:
-        photo_id = f"{photo.volume_id}-{photo.local_id}"
+        if isinstance(photo, ChatPhoto):
+            loc = InputPeerPhotoFileLocation(
+                peer=await self.get_input_entity(user),
+                local_id=photo.photo_big.local_id,
+                volume_id=photo.photo_big.volume_id,
+                big=True
+            )
+            photo_id = f"{loc.volume_id}-{loc.local_id}"
+        elif isinstance(photo, Photo):
+            loc, largest = self._get_largest_photo_size(photo)
+            photo_id = f"{largest.location.volume_id}-{largest.location.local_id}"
+        elif isinstance(photo, (ChatPhotoEmpty, PhotoEmpty)):
+            photo_id = ""
+            loc = None
+        else:
+            raise ValueError(f"Unknown photo type {type(photo)}")
         if self.photo_id != photo_id:
-            file = await util.transfer_file_to_matrix(user.client, self.main_intent, photo)
+            if not photo_id:
+                await self.main_intent.set_room_avatar(self.mxid, "")
+                self.photo_id = ""
+                if save:
+                    self.save()
+                return True
+            file = await util.transfer_file_to_matrix(user.client, self.main_intent, loc)
             if file:
                 await self.main_intent.set_room_avatar(self.mxid, file.mxc)
                 self.photo_id = photo_id
@@ -750,14 +791,9 @@ class Portal:
                 return body + current_extension
         except (ValueError, KeyError):
             pass
-        ext_override = {
-            "image/jpeg": ".jpg"
-        }
         if mime:
-            ext = ext_override.get(mime, mimetypes.guess_extension(mime))
-            return f"matrix_upload{ext}"
-        else:
-            return ""
+            return f"matrix_upload{sane_mimetypes.guess_extension(mime)}"
+        return ""
 
     def get_config(self, key: str) -> Any:
         local = util.recursive_get(self.local_config, key)
@@ -899,7 +935,7 @@ class Portal:
                                 ) -> None:
         if "formatted_body" not in message:
             message["format"] = "org.matrix.custom.html"
-            message["formatted_body"] = escape_html(message.get("body", ""))
+            message["formatted_body"] = escape_html(message.get("body", "")).replace("\n", "<br/>")
         body = message["formatted_body"]
 
         tpl = (self.get_config(f"message_formats.[{msgtype}]")
@@ -916,9 +952,14 @@ class Portal:
         msgtype = message.get("msgtype", "m.text")
         if msgtype == "m.emote":
             await self._apply_msg_format(sender, msgtype, message)
+            if "m.new_content" in message:
+                await self._apply_msg_format(sender, msgtype, message["m.new_content"])
+                message["m.new_content"]["msgtype"] = "m.text"
             message["msgtype"] = "m.text"
         elif use_relaybot:
             await self._apply_msg_format(sender, msgtype, message)
+            if "m.new_content" in message:
+                await self._apply_msg_format(sender, msgtype, message["m.new_content"])
 
     @staticmethod
     def _matrix_event_to_entities(event: Dict[str, Any]
@@ -950,15 +991,26 @@ class Portal:
             return None
 
     async def _handle_matrix_text(self, sender_id: TelegramID, event_id: MatrixEventID,
-                                  space: TelegramID, client: 'MautrixTelegramClient', message: Dict,
-                                  reply_to: TelegramID) -> None:
+                                  space: TelegramID, client: 'MautrixTelegramClient',
+                                  message: Dict, reply_to: TelegramID) -> None:
         lock = self.require_send_lock(sender_id)
         async with lock:
             lp = self.get_config("telegram_link_preview")
+            relates_to = message.get("m.relates_to", None) or {}
+            if relates_to.get("rel_type", None) == "m.replace":
+                orig_msg = DBMessage.get_by_mxid(relates_to.get("event_id", ""), self.mxid, space)
+                if orig_msg and "m.new_content" in message:
+                    message = message["m.new_content"]
+                    formatter.matrix_reply_to_telegram(message, space, room_id=self.mxid)
+                    response = await client.edit_message(self.peer, orig_msg.tgid, message,
+                                                         parse_mode=self._matrix_event_to_entities,
+                                                         link_preview=lp)
+                    self._add_telegram_message_to_db(event_id, space, -1, response)
+                    return
             response = await client.send_message(self.peer, message, reply_to=reply_to,
                                                  parse_mode=self._matrix_event_to_entities,
                                                  link_preview=lp)
-            self._add_telegram_message_to_db(event_id, space, response)
+            self._add_telegram_message_to_db(event_id, space, 0, response)
 
     async def _handle_matrix_file(self, msgtype: str, sender_id: TelegramID,
                                   event_id: MatrixEventID, space: TelegramID,
@@ -993,9 +1045,23 @@ class Portal:
             max_image_size=config["bridge.image_as_file_size"] * 1000 ** 2)
         lock = self.require_send_lock(sender_id)
         async with lock:
-            response = await client.send_media(self.peer, media, reply_to=reply_to,
-                                               caption=caption)
-            self._add_telegram_message_to_db(event_id, space, response)
+            relates_to = message.get("m.relates_to", None) or {}
+            if relates_to.get("rel_type", None) == "m.replace":
+                orig_msg = DBMessage.get_by_mxid(relates_to.get("event_id", ""), self.mxid, space)
+                if orig_msg:
+                    response = await client.edit_message(self.peer, orig_msg.tgid,
+                                                         caption, file=media)
+                    self._add_telegram_message_to_db(event_id, space, -1, response)
+                    return
+            try:
+                response = await client.send_media(self.peer, media, reply_to=reply_to,
+                                                   caption=caption)
+            except (PhotoInvalidDimensionsError, PhotoSaveFileInvalidError, PhotoExtInvalidError):
+                media = InputMediaUploadedDocument(file=media.file, mime_type=mime,
+                                                   attributes=attributes)
+                response = await client.send_media(self.peer, media, reply_to=reply_to,
+                                                   caption=caption)
+            self._add_telegram_message_to_db(event_id, space, 0, response)
 
     async def _handle_matrix_location(self, sender_id: TelegramID, event_id: MatrixEventID,
                                       space: TelegramID, client: 'MautrixTelegramClient',
@@ -1011,19 +1077,31 @@ class Portal:
 
         lock = self.require_send_lock(sender_id)
         async with lock:
+            relates_to = message.get("m.relates_to", None) or {}
+            if relates_to.get("rel_type", None) == "m.replace":
+                orig_msg = DBMessage.get_by_mxid(relates_to.get("event_id", ""), self.mxid, space)
+                if orig_msg:
+                    response = await client.edit_message(self.peer, orig_msg.tgid,
+                                                         caption, file=media)
+                    self._add_telegram_message_to_db(event_id, space, -1, response)
+                    return
             response = await client.send_media(self.peer, media, reply_to=reply_to,
                                                caption=caption, entities=entities)
-            self._add_telegram_message_to_db(event_id, space, response)
+            self._add_telegram_message_to_db(event_id, space, 0, response)
 
     def _add_telegram_message_to_db(self, event_id: MatrixEventID, space: TelegramID,
-                                    response: TypeMessage) -> None:
+                                    edit_index: int, response: TypeMessage) -> None:
         self.log.debug("Handled Matrix message: %s", response)
-        self.is_duplicate(response, (event_id, space))
+        self.is_duplicate(response, (event_id, space), force_hash=edit_index != 0)
+        if edit_index < 0:
+            prev_edit = DBMessage.get_one_by_tgid(TelegramID(response.id), space, -1)
+            edit_index = prev_edit.edit_index + 1
         DBMessage(
             tgid=TelegramID(response.id),
             tg_space=space,
             mx_room=self.mxid,
-            mxid=event_id).insert()
+            mxid=event_id,
+            edit_index=edit_index).insert()
 
     async def handle_matrix_message(self, sender: 'u.User', message: Dict[str, Any],
                                     event_id: MatrixEventID) -> None:
@@ -1087,7 +1165,10 @@ class Portal:
         message = DBMessage.get_by_mxid(event_id, self.mxid, space)
         if not message:
             return
-        await real_deleter.client.delete_messages(self.peer, [message.tgid])
+        if message.edit_index == 0:
+            await real_deleter.client.delete_messages(self.peer, [message.tgid])
+        else:
+            self.log.debug(f"Ignoring deletion of edit event {message.mxid} in {message.mx_room}")
 
     async def _update_telegram_power_level(self, sender: 'u.User', user_id: TelegramID,
                                            level: int) -> None:
@@ -1151,7 +1232,7 @@ class Portal:
 
         file = await self.main_intent.download_file(url)
         mime = magic.from_buffer(file, mime=True)
-        ext = mimetypes.guess_extension(mime)
+        ext = sane_mimetypes.guess_extension(mime)
         uploaded = await sender.client.upload_file(file, file_name=f"avatar{ext}", use_cache=False)
         photo = InputChatUploadedPhoto(file=uploaded)
 
@@ -1166,8 +1247,8 @@ class Portal:
                                and isinstance(update.message, MessageService)
                                and isinstance(update.message.action, MessageActionChatEditPhoto))
             if is_photo_update:
-                loc = self._get_largest_photo_size(update.message.action.photo).location
-                self.photo_id = f"{loc.volume_id}-{loc.local_id}"
+                loc, size = self._get_largest_photo_size(update.message.action.photo)
+                self.photo_id = f"{size.location.volume_id}-{size.location.local_id}"
                 self.save()
                 break
 
@@ -1258,8 +1339,13 @@ class Portal:
 
         invites = await self._get_telegram_users_in_matrix_room()
         if len(invites) < 2:
-            raise ValueError("Not enough Telegram users to create a chat")
-
+            if self.bot is not None:
+                info, mxid = await self.bot.get_me()
+                raise ValueError("Not enough Telegram users to create a chat. "
+                                 "Invite more Telegram ghost users to the room, such as the "
+                                 f"relaybot ([{info.first_name}](https://matrix.to/#/{mxid})).")
+            raise ValueError("Not enough Telegram users to create a chat. "
+                             "Invite more Telegram ghost users to the room.")
         if self.peer_type == "chat":
             response = await source.client(CreateChatRequest(title=self.title, users=invites))
             entity = response.chats[0]
@@ -1311,12 +1397,14 @@ class Portal:
     def get_external_url(self, evt: Message) -> Optional[str]:
         if self.peer_type == "channel" and self.username is not None:
             return f"https://t.me/{self.username}/{evt.id}"
+        elif self.peer_type != "user":
+            return f"https://t.me/c/{self.tgid}/{evt.id}"
         return None
 
     async def handle_telegram_photo(self, source: 'AbstractUser', intent: IntentAPI, evt: Message,
                                     relates_to: Dict = None) -> Optional[Dict]:
-        largest_size = self._get_largest_photo_size(evt.media.photo)
-        file = await util.transfer_file_to_matrix(source.client, intent, largest_size.location)
+        loc, largest_size = self._get_largest_photo_size(evt.media.photo)
+        file = await util.transfer_file_to_matrix(source.client, intent, loc)
         if not file:
             return None
         if self.get_config("inline_images") and (evt.message
@@ -1337,18 +1425,16 @@ class Portal:
             "orientation": 0,
             "mimetype": file.mime_type,
         }
-        ext_override = {
-            "image/jpeg": ".jpg"
-        }
-        name = "image" + ext_override.get(file.mime_type, mimetypes.guess_extension(file.mime_type))
+        name = f"image{sane_mimetypes.guess_extension(file.mime_type)}"
         await intent.set_typing(self.mxid, is_typing=False)
         result = await intent.send_image(self.mxid, file.mxc, info=info, text=name,
                                          relates_to=relates_to, timestamp=evt.date,
                                          external_url=self.get_external_url(evt))
         if evt.message:
-            text, html, _ = await formatter.telegram_to_matrix(evt, source, self.main_intent)
-            await intent.send_text(self.mxid, text, html=html, timestamp=evt.date,
-                                   external_url=self.get_external_url(evt))
+            text, html, _ = await formatter.telegram_to_matrix(evt, source, self.main_intent,
+                                                               no_reply_fallback=True)
+            result = await intent.send_text(self.mxid, text, html=html, timestamp=evt.date,
+                                            external_url=self.get_external_url(evt))
         return result
 
     @staticmethod
@@ -1374,7 +1460,7 @@ class Portal:
 
     @staticmethod
     def _parse_telegram_document_meta(evt: Message, file: DBTelegramFile, attrs: Dict,
-                                      thumb: TypePhotoSize) -> Tuple[Dict, str]:
+                                      thumb_size: TypePhotoSize) -> Tuple[Dict, str]:
         document = evt.media.document
         name = evt.message or attrs["name"]
         if attrs["is_sticker"]:
@@ -1385,7 +1471,11 @@ class Portal:
                 except ValueError:
                     name = alt
 
-        mime_type = document.mime_type or file.mime_type
+        generic_types = ("text/plain", "application/octet-stream")
+        if file.mime_type in generic_types and document.mime_type not in generic_types:
+            mime_type = document.mime_type or file.mime_type
+        else:
+            mime_type = file.mime_type or document.mime_type
         info = {
             "size": file.size,
             "mimetype": mime_type,
@@ -1402,8 +1492,8 @@ class Portal:
             info["thumbnail_url"] = file.thumbnail.mxc
             info["thumbnail_info"] = {
                 "mimetype": file.thumbnail.mime_type,
-                "h": file.thumbnail.height or thumb.h,
-                "w": file.thumbnail.width or thumb.w,
+                "h": file.thumbnail.height or thumb_size.h,
+                "w": file.thumbnail.width or thumb_size.w,
                 "size": file.thumbnail.size,
             }
 
@@ -1412,18 +1502,25 @@ class Portal:
     async def handle_telegram_document(self, source: 'AbstractUser', intent: IntentAPI,
                                        evt: Message, relates_to: dict = None) -> Optional[Dict]:
         document = evt.media.document
+
         attrs = self._parse_telegram_document_attributes(document.attributes)
 
-        thumb = self._get_largest_photo_size(document.thumbs)
-        if thumb and not isinstance(thumb, (PhotoSize, PhotoCachedSize)):
-            self.log.debug(f"Unsupported thumbnail type {type(thumb)}")
-            thumb = None
-        file = await util.transfer_file_to_matrix(source.client, intent, document, thumb,
+        if document.size > config["bridge.max_document_size"] * 1000 ** 2:
+            name = attrs["name"] or ""
+            caption = f"\n{evt.message}" if evt.message else ""
+            return await intent.send_notice(self.mxid, f"Too large file {name}{caption}")
+
+        thumb_loc, thumb_size = self._get_largest_photo_size(document)
+        if thumb_size and not isinstance(thumb_size, (PhotoSize, PhotoCachedSize)):
+            self.log.debug(f"Unsupported thumbnail type {type(thumb_size)}")
+            thumb_loc = None
+            thumb_size = None
+        file = await util.transfer_file_to_matrix(source.client, intent, document, thumb_loc,
                                                   is_sticker=attrs["is_sticker"])
         if not file:
             return None
 
-        info, name = self._parse_telegram_document_meta(evt, file, attrs, thumb)
+        info, name = self._parse_telegram_document_meta(evt, file, attrs, thumb_size)
 
         await intent.set_typing(self.mxid, is_typing=False)
 
@@ -1490,7 +1587,7 @@ class Portal:
                                       external_url=self.get_external_url(evt))
 
     async def handle_telegram_unsupported(self, source: 'AbstractUser', intent: IntentAPI,
-                                          evt: Message, _: dict = None) -> dict:
+                                          evt: Message, relates_to: dict = None) -> dict:
         override_text = ("This message is not supported on your version of Mautrix-Telegram. "
                          "Please check https://github.com/tulir/mautrix-telegram or ask your "
                          "bridge administrator about possible updates.")
@@ -1511,20 +1608,24 @@ class Portal:
         poll = evt.media.poll  # type: Poll
         poll_id = self._encode_msgid(source, evt)
 
-        def enc(answer: PollAnswer) -> str:
-            return base64.b64encode(answer.option).decode("utf-8").rstrip("=")
+        _n = 0
 
-        text = (f"Poll ID {poll_id}: {poll.question}\n"
-                + "\n".join(f"* {enc(answer)}: {answer.text}" for answer in poll.answers) +
+        def n() -> int:
+            nonlocal _n
+            _n += 1
+            return _n
+
+        text = (f"Poll: {poll.question}\n"
+                + "\n".join(f"{n()}. {answer.text}" for answer in poll.answers) +
                 "\n"
-                f"Vote with !tg vote <poll ID> <choice ID>")
+                f"Vote with !tg vote {poll_id} <choice number>")
 
-        html = (f"<strong>Poll</strong> ID <code>{poll_id}</code>: {poll.question}<br/>\n"
-                f"<ul>"
-                + "\n".join(f"<li><code>{enc(answer)}</code>: {answer.text}</li>"
+        html = (f"<strong>Poll</strong>: {poll.question}<br/>\n"
+                f"<ol>"
+                + "\n".join(f"<li>{answer.text}</li>"
                             for answer in poll.answers) +
-                "</ul>\n"
-                f"Vote with <code>!tg vote &lt;poll ID&gt; &lt;choice ID&gt;</code>")
+                "</ol>\n"
+                f"Vote with <code>!tg vote {poll_id} &lt;choice number&gt;</code>")
         await intent.set_typing(self.mxid, is_typing=False)
         return await intent.send_text(self.mxid, text, html=html, relates_to=relates_to,
                                       msgtype="m.text", timestamp=evt.date,
@@ -1559,7 +1660,8 @@ class Portal:
         play_id = self._encode_msgid(source, evt)
         command = f"!tg play {play_id}"
         override_text = f"Run {command} in your bridge management room to play {game.title}"
-        override_entities = [MessageEntityPre(offset=len("Run "), length=len(command), language="")]
+        override_entities = [
+            MessageEntityPre(offset=len("Run "), length=len(command), language="")]
         text, html, relates_to = await formatter.telegram_to_matrix(
             evt, source, self.main_intent,
             override_text=override_text, override_entities=override_entities)
@@ -1577,9 +1679,6 @@ class Portal:
                                    evt: Message) -> None:
         if not self.mxid:
             return
-        elif not self.get_config("edits_as_replies"):
-            self.log.debug("Edits as replies disabled, ignoring edit event...")
-            return
         elif hasattr(evt, "media") and isinstance(evt.media, (MessageMediaGame,)):
             self.log.debug("Ignoring game message edit event")
             return
@@ -1597,28 +1696,52 @@ class Portal:
         if duplicate_found:
             mxid, other_tg_space = duplicate_found
             if tg_space != other_tg_space:
-                DBMessage.update_by_tgid(TelegramID(evt.id), tg_space,
-                                         mxid=mxid,
-                                         mx_room=self.mxid)
+                prev_edit_msg = DBMessage.get_one_by_tgid(TelegramID(evt.id), tg_space, -1)
+                if not prev_edit_msg:
+                    return
+                DBMessage(mxid=mxid, mx_room=self.mxid, tg_space=tg_space, tgid=evt.id,
+                          edit_index=prev_edit_msg.edit_index + 1).insert()
             return
 
-        evt.reply_to_msg_id = evt.id
-        text, html, relates_to = await formatter.telegram_to_matrix(evt, source, self.main_intent,
-                                                                    is_edit=True)
-        intent = sender.intent if sender else self.main_intent
-        await intent.set_typing(self.mxid, is_typing=False)
-        response = await intent.send_text(self.mxid, text, html=html, relates_to=relates_to,
-                                          external_url=self.get_external_url(evt))
-
-        mxid = response["event_id"]
-
-        msg = DBMessage.get_by_tgid(TelegramID(evt.id), tg_space)
-        if not msg:
+        text, html, _ = await formatter.telegram_to_matrix(evt, source, self.main_intent,
+                                                           no_reply_fallback=True)
+        editing_msg = DBMessage.get_one_by_tgid(TelegramID(evt.id), tg_space)
+        if not editing_msg:
             self.log.info(f"Didn't find edited message {evt.id}@{tg_space} (src {source.tgid}) "
                           "in database.")
-            # Oh crap
             return
-        msg.update(mxid=mxid, mx_room=self.mxid)
+
+        msgtype = ("m.notice"
+                   if sender and sender.is_bot and self.get_config("bot_messages_as_notices")
+                   else "m.text")
+        content = {
+            "body": f"Edit: {text}",
+            "msgtype": msgtype,
+            "format": "org.matrix.custom.html",
+            "formatted_body": (f"<a href='https://matrix.to/#/{editing_msg.mx_room}/"
+                               f"{editing_msg.mxid}'>Edit</a>: "
+                               f"{html or escape_html(text)}"),
+            "external_url": self.get_external_url(evt),
+            "m.new_content": {
+                "body": text,
+                "msgtype": "m.text",
+                **({"format": "org.matrix.custom.html",
+                    "formatted_body": html} if html else {}),
+            },
+            "m.relates_to": {
+                "rel_type": "m.replace",
+                "event_id": editing_msg.mxid,
+            },
+        }
+
+        intent = sender.intent if sender else self.main_intent
+        await intent.set_typing(self.mxid, is_typing=False)
+        response = await intent.send_message(self.mxid, content)
+        mxid = response["event_id"]
+
+        prev_edit_msg = DBMessage.get_one_by_tgid(TelegramID(evt.id), tg_space, -1) or editing_msg
+        DBMessage(mxid=mxid, mx_room=self.mxid, tg_space=tg_space, tgid=evt.id,
+                  edit_index=prev_edit_msg.edit_index + 1).insert()
         DBMessage.update_by_mxid(temporary_identifier, self.mxid, mxid=mxid)
 
     async def handle_telegram_message(self, source: 'AbstractUser', sender: p.Puppet,
@@ -1642,11 +1765,11 @@ class Portal:
                            f"as it was already handled (in space {other_tg_space})")
             if tg_space != other_tg_space:
                 DBMessage(tgid=TelegramID(evt.id), mx_room=self.mxid, mxid=mxid,
-                          tg_space=tg_space).insert()
+                          tg_space=tg_space, edit_index=0).insert()
             return
 
         if self.dedup_pre_db_check and self.peer_type == "channel":
-            msg = DBMessage.get_by_tgid(TelegramID(evt.id), tg_space)
+            msg = DBMessage.get_one_by_tgid(TelegramID(evt.id), tg_space)
             if msg:
                 self.log.debug(f"Ignoring message {evt.id} (src {source.tgid}) as it was already"
                                f"handled into {msg.mxid}. This duplicate was catched in the db "
@@ -1655,13 +1778,13 @@ class Portal:
                 return
 
         if sender and not sender.displayname:
-            self.log.debug(f"Telegram user {sender.tgid} sent a message, but doesn't have a"
+            self.log.debug(f"Telegram user {sender.tgid} sent a message, but doesn't have a "
                            "displayname, updating info...")
             entity = await source.client.get_entity(PeerUser(sender.tgid))
             await sender.update_info(source, entity)
 
-        allowed_media = (MessageMediaPhoto, MessageMediaDocument, MessageMediaGeo, MessageMediaGame,
-                         MessageMediaPoll, MessageMediaUnsupported)
+        allowed_media = (MessageMediaPhoto, MessageMediaDocument, MessageMediaGeo,
+                         MessageMediaGame, MessageMediaPoll, MessageMediaUnsupported)
         media = evt.media if hasattr(evt, "media") and isinstance(evt.media,
                                                                   allowed_media) else None
         intent = sender.intent if sender else self.main_intent
@@ -1701,7 +1824,7 @@ class Portal:
         self.log.debug("Handled Telegram message: %s", evt)
         try:
             DBMessage(tgid=TelegramID(evt.id), mx_room=self.mxid, mxid=mxid,
-                      tg_space=tg_space).insert()
+                      tg_space=tg_space, edit_index=0).insert()
             DBMessage.update_by_mxid(temporary_identifier, self.mxid, mxid=mxid)
         except IntegrityError as e:
             self.log.exception(f"{e.__class__.__name__} while saving message mapping. "
@@ -1734,8 +1857,7 @@ class Portal:
         if isinstance(action, MessageActionChatEditTitle):
             await self.update_title(action.title, save=True)
         elif isinstance(action, MessageActionChatEditPhoto):
-            largest_size = self._get_largest_photo_size(action.photo)
-            await self.update_avatar(source, largest_size.location, save=True)
+            await self.update_avatar(source, action.photo, save=True)
         elif isinstance(action, MessageActionChatDeletePhoto):
             await self.remove_avatar(source, save=True)
         elif isinstance(action, MessageActionChatAddUser):
@@ -1780,7 +1902,7 @@ class Portal:
         self._temp_pinned_message_id = None
         self._temp_pinned_message_sender = None
 
-        message = DBMessage.get_by_tgid(msg_id, self._temp_pinned_message_id_space)
+        message = DBMessage.get_one_by_tgid(msg_id, self._temp_pinned_message_id_space)
         if message:
             await intent.set_pinned_messages(self.mxid, [message.mxid])
         else:
@@ -1902,7 +2024,7 @@ class Portal:
             existing.delete()
         except KeyError:
             pass
-        self.db_instance.update(tgid=new_id, tg_receiver=new_id)
+        self.db_instance.update(tgid=new_id, tg_receiver=new_id, peer_type=self.peer_type)
         old_id = self.tgid
         self.tgid = new_id
         self.tg_receiver = new_id

mautrix_telegram/puppet.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -22,7 +22,8 @@ import asyncio
 import logging
 import re
 
-from telethon.tl.types import UserProfilePhoto, User, FileLocation, UpdateUserName, PeerUser
+from telethon.tl.types import (UserProfilePhoto, User, UpdateUserName, PeerUser, TypeInputPeer,
+                               InputPeerPhotoFileLocation, UserProfilePhotoEmpty)
 from mautrix_appservice import AppService, IntentAPI, IntentError, MatrixRequestError
 
 from .types import MatrixUserID, TelegramID
@@ -61,6 +62,7 @@ class Puppet:
                  photo_id: Optional[str] = None,
                  is_bot: bool = False,
                  is_registered: bool = False,
+                 disable_updates: bool = False,
                  db_instance: Optional[DBPuppet] = None) -> None:
         self.id = id  # type: TelegramID
         self.access_token = access_token  # type: Optional[str]
@@ -73,6 +75,7 @@ class Puppet:
         self.photo_id = photo_id  # type: Optional[str]
         self.is_bot = is_bot  # type: bool
         self.is_registered = is_registered  # type: bool
+        self.disable_updates = disable_updates  # type: bool
         self._db_instance = db_instance  # type: Optional[DBPuppet]
 
         self.default_mxid_intent = self.az.intent.user(self.default_mxid)
@@ -111,6 +114,9 @@ class Puppet:
         match = regex.match(self.displayname)
         return match.group(1) or self.displayname
 
+    def get_input_entity(self, user: 'AbstractUser') -> Awaitable[TypeInputPeer]:
+        return user.client.get_input_entity(PeerUser(user_id=self.tgid))
+
     # region Custom puppet management
     def _fresh_intent(self) -> IntentAPI:
         return (self.az.intent.user(self.custom_mxid, self.access_token)
@@ -215,13 +221,13 @@ class Puppet:
         return new_events
 
     def handle_sync(self, presence: List, ephemeral: Dict) -> None:
-        presence_events = [self.mx.try_handle_event(event) for event in presence]
+        presence_events = [self.mx.try_handle_ephemeral_event(event) for event in presence]
 
         for room_id, events in ephemeral.items():
             for event in events:
                 event["room_id"] = room_id
 
-        ephemeral_events = [self.mx.try_handle_event(event)
+        ephemeral_events = [self.mx.try_handle_ephemeral_event(event)
                             for events in ephemeral.values()
                             for event in self.filter_events(events)]
 
@@ -281,23 +287,26 @@ class Puppet:
         return DBPuppet(id=self.id, access_token=self.access_token, custom_mxid=self.custom_mxid,
                         username=self.username, displayname=self.displayname,
                         displayname_source=self.displayname_source, photo_id=self.photo_id,
-                        is_bot=self.is_bot, matrix_registered=self.is_registered)
+                        is_bot=self.is_bot, matrix_registered=self.is_registered,
+                        disable_updates=self.disable_updates)
 
     @classmethod
     def from_db(cls, db_puppet: DBPuppet) -> 'Puppet':
         return Puppet(db_puppet.id, db_puppet.access_token, db_puppet.custom_mxid,
                       db_puppet.username, db_puppet.displayname, db_puppet.displayname_source,
                       db_puppet.photo_id, db_puppet.is_bot, db_puppet.matrix_registered,
-                      db_instance=db_puppet)
+                      db_puppet.disable_updates, db_instance=db_puppet)
 
     def save(self) -> None:
         self.db_instance.update(access_token=self.access_token, custom_mxid=self.custom_mxid,
                                 username=self.username, displayname=self.displayname,
                                 displayname_source=self.displayname_source, photo_id=self.photo_id,
-                                is_bot=self.is_bot, matrix_registered=self.is_registered)
+                                is_bot=self.is_bot, matrix_registered=self.is_registered,
+                                disable_updates=self.disable_updates)
 
     # endregion
     # region Info updating
+
     def similarity(self, query: str) -> int:
         username_similarity = (SequenceMatcher(None, self.username, query).ratio()
                                if self.username else 0)
@@ -334,6 +343,8 @@ class Puppet:
             displayname=name)
 
     async def update_info(self, source: 'AbstractUser', info: User) -> None:
+        if self.disable_updates:
+            return
         changed = False
         if self.username != info.username:
             self.username = info.username
@@ -341,7 +352,7 @@ class Puppet:
 
         changed = await self.update_displayname(source, info) or changed
         if isinstance(info.photo, UserProfilePhoto):
-            changed = await self.update_avatar(source, info.photo.photo_big) or changed
+            changed = await self.update_avatar(source, info.photo) or changed
 
         self.is_bot = info.bot
 
@@ -350,33 +361,68 @@ class Puppet:
 
     async def update_displayname(self, source: 'AbstractUser', info: Union[User, UpdateUserName]
                                  ) -> bool:
-        ignore_source = (not source.is_relaybot
-                         and self.displayname_source is not None
-                         and self.displayname_source != source.tgid)
-        if ignore_source:
+        if self.disable_updates:
             return False
-        if isinstance(info, UpdateUserName):
+        allow_source = (source.is_relaybot
+                        or self.displayname_source == source.tgid
+                        # No displayname source, so just trust anything
+                        or self.displayname_source is None
+                        # No phone -> not in contact list -> can't set custom name
+                        or (isinstance(info, User) and info.phone is None))
+        if not allow_source:
+            return False
+        elif isinstance(info, UpdateUserName):
             info = await source.client.get_entity(PeerUser(self.tgid))
 
         displayname = self.get_displayname(info)
         if displayname != self.displayname:
-            await self.default_mxid_intent.set_display_name(displayname)
             self.displayname = displayname
             self.displayname_source = source.tgid
+            try:
+                await self.default_mxid_intent.set_display_name(displayname[:100])
+            except MatrixRequestError:
+                self.log.exception("Failed to set displayname")
+                self.displayname = ""
+                self.displayname_source = None
             return True
         elif source.is_relaybot or self.displayname_source is None:
             self.displayname_source = source.tgid
             return True
         return False
 
-    async def update_avatar(self, source: 'AbstractUser', photo: FileLocation) -> bool:
-        photo_id = f"{photo.volume_id}-{photo.local_id}"
+    async def update_avatar(self, source: 'AbstractUser',
+                            photo: Union[UserProfilePhoto, UserProfilePhotoEmpty]) -> bool:
+        if self.disable_updates:
+            return False
+
+        if isinstance(photo, UserProfilePhotoEmpty):
+            photo_id = ""
+        else:
+            photo_id = str(photo.photo_id)
         if self.photo_id != photo_id:
-            file = await util.transfer_file_to_matrix(source.client, self.default_mxid_intent,
-                                                      photo)
+            if not photo_id:
+                self.photo_id = ""
+                try:
+                    await self.default_mxid_intent.set_avatar("")
+                except MatrixRequestError:
+                    self.log.exception("Failed to set avatar")
+                    self.photo_id = ""
+                return True
+
+            loc = InputPeerPhotoFileLocation(
+                peer=await self.get_input_entity(source),
+                local_id=photo.photo_big.local_id,
+                volume_id=photo.photo_big.volume_id,
+                big=True
+            )
+            file = await util.transfer_file_to_matrix(source.client, self.default_mxid_intent, loc)
             if file:
-                await self.default_mxid_intent.set_avatar(file.mxc)
                 self.photo_id = photo_id
+                try:
+                    await self.default_mxid_intent.set_avatar(file.mxc)
+                except MatrixRequestError:
+                    self.log.exception("Failed to set avatar")
+                    self.photo_id = ""
                 return True
         return False
 

mautrix_telegram/scripts/telematrix_import/__main__.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by

mautrix_telegram/sqlstatestore.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by

mautrix_telegram/tgclient.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by

mautrix_telegram/user.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -65,7 +65,7 @@ class User(AbstractUser):
         self.db_portals = db_portals or []
         self._db_instance = db_instance  # type: Optional[DBUser]
 
-        self.command_status = None  # type: Dict
+        self.command_status = None  # type: Optional[Dict]
 
         (self.relaybot_whitelisted,
          self.whitelisted,
@@ -161,6 +161,12 @@ class User(AbstractUser):
     # endregion
     # region Telegram connection management
 
+    async def try_ensure_started(self) -> None:
+        try:
+            await self.ensure_started()
+        except Exception:
+            self.log.exception("Exception in ensure_started")
+
     def ensure_started(self, even_if_no_session=False) -> Awaitable['User']:
         return super().ensure_started(even_if_no_session)
 
@@ -400,9 +406,9 @@ class User(AbstractUser):
     # endregion
 
 
-def init(context: 'Context') -> List[Awaitable['User']]:
+def init(context: 'Context') -> List[Awaitable[None]]:
     global config
     config = context.config
 
     users = [User.from_db(user) for user in DBUser.all()]
-    return [user.ensure_started() for user in users if user.tgid]
+    return [user.try_ensure_started() for user in users if user.tgid]

mautrix_telegram/util/__init__.py

@@ -3,6 +3,5 @@ from .format_duration import format_duration
 from .signed_token import sign_token, verify_token
 from .recursive_dict import recursive_del, recursive_set, recursive_get
 
-
 def ignore_coro(coro):
     pass

mautrix_telegram/util/file_transfer.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -23,14 +23,16 @@ import asyncio
 import magic
 from sqlalchemy.exc import IntegrityError, InvalidRequestError
 
-from telethon.tl.types import (Document, FileLocation, InputFileLocation, InputDocumentFileLocation,
-                               TypePhotoSize, PhotoSize, PhotoCachedSize)
+from telethon.tl.types import (Document, InputFileLocation, InputDocumentFileLocation,
+                               TypePhotoSize, PhotoSize, PhotoCachedSize, InputPhotoFileLocation,
+                               InputPeerPhotoFileLocation)
 from telethon.errors import (AuthBytesInvalidError, AuthKeyInvalidError, LocationInvalidError,
-                             SecurityError)
+                             SecurityError, FileIdInvalidError)
 from mautrix_appservice import IntentAPI
 
 from ..tgclient import MautrixTelegramClient
 from ..db import TelegramFile as DBTelegramFile
+from ..util import sane_mimetypes
 
 try:
     from PIL import Image
@@ -47,7 +49,8 @@ except ImportError:
 
 log = logging.getLogger("mau.util")  # type: logging.Logger
 
-TypeLocation = Union[Document, InputDocumentFileLocation, FileLocation, InputFileLocation]
+TypeLocation = Union[Document, InputDocumentFileLocation, InputPeerPhotoFileLocation,
+                     InputFileLocation, InputPhotoFileLocation]
 
 
 def convert_image(file: bytes, source_mime: str = "image/webp", target_type: str = "png",
@@ -99,9 +102,9 @@ def _read_video_thumbnail(data: bytes, video_ext: str = "mp4", frame_ext: str =
 
 
 def _location_to_id(location: TypeLocation) -> str:
-    if isinstance(location, (Document, InputDocumentFileLocation)):
+    if isinstance(location, (Document, InputDocumentFileLocation, InputPhotoFileLocation)):
         return f"{location.id}-{location.access_hash}"
-    elif isinstance(location, (FileLocation, InputFileLocation)):
+    elif isinstance(location, (InputFileLocation, InputPeerPhotoFileLocation)):
         return f"{location.volume_id}-{location.local_id}"
 
 
@@ -119,7 +122,7 @@ async def transfer_thumbnail_to_matrix(client: MautrixTelegramClient, intent: In
     if db_file:
         return db_file
 
-    video_ext = mimetypes.guess_extension(mime)
+    video_ext = sane_mimetypes.guess_extension(mime)
     if VideoFileClip and video_ext:
         try:
             file, width, height = _read_video_thumbnail(video, video_ext, frame_ext="png")
@@ -147,9 +150,11 @@ async def transfer_thumbnail_to_matrix(client: MautrixTelegramClient, intent: In
 
 transfer_locks = {}  # type: Dict[str, asyncio.Lock]
 
+TypeThumbnail = Optional[Union[TypeLocation, TypePhotoSize]]
+
 
 async def transfer_file_to_matrix(client: MautrixTelegramClient, intent: IntentAPI,
-                                  location: TypeLocation, thumbnail: Optional[Union[TypeLocation, TypePhotoSize]] = None,
+                                  location: TypeLocation, thumbnail: TypeThumbnail = None,
                                   is_sticker: bool = False) -> Optional[DBTelegramFile]:
     location_id = _location_to_id(location)
     if not location_id:
@@ -171,15 +176,15 @@ async def transfer_file_to_matrix(client: MautrixTelegramClient, intent: IntentA
 
 async def _unlocked_transfer_file_to_matrix(client: MautrixTelegramClient, intent: IntentAPI,
                                             loc_id: str, location: TypeLocation,
-                                            thumbnail: Optional[Union[TypeLocation, TypePhotoSize]],
-                                            is_sticker: bool) -> Optional[DBTelegramFile]:
+                                            thumbnail: TypeThumbnail, is_sticker: bool
+                                            ) -> Optional[DBTelegramFile]:
     db_file = DBTelegramFile.get(loc_id)
     if db_file:
         return db_file
 
     try:
         file = await client.download_file(location)
-    except LocationInvalidError:
+    except (LocationInvalidError, FileIdInvalidError):
         return None
     except (AuthBytesInvalidError, AuthKeyInvalidError, SecurityError) as e:
         log.exception(f"{e.__class__.__name__} while downloading a file.")

mautrix_telegram/util/format_duration.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by

mautrix_telegram/util/recursive_dict.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by

mautrix_telegram/util/sane_mimetypes.py

@@ -0,0 +1,38 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+import mimetypes
+
+mimetypes.init()
+
+sanity_overrides = {
+    "image/jpeg": ".jpeg",
+    "image/tiff": ".tiff",
+    "text/plain": ".txt",
+    "text/html": ".html",
+    "audio/mpeg": ".mp3",
+    "audio/ogg": ".ogg",
+    "application/xml": ".xml",
+    "application/octet-stream": "",
+    "application/x-msdos-program": ".exe",
+}
+
+
+def guess_extension(mime: str) -> str:
+    try:
+        return sanity_overrides[mime]
+    except KeyError:
+        return mimetypes.guess_extension(mime)

mautrix_telegram/util/signed_token.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by

mautrix_telegram/web/common/auth_api.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -56,7 +56,7 @@ class AuthAPI(abc.ABC):
                                               error="You have already logged in with your Matrix "
                                                     "account.", errcode="already-logged-in")
 
-        resp = await puppet.switch_mxid(token, user.mxid)
+        resp = await puppet.switch_mxid(token.strip(), user.mxid)
         if resp == PuppetError.OnlyLoginSelf:
             return self.get_mx_login_response(status=403, errcode="only-login-self",
                                               error="You can only log in as your own Matrix user.")
@@ -72,10 +72,15 @@ class AuthAPI(abc.ABC):
                                           errcode="not-yet-implemented")
 
     async def post_login_phone(self, user: User, phone: str) -> web.Response:
+        if not phone or not phone.strip():
+            return self.get_login_response(mxid=user.mxid, state="request", status=400,
+                                           errcode="phone_number_invalid",
+                                           error="Phone number not given.")
         try:
-            await user.client.sign_in(phone or "+123")
+            await user.client.sign_in(phone.strip())
             return self.get_login_response(mxid=user.mxid, state="code", status=200,
-                                           message="Code requested successfully.")
+                                           message="Code requested successfully. Check your SMS "
+                                                   "or Telegram client and enter the code below.")
         except PhoneNumberInvalidError:
             return self.get_login_response(mxid=user.mxid, state="request", status=400,
                                            errcode="phone_number_invalid",
@@ -87,7 +92,8 @@ class AuthAPI(abc.ABC):
         except PhoneNumberAppSignupForbiddenError:
             return self.get_login_response(mxid=user.mxid, state="request", status=403,
                                            errcode="phone_number_app_signup_forbidden",
-                                           error="You have disabled 3rd party apps on your account.")
+                                           error="You have disabled 3rd party apps on your "
+                                                 "account.")
         except PhoneNumberUnoccupiedError:
             return self.get_login_response(mxid=user.mxid, state="request", status=404,
                                            errcode="phone_number_unoccupied",
@@ -116,10 +122,9 @@ class AuthAPI(abc.ABC):
         if user.command_status and user.command_status["action"] == "Login":
             user.command_status = None
 
-
     async def post_login_token(self, user: User, token: str) -> web.Response:
         try:
-            user_info = await user.client.sign_in(bot_token=token)
+            user_info = await user.client.sign_in(bot_token=token.strip())
             await self.postprocess_login(user, user_info)
             return self.get_login_response(mxid=user.mxid, state="logged-in", status=200,
                                            username=user_info.username, phone=None,
@@ -173,7 +178,7 @@ class AuthAPI(abc.ABC):
 
     async def post_login_password(self, user: User, password: str) -> web.Response:
         try:
-            user_info = await user.client.sign_in(password=password)
+            user_info = await user.client.sign_in(password=password.strip())
             await self.postprocess_login(user, user_info)
             human_tg_id = f"@{user_info.username}" if user_info.username else f"+{user_info.phone}"
             return self.get_login_response(mxid=user.mxid, state="logged-in", status=200,

mautrix_telegram/web/provisioning/__init__.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -247,7 +247,7 @@ class ProvisioningAPI(AuthAPI):
             "group": "chat",
         }[type]
 
-        portal = Portal(tgid=None, mxid=room_id, title=title, about=about, peer_type=type)
+        portal = Portal(tgid=TelegramID(0), mxid=room_id, title=title, about=about, peer_type=type)
         try:
             await portal.create_telegram_chat(user, supergroup=supergroup)
         except ValueError as e:
@@ -365,7 +365,7 @@ class ProvisioningAPI(AuthAPI):
 
     async def bridge_info(self, request: web.Request) -> web.Response:
         return web.json_response({
-            "relaybot_username": self.context.bot.username,
+            "relaybot_username": self.context.bot.username if self.context.bot is not None else None,
         }, status=200)
 
     @staticmethod

mautrix_telegram/web/public/__init__.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -87,7 +87,8 @@ class PublicBridgeWebsite(AuthAPI):
         return self.get_login_response(mxid=user.mxid, human_tg_id=user.human_tg_id)
 
     async def get_matrix_login(self, request: web.Request) -> web.Response:
-        mxid = self.verify_token(request.rel_url.query.get("token", None), endpoint="/matrix-login")
+        mxid = self.verify_token(request.rel_url.query.get("token", None),
+                                 endpoint="/matrix-login")
         if not mxid:
             return self.get_mx_login_response(status=401, state="invalid-token")
         user = User.get_by_mxid(mxid, create=False) if mxid else None
@@ -124,7 +125,8 @@ class PublicBridgeWebsite(AuthAPI):
                                                       error=error, message=message, mxid=mxid))
 
     async def post_matrix_login(self, request: web.Request) -> web.Response:
-        mxid = self.verify_token(request.rel_url.query.get("token", None), endpoint="/matrix-login")
+        mxid = self.verify_token(request.rel_url.query.get("token", None),
+                                 endpoint="/matrix-login")
         if not mxid:
             return self.get_mx_login_response(status=401, state="invalid-token")
 
@@ -167,7 +169,13 @@ class PublicBridgeWebsite(AuthAPI):
         elif "bot_token" in data:
             return await self.post_login_token(user, data["bot_token"])
         elif "code" in data:
-            resp = await self.post_login_code(user, data["code"],
+            try:
+                code = int(data["code"].strip())
+            except ValueError:
+                return self.get_login_response(mxid=user.mxid, state="code", status=400,
+                                               errcode="phone_code_invalid",
+                                               error="Phone code must be a number.")
+            resp = await self.post_login_code(user, code,
                                               password_in_data="password" in data)
             if resp or "password" not in data:
                 return resp

mautrix_telegram/web/public/login.css

@@ -1,6 +1,6 @@
 /*
  * mautrix-telegram - A Matrix-Telegram puppeting bridge
- * Copyright (C) 2018 Tulir Asokan
+ * Copyright (C) 2019 Tulir Asokan
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU Affero General Public License as published by

mautrix_telegram/web/public/login.html.mako

@@ -1,6 +1,6 @@
 <!--
 mautrix-telegram - A Matrix-Telegram puppeting bridge
-Copyright (C) 2018 Tulir Asokan
+Copyright (C) 2019 Tulir Asokan
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU Affero General Public License as published by
@@ -16,7 +16,7 @@ You should have received a copy of the GNU Affero General Public License
 along with this program.  If not, see <https://www.gnu.org/licenses/>.
 -->
 <!DOCTYPE html>
-<html>
+<html lang="en">
 <head>
 	<title>Login - Mautrix-Telegram bridge</title>
 	<link rel="icon" type="image/png" href="favicon.png"/>
@@ -25,9 +25,10 @@ along with this program.  If not, see <https://www.gnu.org/licenses/>.
 	<meta property="og:image" content="favicon.png">
 	<meta charset="utf-8">
 	<link rel="stylesheet" href="//fonts.googleapis.com/css?family=Roboto:300,700">
-	<link rel="stylesheet" href="//cdn.rawgit.com/necolas/normalize.css/master/normalize.css">
 	<link rel="stylesheet"
-		  href="//cdn.rawgit.com/milligram/milligram/master/dist/milligram.min.css">
+		  href="https://cdnjs.cloudflare.com/ajax/libs/normalize/8.0.1/normalize.min.css">
+	<link rel="stylesheet"
+		  href="https://cdnjs.cloudflare.com/ajax/libs/milligram/1.3.0/milligram.min.css">
 	<link rel="stylesheet" href="login.css"/>
 
 	<script>
@@ -94,13 +95,14 @@ along with this program.  If not, see <https://www.gnu.org/licenses/>.
 				% if state == "request":
 					<label for="value">Phone number</label>
 					<input type="tel" id="value" name="phone" placeholder="Enter phone number"/>
-					<button type="submit">Request code</button>
-					<button class="button-clear" type="button" onclick="switchToBotLogin()">
+					<button type="submit">Start</button>
+					<button class="button-clear float-right" type="button" onclick="switchToBotLogin()">
 						Use bot token
 					</button>
 				% elif state == "bot_token":
 					<label for="value">Bot token</label>
-					<input type="text" id="value" name="bot_token" placeholder="Enter bot API token"/>
+					<input type="text" id="value" name="bot_token"
+						   placeholder="Enter bot API token"/>
 					<button type="submit">Sign in</button>
 				% elif state == "code":
 					<label for="value">Phone code</label>

mautrix_telegram/web/public/matrix-login.html.mako

@@ -1,6 +1,6 @@
 <!--
 mautrix-telegram - A Matrix-Telegram puppeting bridge
-Copyright (C) 2018 Tulir Asokan
+Copyright (C) 2019 Tulir Asokan
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU Affero General Public License as published by
@@ -16,7 +16,7 @@ You should have received a copy of the GNU Affero General Public License
 along with this program.  If not, see <https://www.gnu.org/licenses/>.
 -->
 <!DOCTYPE html>
-<html>
+<html lang="en">
 <head>
 	<title>Matrix login - Mautrix-Telegram bridge</title>
 	<link rel="icon" type="image/png" href="favicon.png"/>
@@ -25,9 +25,10 @@ along with this program.  If not, see <https://www.gnu.org/licenses/>.
 	<meta property="og:image" content="favicon.png">
 	<meta charset="utf-8">
 	<link rel="stylesheet" href="//fonts.googleapis.com/css?family=Roboto:300,700">
-	<link rel="stylesheet" href="//cdn.rawgit.com/necolas/normalize.css/master/normalize.css">
 	<link rel="stylesheet"
-		  href="//cdn.rawgit.com/milligram/milligram/master/dist/milligram.min.css">
+		  href="https://cdnjs.cloudflare.com/ajax/libs/normalize/8.0.1/normalize.min.css">
+	<link rel="stylesheet"
+		  href="https://cdnjs.cloudflare.com/ajax/libs/milligram/1.3.0/milligram.min.css">
 	<link rel="stylesheet" href="login.css"/>
 </head>
 <body>

optional-requirements.txt

@@ -1,4 +1,4 @@
-lxml
 cryptg
 Pillow
 moviepy
+prometheus-client

setup.py

@@ -3,10 +3,10 @@ import glob
 import mautrix_telegram
 
 extras = {
-    "highlight_edits": ["lxml>=4.1.1,<5"],
-    "fast_crypto": ["cryptg>=0.1,<0.2"],
-    "webp_convert": ["Pillow>=4.3.0,<6"],
+    "fast_crypto": ["cryptg>=0.1,<0.3"],
+    "webp_convert": ["Pillow>=4.3.0,<7"],
     "hq_thumbnails": ["moviepy>=1.0,<2.0"],
+    "metrics": ["prometheus-client>=0.6.0,<0.8.0"],
 }
 extras["all"] = list({dep for deps in extras.values() for dep in deps})
 
@@ -31,15 +31,15 @@ setuptools.setup(
 
     install_requires=[
         "aiohttp>=3.0.1,<4",
-        "mautrix-appservice>=0.3.8,<0.4.0",
+        "mautrix-appservice>=0.3.11,<0.4.0",
         "SQLAlchemy>=1.2.3,<2",
         "alembic>=1.0.0,<2",
         "commonmark>=0.8.1,<1",
         "ruamel.yaml>=0.15.35,<0.16",
         "future-fstrings>=0.4.2",
         "python-magic>=0.4.15,<0.5",
-        "telethon>=1.5.5,<1.7",
-        "telethon-session-sqlalchemy>=0.2.11,<0.3",
+        "telethon>=1.9,<1.10",
+        "telethon-session-sqlalchemy>=0.2.14,<0.3",
     ],
     extras_require=extras,