Bump version to 0.6.1rc2

Users who the bridge only saw via logged in users with the target user
in their contact lists wouldn't get their displayname updated due to an
invalid condition in the update_displayname function.

.gitignore

@@ -1,11 +1,17 @@
 .idea/
 
 .venv
+env/
 pip-selfcheck.json
 *.pyc
 __pycache__
+build
+dist
+*.egg-info
+.eggs
 
 config.yaml
 registration.yaml
 *.log*
 *.db
+*.bak

.gitlab-ci.yml

@@ -0,0 +1,38 @@
+image: docker:stable
+
+stages:
+- build
+- push
+
+default:
+  before_script:
+  - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
+
+build:
+  stage: build
+  script:
+  - docker pull $CI_REGISTRY_IMAGE:latest || true
+  - docker build --pull --cache-from $CI_REGISTRY_IMAGE:latest --tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA .
+  - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA
+
+push latest:
+  stage: push
+  only:
+  - master
+  variables:
+    GIT_STRATEGY: none
+  script:
+  - docker pull $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA
+  - docker tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA $CI_REGISTRY_IMAGE:latest
+  - docker push $CI_REGISTRY_IMAGE:latest
+
+push tag:
+  stage: push
+  variables:
+    GIT_STRATEGY: none
+  except:
+  - master
+  script:
+  - docker pull $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA
+  - docker tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME
+  - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME

Dockerfile

@@ -1,4 +1,4 @@
-FROM docker.io/alpine:3.9
+FROM docker.io/alpine:3.10
 
 ENV UID=1337 \
     GID=1337 \
@@ -10,25 +10,29 @@ RUN apk add --no-cache \
       py3-virtualenv \
       py3-pillow \
       py3-aiohttp \
-      py3-lxml \
       py3-magic \
       py3-sqlalchemy \
       py3-markdown \
       py3-psycopg2 \
+      py3-ruamel.yaml \
       # Indirect dependencies
-      py3-numpy \
-      py3-asn1crypto \
-      py3-future \
-      py3-markupsafe \
-      py3-mako \
-      py3-decorator \
-      py3-dateutil \
-      py3-idna \
-      py3-six \
-      py3-asn1 \
-      py3-rsa \
+      #commonmark
+        py3-future \
+      #alembic
+        py3-mako \
+        py3-dateutil \
+        py3-markupsafe \
+      #moviepy
+        py3-decorator \
+        #py3-tqdm \
+        py3-requests \
+        #imageio
+          py3-numpy \
+      #telethon
+        py3-rsa \
       # Other dependencies
       python3-dev \
+      libffi-dev \
       build-base \
       ffmpeg \
       ca-certificates \

ROADMAP.md

@@ -3,6 +3,7 @@
 * Matrix → Telegram
   * [x] Message content (text, formatting, files, etc..)
   * [x] Message redactions
+  * [x] Message edits
   * [ ] ‡ Message history
   * [x] Presence
   * [x] Typing notifications
@@ -21,6 +22,10 @@
     * [ ] ‡ Changes to displayname/avatar
 * Telegram → Matrix
   * [x] Message content (text, formatting, files, etc..)
+  * [ ] Advanced message content/media
+    * [x] Polls
+	* [x] Games
+	* [ ] Buttons
   * [x] Message deletions
   * [x] Message edits
   * [ ] Message history
@@ -48,6 +53,8 @@
   * [x] Option to use bot to relay messages for unauthenticated Matrix users
   * [x] Option to use own Matrix account for messages sent from other Telegram clients
   * [ ] ‡ Calls (hard, not yet supported by Telethon)
+  * [ ] ‡ Secret chats (not yet supported by Telethon)
+  * [ ] ‡ E2EE in Matrix rooms (not yet supported 
 
 † Information not automatically sent from source, i.e. implementation may not be possible  
 ‡ Maybe, i.e. this feature may or may not be implemented at some point

alembic/versions/17574c57f3f8_add_disable_updates_field_for_puppets.py

@@ -0,0 +1,27 @@
+"""Add disable_updates field for puppets
+
+Revision ID: 17574c57f3f8
+Revises: a9119be92164
+Create Date: 2019-05-15 00:24:46.967529
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '17574c57f3f8'
+down_revision = 'a9119be92164'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    with op.batch_alter_table("puppet") as batch_op:
+        batch_op.add_column(sa.Column("disable_updates", sa.Boolean(), nullable=False,
+                                      server_default=sa.sql.expression.false()))
+
+
+def downgrade():
+    with op.batch_alter_table("puppet") as batch_op:
+        batch_op.drop_column("disable_updates")

alembic/versions/1fa46383a9d3_add_is_bot_field_to_puppets.py

@@ -17,7 +17,8 @@ depends_on = None
 
 
 def upgrade():
-    op.add_column('puppet', sa.Column('is_bot', sa.Boolean(), nullable=True))
+    with op.batch_alter_table("puppet") as batch_op:
+        batch_op.add_column(sa.Column('is_bot', sa.Boolean(), nullable=True))
 
 
 def downgrade():

alembic/versions/30eca60587f1_add_megagroup_field_to_portals.py

@@ -16,7 +16,8 @@ depends_on = None
 
 
 def upgrade():
-    op.add_column('portal', sa.Column('megagroup', sa.Boolean()))
+    with op.batch_alter_table("portal") as batch_op:
+        batch_op.add_column(sa.Column('megagroup', sa.Boolean()))
 
 
 def downgrade():

alembic/versions/6ca3d74d51e4_move_state_store_to_main_database.py

@@ -57,7 +57,8 @@ class Puppet(Base):
 
 
 def upgrade():
-    op.add_column("puppet", sa.Column("matrix_registered", sa.Boolean(), nullable=False,
+    with op.batch_alter_table("puppet") as batch_op:
+        batch_op.add_column(sa.Column("matrix_registered", sa.Boolean(), nullable=False,
                                       server_default=sa.sql.expression.false()))
     op.create_table("mx_room_state",
                     sa.Column("room_id", sa.String(), nullable=False),

alembic/versions/9e9c89b0b877_add_edit_index_to_messages.py

@@ -0,0 +1,48 @@
+"""Add edit index to messages
+
+Revision ID: 9e9c89b0b877
+Revises: 17574c57f3f8
+Create Date: 2019-05-29 15:28:23.128377
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = '9e9c89b0b877'
+down_revision = '17574c57f3f8'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table('_message_temp',
+                    sa.Column('mxid', sa.String),
+                    sa.Column('mx_room', sa.String),
+                    sa.Column('tgid', sa.Integer),
+                    sa.Column('tg_space', sa.Integer),
+                    sa.Column('edit_index', sa.Integer),
+                    sa.PrimaryKeyConstraint('tgid', 'tg_space', 'edit_index'),
+                    sa.UniqueConstraint("mxid", "mx_room", "tg_space", name="_mx_id_room_2"))
+    c = op.get_bind()
+    c.execute("INSERT INTO _message_temp (mxid, mx_room, tgid, tg_space, edit_index) "
+              "SELECT message.mxid, message.mx_room, message.tgid, message.tg_space, 0 "
+              "FROM message")
+    c.execute("DROP TABLE message")
+    c.execute("ALTER TABLE _message_temp RENAME TO message")
+
+
+def downgrade():
+    op.create_table('_message_temp',
+                    sa.Column('mxid', sa.String),
+                    sa.Column('mx_room', sa.String),
+                    sa.Column('tgid', sa.Integer),
+                    sa.Column('tg_space', sa.Integer),
+                    sa.PrimaryKeyConstraint('tgid', 'tg_space'),
+                    sa.UniqueConstraint("mxid", "mx_room", "tg_space", name="_mx_id_room"))
+    c = op.get_bind()
+    c.execute("INSERT INTO _message_temp (mxid, mx_room, tgid, tg_space) "
+              "SELECT message.mxid, message.mx_room, message.tgid, message.tg_space "
+              "FROM message")
+    c.execute("DROP TABLE message")
+    c.execute("ALTER TABLE _message_temp RENAME TO message")

example-config.yaml

@@ -60,10 +60,19 @@ appservice:
     bot_displayname: Telegram bridge bot
     bot_avatar: mxc://maunium.net/tJCRmUyJDsgRNgqhOgoiHWbX
 
+    # Community ID for bridged users (changes registration file) and rooms.
+    # Must be created manually.
+    community_id: false
+
     # Authentication tokens for AS <-> HS communication. Autogenerated; do not modify.
     as_token: "This value is generated when generating the registration"
     hs_token: "This value is generated when generating the registration"
 
+# Prometheus telemetry config. Requires prometheus-client to be installed.
+metrics:
+    enabled: false
+    listen_port: 8000
+
 # Bridge config
 bridge:
     # Localpart template of MXIDs for Telegram users.
@@ -126,15 +135,11 @@ bridge:
     # Only enable this if your displayname_template has some static part that the bridge can use to
     # reliably identify what is a plaintext highlight.
     plaintext_highlights: false
-    # Show message editing as a reply to the original message.
-    # If this is false, message edits are not shown at all, as Matrix does not support editing yet.
-    edits_as_replies: true
-    # Highlight changed/added parts in edits. Requires lxml.
-    highlight_edits: false
     # Whether or not to make portals of publicly joinable channels/supergroups publicly joinable on Matrix.
     public_portals: true
     # Whether or not to fetch and handle Telegram updates at startup from the time the bridge was down.
     # Currently only works for private chats and normal groups.
+    # WARNING: This feature seems to be broken in the telegram library.
     catch_up: false
     # Whether or not to use /sync to get presence, read receipts and typing notifications when using
     # your own Matrix account as the Matrix puppet for your Telegram account.
@@ -144,6 +149,10 @@ bridge:
     # Use inline images instead of a separate message for the caption.
     # N.B. Inline images are not supported on all clients (e.g. Riot iOS).
     inline_images: false
+    # Maximum size of image in megabytes before sending to Telegram as a document.
+    image_as_file_size: 10
+    # Maximum size of Telegram documents in megabytes to bridge.
+    max_document_size: 100
 
     # Whether to bridge Telegram bot messages as m.notices or m.texts.
     bot_messages_as_notices: true
@@ -249,6 +258,40 @@ telegram:
     api_hash: tjyd5yge35lbodk1xwzw2jstp90k55qz
     # (Optional) Create your own bot at https://t.me/BotFather
     bot_token: disabled
+
+    # Telethon connection options.
+    connection:
+        # The timeout in seconds to be used when connecting.
+        timeout: 120
+        # How many times the reconnection should retry, either on the initial connection or when
+        # Telegram disconnects us. May be set to a negative or null value for infinite retries, but
+        # this is not recommended, since the program can get stuck in an infinite loop.
+        retries: 5
+        # The delay in seconds to sleep between automatic reconnections.
+        retry_delay: 1
+        # The threshold below which the library should automatically sleep on flood wait errors
+        # (inclusive). For instance, if a FloodWaitError for 17s occurs and flood_sleep_threshold
+        # is 20s, the library will sleep automatically. If the error was for 21s, it would raise
+        # the error instead. Values larger than a day (86400) will be changed to a day.
+        flood_sleep_threshold: 60
+        # How many times a request should be retried. Request are retried when Telegram is having
+        # internal issues, when there is a FloodWaitError less than flood_sleep_threshold, or when
+        # there's a migrate error. May take a negative or null value for infinite retries, but this
+        # is not recommended, since some requests can always trigger a call fail (such as searching
+        # for messages).
+        request_retries: 5
+
+    # Device info sent to Telegram.
+    device_info:
+        # "auto" = OS name+version.
+        device_model: auto
+        # "auto" = Telethon version.
+        system_version: auto
+        # "auto" = mautrix-telegram version.
+        app_version: auto
+        lang_code: en
+        system_lang_code: en
+
     # Custom server to connect to.
     server:
         # Set to true to use these server settings. If false, will automatically
@@ -260,6 +303,7 @@ telegram:
         ip: 149.154.167.40
         # The port to connect to. 443 may not work, 80 is better and both are equally secure.
         port: 80
+
     # Telethon proxy configuration.
     # You must install PySocks from pip for proxies to work.
     proxy:

mautrix_telegram/__init__.py

@@ -1,2 +1,2 @@
-__version__ = "0.5.0rc2"
+__version__ = "0.6.1rc2"
 __author__ = "Tulir Asokan <tulir@maunium.net>"

mautrix_telegram/__main__.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -22,8 +22,8 @@ import logging.config
 import sys
 import copy
 import signal
+import os
 
-from sqlalchemy import orm
 import sqlalchemy as sql
 
 from mautrix_appservice import AppService
@@ -44,6 +44,11 @@ from .sqlstatestore import SQLStateStore
 from .user import User, init as init_user
 from . import __version__
 
+try:
+    import prometheus_client as prometheus
+except ImportError:
+    prometheus = None
+
 parser = argparse.ArgumentParser(
     description="A Matrix-Telegram puppeting bridge.",
     prog="python -m mautrix-telegram")
@@ -58,7 +63,7 @@ parser.add_argument("-r", "--registration", type=str, default="registration.yaml
                     metavar="<path>", help="the path to save the generated registration to")
 args = parser.parse_args()
 
-config = Config(args.config, args.registration, args.base_config)
+config = Config(args.config, args.registration, args.base_config, os.environ)
 config.load()
 config.update()
 
@@ -73,15 +78,20 @@ log = logging.getLogger("mau.init")  # type: logging.Logger
 log.debug(f"Initializing mautrix-telegram {__version__}")
 
 db_engine = sql.create_engine(config["appservice.database"] or "sqlite:///mautrix-telegram.db")
-db_factory = orm.sessionmaker(bind=db_engine)
-db_session = orm.scoping.scoped_session(db_factory)
 Base.metadata.bind = db_engine
 
-session_container = AlchemySessionContainer(engine=db_engine, session=db_session,
-                                            table_base=Base, table_prefix="telethon_",
-                                            manage_tables=False)
+session_container = AlchemySessionContainer(engine=db_engine, table_base=Base, session=False,
+                                            table_prefix="telethon_", manage_tables=False)
 session_container.core_mode = True
 
+try:
+    import uvloop
+
+    asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
+    log.debug("Using uvloop for asyncio")
+except ImportError:
+    pass
+
 loop = asyncio.get_event_loop()  # type: asyncio.AbstractEventLoop
 
 state_store = SQLStateStore()
@@ -94,8 +104,8 @@ appserv = AppService(config["homeserver.address"], config["homeserver.domain"],
                      aiohttp_params={
                          "client_max_size": config["appservice.max_body_size"] * mebibyte
                      })
-
-context = Context(appserv, db_session, config, loop, session_container)
+bot = init_bot(config)
+context = Context(appserv, config, loop, session_container, bot)
 
 if config["appservice.public.enabled"]:
     public_website = PublicBridgeWebsite(loop)
@@ -108,12 +118,18 @@ if config["appservice.provisioning.enabled"]:
                            provisioning_api.app)
     context.provisioning_api = provisioning_api
 
+context.mx = MatrixHandler(context)
+
+if config["metrics.enabled"]:
+    if prometheus:
+        prometheus.start_http_server(config["metrics.listen_port"])
+    else:
+        log.warn("Metrics are enabled in the config, but prometheus-async is not installed.")
+
 with appserv.run(config["appservice.hostname"], config["appservice.port"]) as start:
     start_ts = time()
     init_db(db_engine)
     init_abstract_user(context)
-    context.bot = init_bot(context)
-    context.mx = MatrixHandler(context)
     init_formatter(context)
     init_portal(context)
     startup_actions = (init_puppet(context) +
@@ -142,3 +158,6 @@ with appserv.run(config["appservice.hostname"], config["appservice.port"]) as st
             asyncio.gather(*[user.stop() for user in User.by_tgid.values()], loop=loop))
         log.debug("Clients stopped, shutting down")
         sys.exit(0)
+    except Exception as e:
+        log.exception("Unexpected error")
+        sys.exit(1)

mautrix_telegram/abstract_user.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -14,13 +14,13 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Tuple, Optional, List, Union, TYPE_CHECKING
+from typing import Tuple, Optional, List, Union, Dict, TYPE_CHECKING
 from abc import ABC, abstractmethod
 import asyncio
 import logging
 import platform
+import time
 
-from sqlalchemy import orm
 from telethon.tl.patched import MessageService, Message
 from telethon.tl.types import (
     Channel, ChannelForbidden, Chat, ChatForbidden, MessageActionChannelMigrateFrom, PeerUser,
@@ -51,12 +51,19 @@ UpdateMessage = Union[UpdateShortChatMessage, UpdateShortMessage, UpdateNewChann
                       UpdateNewMessage, UpdateEditMessage, UpdateEditChannelMessage]
 UpdateMessageContent = Union[UpdateShortMessage, UpdateShortChatMessage, Message, MessageService]
 
+try:
+    from prometheus_client import Histogram
+
+    UPDATE_TIME = Histogram("telegram_update", "Time spent processing Telegram updates",
+                            ["update_type"])
+except ImportError:
+    Histogram = None
+    UPDATE_TIME = None
 
 class AbstractUser(ABC):
     session_container = None  # type: AlchemySessionContainer
     loop = None  # type: asyncio.AbstractEventLoop
     log = None  # type: logging.Logger
-    db = None  # type: orm.Session
     az = None  # type: AppService
     bot = None  # type: Bot
     ignore_incoming_bot_events = True  # type: bool
@@ -97,27 +104,43 @@ class AbstractUser(ABC):
 
     def _init_client(self) -> None:
         self.log.debug(f"Initializing client for {self.name}")
-        device = f"{platform.system()} {platform.release()}"
-        sysversion = MautrixTelegramClient.__version__
+
         self.session = self.session_container.new_session(self.name)
         if config["telegram.server.enabled"]:
             self.session.set_dc(config["telegram.server.dc"],
                                 config["telegram.server.ip"],
                                 config["telegram.server.port"])
+
         if self.is_relaybot:
             base_logger = logging.getLogger("telethon.relaybot")
         else:
             base_logger = logging.getLogger(f"telethon.{self.tgid or -hash(self.mxid)}")
-        self.client = MautrixTelegramClient(session=self.session,
-                                            api_id=config["telegram.api_id"],
-                                            api_hash=config["telegram.api_hash"],
-                                            loop=self.loop,
-                                            app_version=__version__,
-                                            system_version=sysversion,
-                                            device_model=device,
-                                            timeout=120,
-                                            base_logger=base_logger,
-                                            proxy=self._proxy_settings)
+
+        device = config["telegram.device_info.device_model"]
+        sysversion = config["telegram.device_info.system_version"]
+        appversion = config["telegram.device_info.app_version"]
+
+        self.client = MautrixTelegramClient(
+            session=self.session,
+
+            api_id=config["telegram.api_id"],
+            api_hash=config["telegram.api_hash"],
+
+            app_version=__version__ if appversion == "auto" else appversion,
+            system_version=MautrixTelegramClient.__version__ if sysversion == "auto" else sysversion,
+            device_model=f"{platform.system()} {platform.release()}" if device == "auto" else device,
+
+            timeout=config["telegram.connection.timeout"],
+            connection_retries=config["telegram.connection.retries"],
+            retry_delay=config["telegram.connection.retry_delay"],
+            flood_sleep_threshold=config["telegram.connection.flood_sleep_threshold"],
+            request_retries=config["telegram.connection.request_retries"],
+
+            proxy=self._proxy_settings,
+
+            loop=self.loop,
+            base_logger=base_logger
+        )
         self.client.add_event_handler(self._update_catch)
 
     @abstractmethod
@@ -137,11 +160,14 @@ class AbstractUser(ABC):
         raise NotImplementedError()
 
     async def _update_catch(self, update: TypeUpdate) -> None:
+        start_time = time.time()
         try:
             if not await self.update(update):
                 await self._update(update)
         except Exception:
             self.log.exception("Failed to handle Telegram update")
+        if UPDATE_TIME:
+            UPDATE_TIME.labels(update_type=type(update).__name__).observe(time.time() - start_time)
 
     async def get_dialogs(self, limit: int = None) -> List[Union[Chat, Channel]]:
         if self.is_bot:
@@ -175,11 +201,8 @@ class AbstractUser(ABC):
     async def ensure_started(self, even_if_no_session=False) -> 'AbstractUser':
         if not self.puppet_whitelisted or self.connected:
             return self
-        session_count = self.session_container.Session.query.filter(
-            self.session_container.Session.session_id == self.mxid).count()
-        self.log.debug("ensure_started(%s, even_if_no_session=%s, session_count=%s)",
-                       self.mxid, even_if_no_session, session_count)
-        if even_if_no_session or session_count > 0:
+        self.log.debug("ensure_started(%s, even_if_no_session=%s)", self.mxid, even_if_no_session)
+        if even_if_no_session or self.session_container.has_session(self.mxid):
             await self.start(delete_unless_authenticated=not even_if_no_session)
         return self
 
@@ -190,6 +213,8 @@ class AbstractUser(ABC):
     # region Telegram update handling
 
     async def _update(self, update: TypeUpdate) -> None:
+        asyncio.ensure_future(self._handle_entity_updates(getattr(update, "_entities", {})),
+                              loop=self.loop)
         if isinstance(update, (UpdateShortChatMessage, UpdateShortMessage, UpdateNewChannelMessage,
                                UpdateNewMessage, UpdateEditMessage, UpdateEditChannelMessage)):
             await self.update_message(update)
@@ -239,7 +264,7 @@ class AbstractUser(ABC):
             return
 
         # We check that these are user read receipts, so tg_space is always the user ID.
-        message = DBMessage.get_by_tgid(TelegramID(update.max_id), self.tgid)
+        message = DBMessage.get_one_by_tgid(TelegramID(update.max_id), self.tgid, edit_index=-1)
         if not message:
             return
 
@@ -266,6 +291,16 @@ class AbstractUser(ABC):
         sender = pu.Puppet.get(TelegramID(update.user_id))
         await portal.handle_telegram_typing(sender, update)
 
+    async def _handle_entity_updates(self, entities: Dict[int, Union[User, Chat, Channel]]
+                                     ) -> None:
+        try:
+            users = (entity for entity in entities.values() if isinstance(entity, User))
+            puppets = ((pu.Puppet.get(TelegramID(user.id)), user) for user in users)
+            await asyncio.gather(*[puppet.update_info(self, info)
+                                   for puppet, info in puppets if puppet])
+        except Exception:
+            self.log.exception("Failed to handle entity updates")
+
     async def update_others_info(self, update: Union[UpdateUserName, UpdateUserPhoto]) -> None:
         # TODO duplication not checked
         puppet = pu.Puppet.get(TelegramID(update.user_id))
@@ -274,7 +309,7 @@ class AbstractUser(ABC):
             if await puppet.update_displayname(self, update):
                 puppet.save()
         elif isinstance(update, UpdateUserPhoto):
-            if await puppet.update_avatar(self, update.photo.photo_big):
+            if await puppet.update_avatar(self, update.photo):
                 puppet.save()
         else:
             self.log.warning("Unexpected other user info update: %s", update)
@@ -314,7 +349,8 @@ class AbstractUser(ABC):
         return update, sender, portal
 
     @staticmethod
-    async def _try_redact(portal: po.Portal, message: DBMessage) -> None:
+    async def _try_redact(message: DBMessage) -> None:
+        portal = po.Portal.get_by_mxid(message.mx_room)
         if not portal:
             return
         try:
@@ -326,30 +362,26 @@ class AbstractUser(ABC):
         if len(update.messages) > MAX_DELETIONS:
             return
 
-        for message in update.messages:
-            message = DBMessage.get_by_tgid(TelegramID(message), self.tgid)
-            if not message:
-                continue
-            message.delete()
-            number_left = DBMessage.count_spaces_by_mxid(message.mxid, message.mx_room)
-            if number_left == 0:
-                portal = po.Portal.get_by_mxid(message.mx_room)
-                await self._try_redact(portal, message)
+        for message_id in update.messages:
+            messages = DBMessage.get_all_by_tgid(TelegramID(message_id), self.tgid)
+            for message in messages:
+                message.delete()
+                number_left = DBMessage.count_spaces_by_mxid(message.mxid, message.mx_room)
+                if number_left == 0:
+                    portal = po.Portal.get_by_mxid(message.mx_room)
+                    await self._try_redact(message)
 
     async def delete_channel_message(self, update: UpdateDeleteChannelMessages) -> None:
         if len(update.messages) > MAX_DELETIONS:
             return
 
-        portal = po.Portal.get_by_tgid(TelegramID(update.channel_id))
-        if not portal:
-            return
+        channel_id = TelegramID(update.channel_id)
 
-        for message in update.messages:
-            message = DBMessage.get_by_tgid(TelegramID(message), portal.tgid)
-            if not message:
-                continue
-            message.delete()
-            await self._try_redact(portal, message)
+        for message_id in update.messages:
+            messages = DBMessage.get_all_by_tgid(TelegramID(message_id), channel_id)
+            for message in messages:
+                message.delete()
+                await self._try_redact(message)
 
     async def update_message(self, original_update: UpdateMessage) -> None:
         update, sender, portal = self.get_message_details(original_update)
@@ -375,10 +407,7 @@ class AbstractUser(ABC):
 
         user = sender.tgid if sender else "admin"
         if isinstance(original_update, (UpdateEditMessage, UpdateEditChannelMessage)):
-            if config["bridge.edits_as_replies"]:
-                self.log.debug("Handling edit %s to %s by %s", update, portal.tgid_log, user)
-                return await portal.handle_telegram_edit(self, sender, update)
-            return
+            return await portal.handle_telegram_edit(self, sender, update)
 
         self.log.debug("Handling message %s to %s by %s", update, portal.tgid_log, user)
         return await portal.handle_telegram_message(self, sender, update)
@@ -388,7 +417,7 @@ class AbstractUser(ABC):
 
 def init(context: "Context") -> None:
     global config, MAX_DELETIONS
-    AbstractUser.az, AbstractUser.db, config, AbstractUser.loop, AbstractUser.relaybot = context.core
+    AbstractUser.az, config, AbstractUser.loop, AbstractUser.relaybot = context.core
     AbstractUser.ignore_incoming_bot_events = config["bridge.relaybot.ignore_own_incoming_events"]
     AbstractUser.session_container = context.session_container
     MAX_DELETIONS = config.get("bridge.max_telegram_delete", 10)

mautrix_telegram/bot.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -14,7 +14,7 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Awaitable, Callable, Dict, List, Optional, Pattern, TYPE_CHECKING
+from typing import Awaitable, Callable, Dict, List, Optional, Pattern, Tuple, TYPE_CHECKING
 import logging
 import re
 
@@ -23,7 +23,7 @@ from telethon.tl.types import (
     ChannelParticipantAdmin, ChannelParticipantCreator, ChatForbidden, ChatParticipantAdmin,
     ChatParticipantCreator, InputChannel, InputUser, MessageActionChatAddUser,
     MessageActionChatDeleteUser, MessageEntityBotCommand, PeerChannel, PeerChat, TypePeer,
-    UpdateNewChannelMessage, UpdateNewMessage, MessageActionChatMigrateTo)
+    UpdateNewChannelMessage, UpdateNewMessage, MessageActionChatMigrateTo, User)
 from telethon.tl.functions.messages import GetChatsRequest, GetFullChatRequest
 from telethon.tl.functions.channels import GetChannelsRequest, GetParticipantRequest
 from telethon.errors import ChannelInvalidError, ChannelPrivateError
@@ -56,10 +56,18 @@ class Bot(AbstractUser):
         self.username = None  # type: str
         self.is_relaybot = True  # type: bool
         self.is_bot = True  # type: bool
-        self.chats = {chat.id: chat.type for chat in BotChat.all()}  # type: Dict[int, str]
+        self.chats = {}  # type: Dict[int, str]
         self.tg_whitelist = []  # type: List[int]
         self.whitelist_group_admins = (config["bridge.relaybot.whitelist_group_admins"]
                                        or False)  # type: bool
+        self._me_info = None  # type: Optional[User]
+        self._me_mxid = None  # type: Optional[MatrixUserID]
+
+    async def get_me(self, use_cache: bool = True) -> Tuple[User, MatrixUserID]:
+        if not use_cache or not self._me_mxid:
+            self._me_info = await self.client.get_me()
+            self._me_mxid = pu.Puppet.get_mxid_from_id(TelegramID(self._me_info.id))
+        return self._me_info, self._me_mxid
 
     async def init_permissions(self) -> None:
         whitelist = config["bridge.relaybot.whitelist"] or []
@@ -74,6 +82,7 @@ class Bot(AbstractUser):
                 self.tg_whitelist.append(user_id)
 
     async def start(self, delete_unless_authenticated: bool = False) -> 'Bot':
+        self.chats = {chat.id: chat.type for chat in BotChat.all()}
         await super().start(delete_unless_authenticated)
         if not await self.is_logged_in():
             await self.client.sign_in(bot_token=self.token)
@@ -280,9 +289,9 @@ class Bot(AbstractUser):
         return "bot"
 
 
-def init(context: 'Context') -> Optional[Bot]:
+def init(cfg: 'Config') -> Optional[Bot]:
     global config
-    config = context.config
+    config = cfg
     token = config["telegram.bot_token"]
     if token and not token.lower().startswith("disable"):
         return Bot(token)

mautrix_telegram/commands/clean_rooms.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by

mautrix_telegram/commands/handler.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -14,15 +14,16 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
+"""This module contains classes handling commands issued by Matrix users."""
 from typing import Awaitable, Callable, Dict, List, NamedTuple, Optional
-import traceback
 import logging
+import traceback
 
 import commonmark
 
 from telethon.errors import FloodWaitError
 
-from ..types import MatrixRoomID
+from ..types import MatrixRoomID, MatrixEventID
 from ..util import format_duration
 from .. import user as u, context as c
 
@@ -59,9 +60,32 @@ md_parser = commonmark.Parser()
 md_renderer = HtmlEscapingRenderer()
 
 
+def ensure_trailing_newline(s: str) -> str:
+    """Returns the passed string, but with a guaranteed trailing newline."""
+    return s + ("" if s[-1] == "\n" else "\n")
+
+
 class CommandEvent:
-    def __init__(self, processor: 'CommandProcessor', room: MatrixRoomID, sender: u.User,
-                 command: str, args: List[str], is_management: bool, is_portal: bool) -> None:
+    """Holds information about a command issued in a Matrix room.
+
+    When a Matrix command was issued to the bot, CommandEvent will hold
+    information regarding the event.
+
+    Attributes:
+        room_id: The id of the Matrix room in which the command was issued.
+        event_id: The id of the matrix event which contained the command.
+        sender: The user who issued the command.
+        command: The issued command.
+        args: Arguments given with the issued command.
+        is_management: Determines whether the room in which the command wa
+            issued is a management room.
+        is_portal: Determines whether the room in which the command was issued
+            is a portal.
+    """
+
+    def __init__(self, processor: 'CommandProcessor', room: MatrixRoomID, event: MatrixEventID,
+                 sender: u.User, command: str, args: List[str], is_management: bool,
+                 is_portal: bool) -> None:
         self.az = processor.az
         self.log = processor.log
         self.loop = processor.loop
@@ -70,6 +94,7 @@ class CommandEvent:
         self.public_website = processor.public_website
         self.command_prefix = processor.command_prefix
         self.room_id = room
+        self.event_id = event
         self.sender = sender
         self.command = command
         self.args = args
@@ -78,23 +103,102 @@ class CommandEvent:
 
     def reply(self, message: str, allow_html: bool = False, render_markdown: bool = True
               ) -> Awaitable[Dict]:
-        message = message.replace("$cmdprefix+sp ",
-                                  "" if self.is_management else f"{self.command_prefix} ")
-        message = message.replace("$cmdprefix", self.command_prefix)
-        html = None
+        """Write a reply to the room in which the command was issued.
+
+        Replaces occurences of "$cmdprefix" in the message with the command
+        prefix and replaces occurences of "$cmdprefix+sp " with the command
+        prefix if the command was not issued in a management room.
+        If allow_html and render_markdown are both False, the message will not
+        be rendered to html and sending of html is disabled.
+
+        Args:
+            message: The message to post in the room.
+            allow_html: Escape html in the message or don't render html at all
+                if markdown is disabled.
+            render_markdown: Use markdown formatting to render the passed
+                message to html.
+
+        Returns:
+            Handler for the message sending function.
+        """
+        message_cmd = self._replace_command_prefix(message)
+        html = self._render_message(message_cmd, allow_html=allow_html,
+                                    render_markdown=render_markdown)
+
+        return self.az.intent.send_notice(self.room_id, message_cmd, html=html)
+
+    def mark_read(self) -> Awaitable[Dict]:
+        """Marks the command as read by the bot."""
+        return self.az.intent.mark_read(self.room_id, self.event_id)
+
+    def _replace_command_prefix(self, message: str) -> str:
+        """Returns the string with the proper command prefix entered."""
+        message = message.replace(
+            "$cmdprefix+sp ", "" if self.is_management else f"{self.command_prefix} "
+        )
+        return message.replace("$cmdprefix", self.command_prefix)
+
+    @staticmethod
+    def _render_message(message: str, allow_html: bool, render_markdown: bool) -> Optional[str]:
+        """Renders the message as HTML.
+
+        Args:
+            allow_html: Flag to allow custom HTML in the message.
+            render_markdown: If true, markdown styling is applied to the message.
+
+        Returns:
+            The message rendered as HTML.
+            None is returned if no styled output is required.
+        """
+        html = ""
         if render_markdown:
             md_renderer.allow_html = allow_html
             html = md_renderer.render(md_parser.parse(message))
         elif allow_html:
             html = message
-        return self.az.intent.send_notice(self.room_id, message, html=html)
+        return ensure_trailing_newline(html) if html else None
 
 
 class CommandHandler:
+    """A command which can be executed from a Matrix room.
+
+    The command manages its permission and help texts.
+    When called, it will check the permission of the command event and execute
+    the command or, in case of error, report back to the user.
+
+    Attributes:
+        needs_auth: Flag indicating if the sender is required to be logged in.
+        needs_puppeting: Flag indicating if the sender is required to use
+            Telegram puppeteering for this command.
+        needs_matrix_puppeting: Flag indicating if the sender is required to use
+            Matrix pupeteering.
+        needs_admin: Flag for whether only admin users can issue this command.
+        management_only: Whether the command can exclusively be issued in a
+            management room.
+        name: The name of this command.
+        help_section: Section of the help in which this command will appear.
+    """
+
     def __init__(self, handler: Callable[[CommandEvent], Awaitable[Dict]], needs_auth: bool,
                  needs_puppeting: bool, needs_matrix_puppeting: bool, needs_admin: bool,
                  management_only: bool, name: str, help_text: str, help_args: str,
                  help_section: HelpSection) -> None:
+        """
+        Args:
+            handler: The function handling the execution of this command.
+            needs_auth: Flag indicating if the sender is required to be logged in.
+            needs_puppeting: Flag indicating if the sender is required to use
+                Telegram puppeteering for this command.
+            needs_matrix_puppeting: Flag indicating if the sender is required to
+                use Matrix pupeteering.
+            needs_admin: Flag for whether only admin users can issue this command.
+            management_only: Whether the command can exclusively be issued
+                in a management room.
+            name: The name of this command.
+            help_text: The text displayed in the help for this command.
+            help_args: Help text for the arguments of this command.
+            help_section: Section of the help in which this command will appear.
+        """
         self._handler = handler
         self.needs_auth = needs_auth
         self.needs_puppeting = needs_puppeting
@@ -107,6 +211,14 @@ class CommandHandler:
         self.help_section = help_section
 
     async def get_permission_error(self, evt: CommandEvent) -> Optional[str]:
+        """Returns the reason why the command could not be issued.
+
+        Args:
+            evt: The event for which to get the error information.
+
+        Returns:
+            A string describing the error or None if there was no error.
+        """
         if self.management_only and not evt.is_management:
             return (f"`{evt.command}` is a restricted command: "
                     "you may only run it in management rooms.")
@@ -122,6 +234,22 @@ class CommandHandler:
 
     def has_permission(self, is_management: bool, puppet_whitelisted: bool,
                        matrix_puppet_whitelisted: bool, is_admin: bool, is_logged_in: bool) -> bool:
+        """Checks the permission for this command with the given status.
+
+        Args:
+            is_management: If the room in which the command will be issued is a
+                management room.
+            puppet_whitelisted: If the connected Telegram account puppet is
+                allowed to issue the command.
+            matrix_puppet_whitelisted: If the connected Matrix account puppet is
+                allowed to issue the command.
+            is_admin: If the issuing user is an admin.
+            is_logged_in: If the issuing user is logged in.
+
+        Returns:
+            True if a user with the given state is allowed to issue the
+            command.
+        """
         return ((not self.management_only or is_management) and
                 (not self.needs_puppeting or puppet_whitelisted) and
                 (not self.needs_matrix_puppeting or matrix_puppet_whitelisted) and
@@ -129,6 +257,17 @@ class CommandHandler:
                 (not self.needs_auth or is_logged_in))
 
     async def __call__(self, evt: CommandEvent) -> Dict:
+        """Executes the command if evt was issued with proper rights.
+
+        Args:
+            evt: The CommandEvent for which to check permissions.
+
+        Returns:
+            The result of the command or the error message function.
+
+        Raises:
+            FloodWaitError
+        """
         error = await self.get_permission_error(evt)
         if error is not None:
             return await evt.reply(error)
@@ -136,26 +275,22 @@ class CommandHandler:
 
     @property
     def has_help(self) -> bool:
+        """Returns true if this command has a help text."""
         return bool(self.help_section) and bool(self._help_text)
 
     @property
     def help(self) -> str:
+        """Returns the help text to this command."""
         return f"**{self.name}** {self._help_args} - {self._help_text}"
 
 
 def command_handler(_func: Optional[Callable[[CommandEvent], Awaitable[Dict]]] = None, *,
-                    needs_auth: bool = True,
-                    needs_puppeting: bool = True,
-                    needs_matrix_puppeting: bool = False,
-                    needs_admin: bool = False,
-                    management_only: bool = False,
-                    name: Optional[str] = None,
-                    help_text: str = "",
-                    help_args: str = "",
-                    help_section: HelpSection = None
+                    needs_auth: bool = True, needs_puppeting: bool = True,
+                    needs_matrix_puppeting: bool = False, needs_admin: bool = False,
+                    management_only: bool = False, name: Optional[str] = None,
+                    help_text: str = "", help_args: str = "", help_section: HelpSection = None
                     ) -> Callable[[Callable[[CommandEvent], Awaitable[Optional[Dict]]]],
                                   CommandHandler]:
-
     def decorator(func: Callable[[CommandEvent], Awaitable[Optional[Dict]]]) -> CommandHandler:
         actual_name = name or func.__name__.replace("_", "-")
         handler = CommandHandler(func, needs_auth, needs_puppeting, needs_matrix_puppeting,
@@ -168,16 +303,40 @@ def command_handler(_func: Optional[Callable[[CommandEvent], Awaitable[Dict]]] =
 
 
 class CommandProcessor:
+    """Handles the raw commands issued by a user to the Matrix bot."""
     log = logging.getLogger("mau.commands")
 
     def __init__(self, context: c.Context) -> None:
-        self.az, self.db, self.config, self.loop, self.tgbot = context.core
+        self.az, self.config, self.loop, self.tgbot = context.core
         self.public_website = context.public_website
         self.command_prefix = self.config["bridge.command_prefix"]
 
-    async def handle(self, room: MatrixRoomID, sender: u.User, command: str, args: List[str],
-                     is_management: bool, is_portal: bool) -> Optional[Dict]:
-        evt = CommandEvent(self, room, sender, command, args, is_management, is_portal)
+    async def handle(self, room: MatrixRoomID, event_id: MatrixEventID, sender: u.User,
+                     command: str, args: List[str], is_management: bool, is_portal: bool
+                     ) -> Optional[Dict]:
+        """Handles the raw commands issued by a user to the Matrix bot.
+
+        If the command is not known, it might be a followup command and is
+        delegated to a command handler registered for that purpose in the
+        senders command_status as "next".
+
+        Args:
+            room: ID of the Matrix room in which the command was issued.
+            event_id: ID of the event by which the command was issued.
+            sender: The sender who issued the command.
+            command: The issued command, case insensitive.
+            args: Arguments given with the command.
+            is_management: Whether the room is a management room.
+            is_portal: Whether the room is a portal.
+
+        Returns:
+            The result of the error message function or None if no error
+            occured. Unknown and delegated commands do not count as errors.
+        """
+        if not command_handlers or "unknown-command" not in command_handlers:
+            raise ValueError("command_handlers are not properly initialized.")
+
+        evt = CommandEvent(self, room, event_id, sender, command, args, is_management, is_portal)
         orig_command = command
         command = command.lower()
         try:

mautrix_telegram/commands/matrix_auth.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by

mautrix_telegram/commands/meta.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -65,8 +65,8 @@ def _get_management_status(evt: CommandEvent) -> str:
     return "**This is not a management room**: you must prefix commands with `$cmdprefix`."
 
 
-@command_handler(needs_auth=False, needs_puppeting=False,
+@command_handler(name="help", needs_auth=False, needs_puppeting=False,
                  help_section=SECTION_GENERAL,
                  help_text="Show this help message.")
-async def help(evt: CommandEvent) -> Optional[Dict]:
+async def help_cmd(evt: CommandEvent) -> Optional[Dict]:
     return await evt.reply(_get_management_status(evt) + "\n" + await _get_help_text(evt))

mautrix_telegram/commands/portal/admin.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -52,7 +52,7 @@ async def set_power_level(evt: CommandEvent) -> Dict:
 async def clear_db_cache(evt: CommandEvent) -> Dict:
     try:
         section = evt.args[0].lower()
-    except KeyError:
+    except IndexError:
         return await evt.reply("**Usage:** `$cmdprefix+sp clear-db-cache <section>`")
     if section == "portal":
         po.Portal.by_tgid = {}

mautrix_telegram/commands/portal/bridge.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -36,6 +36,10 @@ async def bridge(evt: CommandEvent) -> Dict:
     if len(evt.args) == 0:
         return await evt.reply("**Usage:** "
                                "`$cmdprefix+sp bridge <Telegram chat ID> [Matrix room ID]`")
+    force_use_bot = False
+    if evt.args[0] == "--usebot" and evt.sender.is_admin:
+        force_use_bot = True
+        evt.args = evt.args[1:]
     room_id = MatrixRoomID(evt.args[1]) if len(evt.args) > 1 else evt.room_id
     that_this = "This" if room_id == evt.room_id else "That"
 
@@ -80,6 +84,7 @@ async def bridge(evt: CommandEvent) -> Dict:
             "bridge_to_mxid": room_id,
             "tgid": portal.tgid,
             "peer_type": portal.peer_type,
+            "force_use_bot": force_use_bot,
         }
         return await evt.reply(f"{has_portal_message}"
                                "However, you have the permissions to unbridge that room.\n\n"
@@ -93,6 +98,7 @@ async def bridge(evt: CommandEvent) -> Dict:
         "bridge_to_mxid": room_id,
         "tgid": portal.tgid,
         "peer_type": portal.peer_type,
+        "force_use_bot": force_use_bot,
     }
     return await evt.reply("That Telegram chat has no existing portal. To confirm bridging the "
                            "chat to this room, use `$cmdprefix+sp continue`")
@@ -149,7 +155,7 @@ async def confirm_bridge(evt: CommandEvent) -> Optional[Dict]:
                                "`$cmdprefix+sp cancel` to cancel.")
 
     evt.sender.command_status = None
-    is_logged_in = await evt.sender.is_logged_in()
+    is_logged_in = await evt.sender.is_logged_in() and not status["force_use_bot"]
     user = evt.sender if is_logged_in else evt.tgbot
     try:
         entity = await user.client.get_entity(portal.peer)

mautrix_telegram/commands/portal/config.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -18,9 +18,10 @@ from typing import Dict, Awaitable
 from io import StringIO
 
 from ...config import yaml
-from ... import portal as po, user as u, util
+from ... import portal as po, util
 from .. import command_handler, CommandEvent, SECTION_PORTAL_MANAGEMENT
 
+
 @command_handler(help_section=SECTION_PORTAL_MANAGEMENT,
                  help_text="View or change per-portal settings.",
                  help_args="<`help`|_subcommand_> [...]")
@@ -76,7 +77,6 @@ def config_view(evt: CommandEvent, portal: po.Portal) -> Awaitable[Dict]:
 def config_defaults(evt: CommandEvent) -> Awaitable[Dict]:
     stream = StringIO()
     yaml.dump({
-        "edits_as_replies": evt.config["bridge.edits_as_replies"],
         "bridge_notices": {
             "default": evt.config["bridge.bridge_notices.default"],
             "exceptions": evt.config["bridge.bridge_notices.exceptions"],

mautrix_telegram/commands/portal/create_chat.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -17,6 +17,7 @@
 from typing import Dict
 
 from ... import portal as po
+from ...types import TelegramID
 from .. import command_handler, CommandEvent, SECTION_CREATING_PORTALS
 from .util import user_has_power_level, get_initial_state
 
@@ -50,7 +51,8 @@ async def create(evt: CommandEvent) -> Dict:
         "group": "chat",
     }[type]
 
-    portal = po.Portal(tgid=None, mxid=evt.room_id, title=title, about=about, peer_type=type)
+    portal = po.Portal(tgid=TelegramID(0), peer_type=type,
+                       mxid=evt.room_id, title=title, about=about)
     try:
         await portal.create_telegram_chat(evt.sender, supergroup=supergroup)
     except ValueError as e:

mautrix_telegram/commands/portal/filter.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -46,11 +46,11 @@ async def filter_mode(evt: CommandEvent) -> Dict:
                                "`!filter blacklist <chat ID>`.")
 
 
-@command_handler(needs_admin=True,
+@command_handler(name="filter", needs_admin=True,
                  help_section=SECTION_ADMIN,
                  help_args="<`whitelist`|`blacklist`> <_chat ID_>",
                  help_text="Allow or disallow bridging a specific chat.")
-async def filter(evt: CommandEvent) -> Optional[Dict]:
+async def edit_filter(evt: CommandEvent) -> Optional[Dict]:
     try:
         action = evt.args[0]
         if action not in ("whitelist", "blacklist", "add", "remove"):
@@ -58,11 +58,11 @@ async def filter(evt: CommandEvent) -> Optional[Dict]:
 
         id_str = evt.args[1]
         if id_str.startswith("-100"):
-            id = int(id_str[4:])
+            filter_id = int(id_str[4:])
         elif id_str.startswith("-"):
-            id = int(id_str[1:])
+            filter_id = int(id_str[1:])
         else:
-            id = int(id_str)
+            filter_id = int(id_str)
     except (IndexError, ValueError):
         return await evt.reply("**Usage:** `$cmdprefix+sp filter <whitelist/blacklist> <chat ID>`")
 
@@ -70,26 +70,26 @@ async def filter(evt: CommandEvent) -> Optional[Dict]:
     if mode not in ("blacklist", "whitelist"):
         return await evt.reply(f"Unknown filter mode \"{mode}\". Please fix the bridge config.")
 
-    list = evt.config["bridge.filter.list"]
+    filter_id_list = evt.config["bridge.filter.list"]
 
     if action in ("blacklist", "whitelist"):
         action = "add" if mode == action else "remove"
 
     def save() -> None:
-        evt.config["bridge.filter.list"] = list
+        evt.config["bridge.filter.list"] = filter_id_list
         evt.config.save()
-        po.Portal.filter_list = list
+        po.Portal.filter_list = filter_id_list
 
     if action == "add":
-        if id in list:
+        if filter_id in filter_id_list:
             return await evt.reply(f"That chat is already {mode}ed.")
-        list.append(id)
+        filter_id_list.append(filter_id)
         save()
         return await evt.reply(f"Chat ID added to {mode}.")
     elif action == "remove":
-        if id not in list:
+        if filter_id not in filter_id_list:
             return await evt.reply(f"That chat is not {mode}ed.")
-        list.remove(id)
+        filter_id_list.remove(filter_id)
         save()
         return await evt.reply(f"Chat ID removed from {mode}.")
     return None

mautrix_telegram/commands/portal/misc.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -38,10 +38,10 @@ async def sync_state(evt: CommandEvent) -> Dict:
     await evt.reply("Synchronization complete")
 
 
-@command_handler(needs_admin=False, needs_puppeting=False, needs_auth=False,
+@command_handler(name="id", needs_admin=False, needs_puppeting=False, needs_auth=False,
                  help_section=SECTION_MISC,
                  help_text="Get the ID of the Telegram chat where this room is bridged.")
-async def id(evt: CommandEvent) -> Dict:
+async def get_id(evt: CommandEvent) -> Dict:
     portal = po.Portal.get_by_mxid(evt.room_id)
     if not portal:
         return await evt.reply("This is not a portal room.")

mautrix_telegram/commands/portal/unbridge.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -19,7 +19,7 @@ from typing import Dict, Callable, Optional
 from ...types import MatrixRoomID
 from ... import portal as po
 from .. import command_handler, CommandEvent, SECTION_PORTAL_MANAGEMENT
-from .util import user_has_power_level, get_initial_state
+from .util import user_has_power_level
 
 
 async def _get_portal_and_check_permission(evt: CommandEvent, permission: str,

mautrix_telegram/commands/portal/util.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by

mautrix_telegram/commands/telegram/account.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -17,12 +17,12 @@
 from typing import Dict, Optional
 
 from telethon.errors import (UsernameInvalidError, UsernameNotModifiedError, UsernameOccupiedError,
-                             HashInvalidError)
+                             HashInvalidError, AuthKeyError, FirstNameInvalidError)
 from telethon.tl.types import Authorization
 from telethon.tl.functions.account import (UpdateUsernameRequest, GetAuthorizationsRequest,
-                                           ResetAuthorizationRequest)
+                                           ResetAuthorizationRequest, UpdateProfileRequest)
 
-from mautrix_telegram.commands import command_handler, CommandEvent, SECTION_AUTH
+from .. import command_handler, CommandEvent, SECTION_AUTH
 
 
 @command_handler(needs_auth=True,
@@ -53,6 +53,25 @@ async def username(evt: CommandEvent) -> Optional[Dict]:
         await evt.reply(f"Username changed to {evt.sender.username}")
 
 
+@command_handler(needs_auth=True, help_section=SECTION_AUTH, help_args="<_new displayname_>",
+                 help_text="Change your Telegram displayname.")
+async def displayname(evt: CommandEvent) -> Optional[Dict]:
+    if len(evt.args) == 0:
+        return await evt.reply("**Usage:** `$cmdprefix+sp displayname <new displayname>`")
+    if evt.sender.is_bot:
+        return await evt.reply("Bots can't set their own displayname.")
+
+    first_name, last_name = ((evt.args[0], "")
+                             if len(evt.args) == 1
+                             else (" ".join(evt.args[:-1]), evt.args[-1]))
+    try:
+        await evt.sender.client(UpdateProfileRequest(first_name=first_name, last_name=last_name))
+    except FirstNameInvalidError:
+        return await evt.reply("Invalid first name")
+    await evt.sender.update_info()
+    await evt.reply("Displayname updated")
+
+
 def _format_session(sess: Authorization) -> str:
     return (f"**{sess.app_name} {sess.app_version}**  \n"
             f"  **Platform:** {sess.device_model} {sess.platform} {sess.system_version}  \n"
@@ -87,13 +106,16 @@ async def session(evt: CommandEvent) -> Optional[Dict]:
         try:
             session_hash = int(evt.args[1])
         except ValueError:
-            return await evt.reply("Hash must be a positive integer")
-        if session_hash <= 0:
-            return await evt.reply("Hash must be a positive integer")
+            return await evt.reply("Hash must be an integer")
         try:
             ok = await evt.sender.client(ResetAuthorizationRequest(hash=session_hash))
         except HashInvalidError:
             return await evt.reply("Invalid session hash.")
+        except AuthKeyError as e:
+            if e.message == "FRESH_RESET_AUTHORISATION_FORBIDDEN":
+                return await evt.reply("New sessions can't terminate other sessions. "
+                                       "Please wait a while.")
+            raise
         if ok:
             return await evt.reply("Session terminated successfully.")
         else:

mautrix_telegram/commands/telegram/auth.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -23,9 +23,9 @@ from telethon.errors import (
     PhoneNumberAppSignupForbiddenError, PhoneNumberBannedError, PhoneNumberFloodError,
     PhoneNumberOccupiedError, PhoneNumberUnoccupiedError, SessionPasswordNeededError)
 
-from mautrix_telegram.commands import command_handler, CommandEvent, SECTION_AUTH
-from mautrix_telegram import puppet as pu, user as u
-from mautrix_telegram.util import format_duration, ignore_coro
+from ... import puppet as pu, user as u
+from ...commands import command_handler, CommandEvent, SECTION_AUTH
+from ...util import format_duration, ignore_coro
 
 
 @command_handler(needs_auth=False,
@@ -33,8 +33,8 @@ from mautrix_telegram.util import format_duration, ignore_coro
                  help_text="Check if you're logged into Telegram.")
 async def ping(evt: CommandEvent) -> Optional[Dict]:
     me = await evt.sender.client.get_me() if await evt.sender.is_logged_in() else None
-    human_tg_id = f"@{me.username}" if me.username else f"+{me.phone}"
     if me:
+        human_tg_id = f"@{me.username}" if me.username else f"+{me.phone}"
         return await evt.reply(f"You're logged in as {human_tg_id}")
     else:
         return await evt.reply("You're not logged in.")
@@ -46,11 +46,9 @@ async def ping(evt: CommandEvent) -> Optional[Dict]:
 async def ping_bot(evt: CommandEvent) -> Optional[Dict]:
     if not evt.tgbot:
         return await evt.reply("Telegram message relay bot not configured.")
-    bot_info = await evt.tgbot.client.get_me()
-    mxid = pu.Puppet.get_mxid_from_id(bot_info.id)
-    displayname = bot_info.first_name
+    info, mxid = await evt.tgbot.get_me(use_cache=False)
     return await evt.reply("Telegram message relay bot is active: "
-                           f"[{displayname}](https://matrix.to/#/{mxid}) (ID {bot_info.id})\n\n"
+                           f"[{info.first_name}](https://matrix.to/#/{mxid}) (ID {info.id})\n\n"
                            "To use the bot, simply invite it to a portal room.")
 
 
@@ -126,41 +124,20 @@ async def login(evt: CommandEvent) -> Optional[Dict]:
     if evt.config["appservice.public.enabled"]:
         prefix = evt.config["appservice.public.external"]
         url = f"{prefix}/login?token={evt.public_website.make_token(evt.sender.mxid, '/login')}"
-        if allow_matrix_login:
-            if override_sender:
-                return await evt.reply(
-                    "This bridge instance allows you to log in inside or outside of Matrix, but "
-                    "logging in as another user is only possible via the web interface.\n\n"
-                    f"Please visit [the login page]({url}) to log in as "
-                    f"[{evt.sender.mxid}](https://matrix.to/#/{evt.sender.mxid}).\n\n")
-            return await evt.reply(
-                "This bridge instance allows you to log in inside or outside Matrix.\n\n"
-                "If you would like to log in within Matrix, please send your phone number or bot "
-                "auth token here.\n"
-                "If you would like to log in outside of Matrix, please visit [the login page]"
-                f"({url}).\n\n"
-                "Logging in outside of Matrix is recommended if you have two-factor authentication "
-                "enabled, because in-Matrix login would save your password in the message history."
-                f"\n\n{nb}")
         if override_sender:
-            return await evt.reply(
-                "This bridge instance does not allow logging in inside Matrix, and logging in as "
-                "another user inside Matrix isn't possible anyway.\n\n"
-                f"Please visit [the login page]({url}) to log in as "
-                f"[{evt.sender.mxid}](https://matrix.to/#/{evt.sender.mxid}).")
-        return await evt.reply(
-            "This bridge instance does not allow logging in inside Matrix.\n\n"
-            f"Please visit [the login page]({url}) to log in.\n\n"
-            f"{nb}")
+            return await evt.reply(f"[Click here to log in]({url}) as "
+                                   f"[{evt.sender.mxid}](https://matrix.to/#/{evt.sender.mxid}).")
+        elif allow_matrix_login:
+            return await evt.reply(f"[Click here to log in]({url}). Alternatively, send your phone"
+                                   f" number (or bot auth token) here to log in.\n\n{nb}")
+        return await evt.reply(f"[Click here to log in]({url}).\n\n{nb}")
     elif allow_matrix_login:
         if override_sender:
             return await evt.reply(
                 "This bridge instance does not allow you to log in outside of Matrix. "
                 "Logging in as another user inside Matrix is not currently possible.")
-        return await evt.reply(
-            "This bridge instance does not allow you to log in outside of Matrix.\n\n"
-            "Please send your phone number or bot auth token here to start the login process.\n\n"
-            f"{nb}")
+        return await evt.reply("Please send your phone number (or bot auth token) here to start "
+                               f"the login process.\n\n{nb}")
     return await evt.reply("This bridge instance has been configured to not allow logging in.")
 
 

mautrix_telegram/commands/telegram/misc.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -19,18 +19,21 @@ import codecs
 import base64
 import re
 
-from telethon.errors import (InviteHashInvalidError, InviteHashExpiredError,
+from telethon.errors import (InviteHashInvalidError, InviteHashExpiredError, OptionsTooMuchError,
                              UserAlreadyParticipantError)
-from telethon.tl.types import User as TLUser, TypeUpdates, MessageMediaGame
+from telethon.tl.patched import Message
+from telethon.tl.types import (User as TLUser, TypeUpdates, MessageMediaGame, MessageMediaPoll,
+                               TypePeer)
 from telethon.tl.types.messages import BotCallbackAnswer
 from telethon.tl.functions.messages import (ImportChatInviteRequest, CheckChatInviteRequest,
-                                            GetBotCallbackAnswerRequest)
+                                            GetBotCallbackAnswerRequest, SendVoteRequest)
 from telethon.tl.functions.channels import JoinChannelRequest
 
-from mautrix_telegram import puppet as pu, portal as po
-from mautrix_telegram.db import Message as DBMessage
-from mautrix_telegram.types import TelegramID
-from mautrix_telegram.commands import command_handler, CommandEvent, SECTION_MISC, SECTION_CREATING_PORTALS
+from ... import puppet as pu, portal as po
+from ...abstract_user import AbstractUser
+from ...db import Message as DBMessage
+from ...types import TelegramID
+from ...commands import command_handler, CommandEvent, SECTION_MISC, SECTION_CREATING_PORTALS
 
 
 @command_handler(help_section=SECTION_MISC,
@@ -167,6 +170,45 @@ async def sync(evt: CommandEvent) -> Optional[Dict]:
 PEER_TYPE_CHAT = b"g"
 
 
+class MessageIDError(ValueError):
+    def __init__(self, message: str) -> None:
+        super().__init__(message)
+        self.message = message
+
+
+async def _parse_encoded_msgid(user: AbstractUser, enc_id: str, type_name: str
+                               ) -> Tuple[TypePeer, Message]:
+    try:
+        enc_id += (4 - len(enc_id) % 4) * "="
+        enc_id = base64.b64decode(enc_id)
+        peer_type, enc_id = bytes([enc_id[0]]), enc_id[1:]
+        tgid = TelegramID(int(codecs.encode(enc_id[0:5], "hex_codec"), 16))
+        msg_id = TelegramID(int(codecs.encode(enc_id[5:10], "hex_codec"), 16))
+        space = None
+        if peer_type == PEER_TYPE_CHAT:
+            space = TelegramID(int(codecs.encode(enc_id[10:15], "hex_codec"), 16))
+    except ValueError as e:
+        raise MessageIDError(f"Invalid {type_name} ID (format)") from e
+
+    if peer_type == PEER_TYPE_CHAT:
+        orig_msg = DBMessage.get_one_by_tgid(msg_id, space)
+        if not orig_msg:
+            raise MessageIDError(f"Invalid {type_name} ID (original message not found in db)")
+        new_msg = DBMessage.get_by_mxid(orig_msg.mxid, orig_msg.mx_room, user.tgid)
+        if not new_msg:
+            raise MessageIDError(f"Invalid {type_name} ID (your copy of message not found in db)")
+        msg_id = new_msg.tgid
+    try:
+        peer = await user.client.get_input_entity(tgid)
+    except ValueError as e:
+        raise MessageIDError(f"Invalid {type_name} ID (chat not found)") from e
+
+    msg = await user.client.get_messages(entity=peer, ids=msg_id)
+    if not msg:
+        raise MessageIDError(f"Invalid {type_name} ID (message not found)")
+    return peer, msg
+
+
 @command_handler(help_section=SECTION_MISC,
                  help_args="<_play ID_>",
                  help_text="Play a Telegram game.")
@@ -179,38 +221,63 @@ async def play(evt: CommandEvent) -> Optional[Dict]:
         return await evt.reply("Bots can't play games :(")
 
     try:
-        play_id = evt.args[0]
-        play_id += (4 - len(play_id) % 4) * "="
-        play_id = base64.b64decode(play_id)
-        peer_type, play_id = bytes([play_id[0]]), play_id[1:]
-        tgid = TelegramID(int(codecs.encode(play_id[0:5], "hex_codec"), 16))
-        msg_id = TelegramID(int(codecs.encode(play_id[5:10], "hex_codec"), 16))
-        space = None
-        if peer_type == PEER_TYPE_CHAT:
-            space = TelegramID(int(codecs.encode(play_id[10:15], "hex_codec"), 16))
-    except ValueError:
-        return await evt.reply("Invalid play ID (format)")
+        peer, msg = await _parse_encoded_msgid(evt.sender, evt.args[0], type_name="play")
+    except MessageIDError as e:
+        return await evt.reply(e.message)
 
-    if peer_type == PEER_TYPE_CHAT:
-        orig_msg = DBMessage.get_by_tgid(msg_id, space)
-        if not orig_msg:
-            return await evt.reply("Invalid play ID (original message not found in db)")
-        new_msg = DBMessage.get_by_mxid(orig_msg.mxid, orig_msg.mx_room, evt.sender.tgid)
-        if not new_msg:
-            return await evt.reply("Invalid play ID (your copy of message not found in db)")
-        msg_id = new_msg.tgid
-    try:
-        peer = await evt.sender.client.get_input_entity(tgid)
-    except ValueError:
-        return await evt.reply("Invalid play ID (chat not found)")
-
-    msg = await evt.sender.client.get_messages(entity=peer, ids=msg_id)
-    if not msg or not isinstance(msg.media, MessageMediaGame):
+    if not isinstance(msg.media, MessageMediaGame):
         return await evt.reply("Invalid play ID (message doesn't look like a game)")
 
-    game = await evt.sender.client(GetBotCallbackAnswerRequest(peer=peer, msg_id=msg_id, game=True))
+    game = await evt.sender.client(GetBotCallbackAnswerRequest(peer=peer, msg_id=msg.id, game=True))
     if not isinstance(game, BotCallbackAnswer):
         return await evt.reply("Game request response invalid")
 
     await evt.reply(f"Click [here]({game.url}) to play {msg.media.game.title}:\n\n"
                     f"{msg.media.game.description}")
+
+
+@command_handler(help_section=SECTION_MISC,
+                 help_args="<_poll ID_> <_choice number_>",
+                 help_text="Vote in a Telegram poll.")
+async def vote(evt: CommandEvent) -> Optional[Dict]:
+    if len(evt.args) < 1:
+        return await evt.reply("**Usage:** `$cmdprefix+sp vote <poll ID> <choice number>`")
+    elif not await evt.sender.is_logged_in():
+        return await evt.reply("You must be logged in with a real account to vote in polls.")
+    elif evt.sender.is_bot:
+        return await evt.reply("Bots can't vote in polls :(")
+
+    try:
+        peer, msg = await _parse_encoded_msgid(evt.sender, evt.args[0], type_name="poll")
+    except MessageIDError as e:
+        return await evt.reply(e.message)
+
+    if not isinstance(msg.media, MessageMediaPoll):
+        return await evt.reply("Invalid poll ID (message doesn't look like a poll)")
+
+    options = []
+    for option in evt.args[1:]:
+        try:
+            if len(option) > 10:
+                raise ValueError("option index too long")
+            option_index = int(option) - 1
+        except ValueError:
+            option_index = None
+        if option_index is None:
+            return await evt.reply(f"Invalid option number \"{option}\"",
+                                   render_markdown=False, allow_html=False)
+        elif option_index < 0:
+            return await evt.reply(f"Invalid option number {option}. "
+                                   f"Option numbers must be positive.")
+        elif option_index >= len(msg.media.poll.answers):
+            return await evt.reply(f"Invalid option number {option}. "
+                                   f"The poll only has {len(msg.media.poll.answers)} options.")
+        options.append(msg.media.poll.answers[option_index].option)
+    options = [msg.media.poll.answers[int(option) - 1].option
+               for option in evt.args[1:]]
+    try:
+        resp = await evt.sender.client(SendVoteRequest(peer=peer, msg_id=msg.id, options=options))
+    except OptionsTooMuchError:
+        return await evt.reply("You passed too many options.")
+    # TODO use response
+    return await evt.mark_read()

mautrix_telegram/config.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -103,12 +103,20 @@ class DictWithRecursion:
 
 
 class Config(DictWithRecursion):
-    def __init__(self, path: str, registration_path: str, base_path: str) -> None:
+    def __init__(self, path: str, registration_path: str, base_path: str,
+                 overrides: Dict[str, Any] = None) -> None:
         super().__init__()
         self.path = path  # type: str
         self.registration_path = registration_path  # type: str
         self.base_path = base_path  # type: str
         self._registration = None  # type: Optional[Dict]
+        self._overrides = overrides or {}  # type: Dict[str, Any]
+
+    def __getitem__(self, key: str) -> Any:
+        try:
+            return self._overrides[f"MAUTRIX_TELEGRAM_{key.replace('.', '_').upper()}"]
+        except KeyError:
+            return super().__getitem__(key)
 
     def load(self) -> None:
         with open(self.path, 'r') as stream:
@@ -181,9 +189,14 @@ class Config(DictWithRecursion):
         copy("appservice.bot_displayname")
         copy("appservice.bot_avatar")
 
+        copy("appservice.community_id")
+
         copy("appservice.as_token")
         copy("appservice.hs_token")
 
+        copy("metrics.enabled")
+        copy("metrics.listen_port")
+
         copy("bridge.username_template")
         copy("bridge.alias_template")
         copy("bridge.displayname_template")
@@ -199,13 +212,13 @@ class Config(DictWithRecursion):
         copy("bridge.sync_matrix_state")
         copy("bridge.allow_matrix_login")
         copy("bridge.plaintext_highlights")
-        copy("bridge.edits_as_replies")
-        copy("bridge.highlight_edits")
         copy("bridge.public_portals")
         copy("bridge.catch_up")
         copy("bridge.sync_with_custom_puppets")
         copy("bridge.telegram_link_preview")
         copy("bridge.inline_images")
+        copy("bridge.image_as_file_size")
+        copy("bridge.max_document_size")
 
         copy("bridge.bot_messages_as_notices")
         if isinstance(self["bridge.bridge_notices"], bool):
@@ -256,10 +269,24 @@ class Config(DictWithRecursion):
         copy("telegram.api_id")
         copy("telegram.api_hash")
         copy("telegram.bot_token")
+
+        copy("telegram.connection.timeout")
+        copy("telegram.connection.retries")
+        copy("telegram.connection.retry_delay")
+        copy("telegram.connection.flood_sleep_threshold")
+        copy("telegram.connection.request_retries")
+
+        copy("telegram.device_info.device_model")
+        copy("telegram.device_info.system_version")
+        copy("telegram.device_info.app_version")
+        copy("telegram.device_info.lang_code")
+        copy("telegram.device_info.system_lang_code")
+
         copy("telegram.server.enabled")
         copy("telegram.server.dc")
         copy("telegram.server.ip")
         copy("telegram.server.port")
+
         copy("telegram.proxy.type")
         copy("telegram.proxy.address")
         copy("telegram.proxy.port")
@@ -327,3 +354,6 @@ class Config(DictWithRecursion):
             "sender_localpart": self["appservice.bot_username"],
             "rate_limited": False
         }
+        if self["appservice.community_id"]:
+            self._registration["namespaces"]["users"][0]["group_id"] \
+                = self["appservice.community_id"]

mautrix_telegram/context.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -19,8 +19,6 @@ from typing import Optional, Tuple, TYPE_CHECKING
 if TYPE_CHECKING:
     import asyncio
 
-    from sqlalchemy.orm import scoped_session
-
     from alchemysession import AlchemySessionContainer
     from mautrix_appservice import AppService
 
@@ -31,20 +29,17 @@ if TYPE_CHECKING:
 
 
 class Context:
-    def __init__(self, az: 'AppService', db: 'scoped_session', config: 'Config',
-                 loop: 'asyncio.AbstractEventLoop', session_container: 'AlchemySessionContainer'
-                 ) -> None:
+    def __init__(self, az: 'AppService', config: 'Config', loop: 'asyncio.AbstractEventLoop',
+                 session_container: 'AlchemySessionContainer', bot: Optional['Bot']) -> None:
         self.az = az  # type: AppService
-        self.db = db  # type: scoped_session
         self.config = config  # type: Config
         self.loop = loop  # type: asyncio.AbstractEventLoop
-        self.bot = None  # type: Optional[Bot]
-        self.mx = None  # type: MatrixHandler
+        self.bot = bot  # type: Optional[Bot]
+        self.mx = None  # type: Optional[MatrixHandler]
         self.session_container = session_container  # type: AlchemySessionContainer
-        self.public_website = None  # type: PublicBridgeWebsite
-        self.provisioning_api = None  # type: ProvisioningAPI
+        self.public_website = None  # type: Optional[PublicBridgeWebsite]
+        self.provisioning_api = None  # type: Optional[ProvisioningAPI]
 
     @property
-    def core(self) -> Tuple['AppService', 'scoped_session', 'Config',
-                            'asyncio.AbstractEventLoop', Optional['Bot']]:
-        return (self.az, self.db, self.config, self.loop, self.bot)
+    def core(self) -> Tuple['AppService', 'Config', 'asyncio.AbstractEventLoop', Optional['Bot']]:
+        return self.az, self.config, self.loop, self.bot

mautrix_telegram/db/__init__.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by

mautrix_telegram/db/base.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -44,14 +44,15 @@ class BaseBase:
         pass
 
     def update(self, **values) -> None:
-        self.db.execute(self.t.update()
-                        .where(self._edit_identity)
-                        .values(**values))
+        with self.db.begin() as conn:
+            conn.execute(self.t.update()
+                         .where(self._edit_identity)
+                         .values(**values))
         for key, value in values.items():
             setattr(self, key, value)
 
     def delete(self) -> None:
-        self.db.execute(self.t.delete().where(self._edit_identity))
-
+        with self.db.begin() as conn:
+            conn.execute(self.t.delete().where(self._edit_identity))
 
 Base = declarative_base(cls=BaseBase)

mautrix_telegram/db/bot_chat.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -29,15 +29,17 @@ class BotChat(Base):
     type = Column(String, nullable=False)
 
     @classmethod
-    def delete(cls, id: TelegramID) -> None:
-        cls.db.execute(cls.t.delete().where(cls.c.id == id))
+    def delete(cls, chat_id: TelegramID) -> None:
+        with cls.db.begin() as conn:
+            conn.execute(cls.t.delete().where(cls.c.id == chat_id))
 
     @classmethod
     def all(cls) -> Iterable['BotChat']:
         rows = cls.db.execute(cls.t.select())
         for row in rows:
-            id, type = row
-            yield cls(id=id, type=type)
+            chat_id, chat_type = row
+            yield cls(id=chat_id, type=chat_type)
 
     def insert(self) -> None:
-        self.db.execute(self.t.insert().values(id=self.id, type=self.type))
+        with self.db.begin() as conn:
+            conn.execute(self.t.insert().values(id=self.id, type=self.type))

mautrix_telegram/db/message.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -14,7 +14,7 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from sqlalchemy import Column, UniqueConstraint, Integer, String, and_, func, select
+from sqlalchemy import Column, UniqueConstraint, Integer, String, and_, func, desc, select
 from sqlalchemy.engine.result import RowProxy
 from typing import Optional, List
 
@@ -29,25 +29,44 @@ class Message(Base):
     mx_room = Column(String)  # type: MatrixRoomID
     tgid = Column(Integer, primary_key=True)  # type: TelegramID
     tg_space = Column(Integer, primary_key=True)  # type: TelegramID
+    edit_index = Column(Integer, primary_key=True)  # type: int
 
     __table_args__ = (UniqueConstraint("mxid", "mx_room", "tg_space", name="_mx_id_room"),)
 
     @classmethod
     def _one_or_none(cls, rows: RowProxy) -> Optional['Message']:
         try:
-            mxid, mx_room, tgid, tg_space = next(rows)
-            return cls(mxid=mxid, mx_room=mx_room, tgid=tgid, tg_space=tg_space)
+            mxid, mx_room, tgid, tg_space, edit_index = next(rows)
+            return cls(mxid=mxid, mx_room=mx_room, tgid=tgid, tg_space=tg_space,
+                       edit_index=edit_index)
         except StopIteration:
             return None
 
     @staticmethod
     def _all(rows: RowProxy) -> List['Message']:
-        return [Message(mxid=row[0], mx_room=row[1], tgid=row[2], tg_space=row[3])
+        return [Message(mxid=row[0], mx_room=row[1], tgid=row[2], tg_space=row[3],
+                        edit_index=row[4])
                 for row in rows]
 
     @classmethod
-    def get_by_tgid(cls, tgid: TelegramID, tg_space: TelegramID) -> Optional['Message']:
-        return cls._select_one_or_none(and_(cls.c.tgid == tgid, cls.c.tg_space == tg_space))
+    def get_all_by_tgid(cls, tgid: TelegramID, tg_space: TelegramID) -> List['Message']:
+        return cls._all(cls.db.execute(cls.t.select().where(and_(cls.c.tgid == tgid,
+                                                                 cls.c.tg_space == tg_space))))
+
+    @classmethod
+    def get_one_by_tgid(cls, tgid: TelegramID, tg_space: TelegramID, edit_index: int = 0
+                        ) -> Optional['Message']:
+        query = cls.t.select()
+        if edit_index < 0:
+            query = (query
+                     .where(and_(cls.c.tgid == tgid, cls.c.tg_space == tg_space))
+                     .order_by(desc(cls.c.edit_index))
+                     .limit(1)
+                     .offset(-edit_index - 1))
+        else:
+            query = query.where(and_(cls.c.tgid == tgid, cls.c.tg_space == tg_space,
+                                     cls.c.edit_index == edit_index))
+        return cls._one_or_none(cls.db.execute(query))
 
     @classmethod
     def count_spaces_by_mxid(cls, mxid: MatrixEventID, mx_room: MatrixRoomID) -> int:
@@ -67,21 +86,28 @@ class Message(Base):
                                             cls.c.tg_space == tg_space))
 
     @classmethod
-    def update_by_tgid(cls, s_tgid: TelegramID, s_tg_space: TelegramID, **values) -> None:
-        cls.db.execute(cls.t.update()
-                       .where(and_(cls.c.tgid == s_tgid, cls.c.tg_space == s_tg_space))
-                       .values(**values))
+    def update_by_tgid(cls, s_tgid: TelegramID, s_tg_space: TelegramID, s_edit_index: int,
+                       **values) -> None:
+        with cls.db.begin() as conn:
+            conn.execute(cls.t.update()
+                         .where(and_(cls.c.tgid == s_tgid, cls.c.tg_space == s_tg_space,
+                                     cls.c.edit_index == s_edit_index))
+                         .values(**values))
 
     @classmethod
     def update_by_mxid(cls, s_mxid: MatrixEventID, s_mx_room: MatrixRoomID, **values) -> None:
-        cls.db.execute(cls.t.update()
-                       .where(and_(cls.c.mxid == s_mxid, cls.c.mx_room == s_mx_room))
-                       .values(**values))
+        with cls.db.begin() as conn:
+            conn.execute(cls.t.update()
+                         .where(and_(cls.c.mxid == s_mxid, cls.c.mx_room == s_mx_room))
+                         .values(**values))
 
     @property
     def _edit_identity(self):
-        return and_(self.c.tgid == self.tgid, self.c.tg_space == self.tg_space)
+        return and_(self.c.tgid == self.tgid, self.c.tg_space == self.tg_space,
+                    self.c.edit_index == self.edit_index)
 
     def insert(self) -> None:
-        self.db.execute(self.t.insert().values(mxid=self.mxid, mx_room=self.mx_room, tgid=self.tgid,
-                                               tg_space=self.tg_space))
+        with self.db.begin() as conn:
+            conn.execute(self.t.insert().values(mxid=self.mxid, mx_room=self.mx_room,
+                                                tgid=self.tgid, tg_space=self.tg_space,
+                                                edit_index=self.edit_index))

mautrix_telegram/db/portal.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -74,7 +74,8 @@ class Portal(Base):
         return and_(self.c.tgid == self.tgid, self.c.tg_receiver == self.tg_receiver)
 
     def insert(self) -> None:
-        self.db.execute(self.t.insert().values(
-            tgid=self.tgid, tg_receiver=self.tg_receiver, peer_type=self.peer_type,
-            megagroup=self.megagroup, mxid=self.mxid, config=self.config, username=self.username,
-            title=self.title, about=self.about, photo_id=self.photo_id))
+        with self.db.begin() as conn:
+            conn.execute(self.t.insert().values(
+                tgid=self.tgid, tg_receiver=self.tg_receiver, peer_type=self.peer_type,
+                megagroup=self.megagroup, mxid=self.mxid, config=self.config,
+                username=self.username, title=self.title, about=self.about, photo_id=self.photo_id))

mautrix_telegram/db/puppet.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -35,15 +35,16 @@ class Puppet(Base):
     photo_id = Column(String, nullable=True)
     is_bot = Column(Boolean, nullable=True)
     matrix_registered = Column(Boolean, nullable=False, server_default=expression.false())
+    disable_updates = Column(Boolean, nullable=False, server_default=expression.false())
 
     @classmethod
     def scan(cls, row) -> Optional['Puppet']:
         (id, custom_mxid, access_token, displayname, displayname_source, username, photo_id,
-         is_bot, matrix_registered) = row
+         is_bot, matrix_registered, disable_updates) = row
         return cls(id=id, custom_mxid=custom_mxid, access_token=access_token,
                    displayname=displayname, displayname_source=displayname_source,
                    username=username, photo_id=photo_id, is_bot=is_bot,
-                   matrix_registered=matrix_registered)
+                   matrix_registered=matrix_registered, disable_updates=disable_updates)
 
     @classmethod
     def _one_or_none(cls, rows: RowProxy) -> Optional['Puppet']:
@@ -79,8 +80,9 @@ class Puppet(Base):
         return self.c.id == self.id
 
     def insert(self) -> None:
-        self.db.execute(self.t.insert().values(
-            id=self.id, custom_mxid=self.custom_mxid, access_token=self.access_token,
-            displayname=self.displayname, displayname_source=self.displayname_source,
-            username=self.username, photo_id=self.photo_id, is_bot=self.is_bot,
-            matrix_registered=self.matrix_registered))
+        with self.db.begin() as conn:
+            conn.execute(self.t.insert().values(
+                id=self.id, custom_mxid=self.custom_mxid, access_token=self.access_token,
+                displayname=self.displayname, displayname_source=self.displayname_source,
+                username=self.username, photo_id=self.photo_id, is_bot=self.is_bot,
+                matrix_registered=self.matrix_registered, disable_updates=self.disable_updates))

mautrix_telegram/db/room_state.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -47,14 +47,16 @@ class RoomState(Base):
             return None
 
     def update(self) -> None:
-        self.db.execute(self.t.update()
-                        .where(self.c.room_id == self.room_id)
-                        .values(power_levels=self._power_levels_text))
+        with self.db.begin() as conn:
+            conn.execute(self.t.update()
+                         .where(self.c.room_id == self.room_id)
+                         .values(power_levels=self._power_levels_text))
 
     @property
     def _edit_identity(self):
         return self.c.room_id == self.room_id
 
     def insert(self) -> None:
-        self.db.execute(self.t.insert().values(room_id=self.room_id,
-                                               power_levels=self._power_levels_text))
+        with self.db.begin() as conn:
+            conn.execute(self.t.insert().values(room_id=self.room_id,
+                                                power_levels=self._power_levels_text))

mautrix_telegram/db/telegram_file.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -15,7 +15,6 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 from sqlalchemy import Column, ForeignKey, Integer, BigInteger, String, Boolean
-from sqlalchemy.orm import relationship
 from typing import Optional
 
 from .base import Base
@@ -33,23 +32,25 @@ class TelegramFile(Base):
     width = Column(Integer, nullable=True)
     height = Column(Integer, nullable=True)
     thumbnail_id = Column("thumbnail", String, ForeignKey("telegram_file.id"), nullable=True)
-    thumbnail = relationship("TelegramFile", uselist=False)
+    thumbnail = None  # type: Optional[TelegramFile]
 
     @classmethod
-    def get(cls, id: str) -> Optional['TelegramFile']:
-        rows = cls.db.execute(cls.t.select().where(cls.c.id == id))
+    def get(cls, loc_id: str) -> Optional['TelegramFile']:
+        rows = cls.db.execute(cls.t.select().where(cls.c.id == loc_id))
         try:
-            id, mxc, mime, conv, ts, s, w, h, thumb_id = next(rows)
+            loc_id, mxc, mime, conv, ts, s, w, h, thumb_id = next(rows)
             thumb = None
             if thumb_id:
                 thumb = cls.get(thumb_id)
-            return cls(id=id, mxc=mxc, mime_type=mime, was_converted=conv, timestamp=ts,
+            return cls(id=loc_id, mxc=mxc, mime_type=mime, was_converted=conv, timestamp=ts,
                        size=s, width=w, height=h, thumbnail_id=thumb_id, thumbnail=thumb)
         except StopIteration:
             return None
 
     def insert(self) -> None:
-        self.db.execute(self.t.insert().values(
-            id=self.id, mxc=self.mxc, mime_type=self.mime_type, was_converted=self.was_converted,
-            timestamp=self.timestamp, size=self.size, width=self.width, height=self.height,
-            thumbnail=self.thumbnail.id if self.thumbnail else self.thumbnail_id))
+        with self.db.begin() as conn:
+            conn.execute(self.t.insert().values(
+                id=self.id, mxc=self.mxc, mime_type=self.mime_type,
+                was_converted=self.was_converted, timestamp=self.timestamp, size=self.size,
+                width=self.width, height=self.height,
+                thumbnail=self.thumbnail.id if self.thumbnail else self.thumbnail_id))

mautrix_telegram/db/user.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -18,7 +18,7 @@ from sqlalchemy import Column, ForeignKey, ForeignKeyConstraint, Integer, String
 from sqlalchemy.engine.result import RowProxy
 from typing import Optional, Iterable, Tuple
 
-from ..types import MatrixUserID, MatrixRoomID, TelegramID
+from ..types import MatrixUserID, TelegramID
 from .base import Base
 
 
@@ -65,9 +65,10 @@ class User(Base):
         return self.c.mxid == self.mxid
 
     def insert(self) -> None:
-        self.db.execute(self.t.insert().values(
-            mxid=self.mxid, tgid=self.tgid, tg_username=self.tg_username, tg_phone=self.tg_phone,
-            saved_contacts=self.saved_contacts))
+        with self.db.begin() as conn:
+            conn.execute(self.t.insert().values(
+                mxid=self.mxid, tgid=self.tgid, tg_username=self.tg_username,
+                tg_phone=self.tg_phone, saved_contacts=self.saved_contacts))
 
     @property
     def contacts(self) -> Iterable[TelegramID]:
@@ -78,10 +79,11 @@ class User(Base):
 
     @contacts.setter
     def contacts(self, puppets: Iterable[TelegramID]) -> None:
-        self.db.execute(Contact.t.delete().where(Contact.c.user == self.tgid))
-        if puppets:
-            self.db.execute(Contact.t.insert(), [{"user": self.tgid, "contact": tgid}
-                                                 for tgid in puppets])
+        with self.db.begin() as conn:
+            conn.execute(Contact.t.delete().where(Contact.c.user == self.tgid))
+            insert_puppets = [{"user": self.tgid, "contact": tgid} for tgid in puppets]
+            if insert_puppets:
+                conn.execute(Contact.t.insert(), insert_puppets)
 
     @property
     def portals(self) -> Iterable[Tuple[TelegramID, TelegramID]]:
@@ -92,19 +94,20 @@ class User(Base):
 
     @portals.setter
     def portals(self, portals: Iterable[Tuple[TelegramID, TelegramID]]) -> None:
-        self.db.execute(UserPortal.t.delete().where(UserPortal.c.user == self.tgid))
-        if portals:
-            self.db.execute(UserPortal.t.insert(),
-                            [{
-                                "user": self.tgid,
-                                "portal": tgid,
-                                "portal_receiver": tg_receiver
-                            } for tgid, tg_receiver in portals])
+        with self.db.begin() as conn:
+            conn.execute(UserPortal.t.delete().where(UserPortal.c.user == self.tgid))
+            insert_portals = [{
+                "user": self.tgid,
+                "portal": tgid,
+                "portal_receiver": tg_receiver
+            } for tgid, tg_receiver in portals]
+            if insert_portals:
+                conn.execute(UserPortal.t.insert(), insert_portals)
 
     def delete(self) -> None:
         super().delete()
-        self.portals = None
-        self.contacts = None
+        self.portals = []
+        self.contacts = []
 
 
 class UserPortal(Base):
@@ -125,4 +128,3 @@ class Contact(Base):
 
     user = Column(Integer, ForeignKey("user.tgid"), primary_key=True)  # type: TelegramID
     contact = Column(Integer, ForeignKey("puppet.id"), primary_key=True)  # type: TelegramID
-

mautrix_telegram/db/user_profile.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -50,7 +50,8 @@ class UserProfile(Base):
 
     @classmethod
     def delete_all(cls, room_id: MatrixRoomID) -> None:
-        cls.db.execute(cls.t.delete().where(cls.c.room_id == room_id))
+        with cls.db.begin() as conn:
+            conn.execute(cls.t.delete().where(cls.c.room_id == room_id))
 
     def update(self) -> None:
         super().update(membership=self.membership, displayname=self.displayname,
@@ -61,8 +62,8 @@ class UserProfile(Base):
         return and_(self.c.room_id == self.room_id, self.c.user_id == self.user_id)
 
     def insert(self) -> None:
-        self.db.execute(self.t.insert().values(room_id=self.room_id, user_id=self.user_id,
-                                               membership=self.membership,
-                                               displayname=self.displayname,
-                                               avatar_url=self.avatar_url))
-
+        with self.db.begin() as conn:
+            conn.execute(self.t.insert().values(room_id=self.room_id, user_id=self.user_id,
+                                                membership=self.membership,
+                                                displayname=self.displayname,
+                                                avatar_url=self.avatar_url))

mautrix_telegram/formatter/__init__.py

@@ -1,9 +1,8 @@
 from .from_matrix import (matrix_reply_to_telegram, matrix_to_telegram, matrix_text_to_telegram,
                           init_mx)
-from .from_telegram import (telegram_reply_to_matrix, telegram_to_matrix, init_tg)
+from .from_telegram import telegram_reply_to_matrix, telegram_to_matrix
 from .. import context as c
 
 
 def init(context: c.Context) -> None:
     init_mx(context)
-    init_tg(context)

mautrix_telegram/formatter/from_matrix/__init__.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -36,7 +36,7 @@ should_bridge_plaintext_highlights = False  # type: bool
 
 command_regex = re.compile(r"^!([A-Za-z0-9@]+)")  # type: Pattern
 not_command_regex = re.compile(r"^\\(![A-Za-z0-9@]+)")  # type: Pattern
-plain_mention_regex = None  # type: Pattern
+plain_mention_regex = None  # type: Optional[Pattern]
 
 
 def plain_mention_to_html(match: Match) -> str:
@@ -76,7 +76,6 @@ def matrix_to_telegram(html: str) -> ParsedMessage:
         if should_bridge_plaintext_highlights:
             html = plain_mention_regex.sub(plain_mention_to_html, html)
 
-        html = add_surrogates(html)
         text, entities = parse_html(add_surrogates(html))
         text = remove_surrogates(text.strip())
         text, entities = cut_long_message(text, entities)
@@ -88,25 +87,28 @@ def matrix_to_telegram(html: str) -> ParsedMessage:
 
 def matrix_reply_to_telegram(content: Dict[str, Any], tg_space: TelegramID,
                              room_id: Optional[MatrixRoomID] = None) -> Optional[TelegramID]:
+    relates_to = content.get("m.relates_to", None) or {}
+    if not relates_to:
+        return None
+    reply = (relates_to if relates_to.get("rel_type", None) == "m.reference"
+             else relates_to.get("m.in_reply_to", None) or {})
+    if not reply:
+        return None
+    room_id = room_id or reply.get("room_id", None)
+    event_id = reply.get("event_id", None)
+    if not event_id:
+        return
+
     try:
-        reply = content.get("m.relates_to", {}).get("m.in_reply_to", {})
-        if not reply:
-            return None
-        room_id = room_id or reply["room_id"]
-        event_id = reply["event_id"]
-
-        try:
-            if content["format"] == "org.matrix.custom.html":
-                content["formatted_body"] = trim_reply_fallback_html(content["formatted_body"])
-        except KeyError:
-            pass
-        content["body"] = trim_reply_fallback_text(content["body"])
-
-        message = DBMessage.get_by_mxid(event_id, room_id, tg_space)
-        if message:
-            return message.tgid
+        if content["format"] == "org.matrix.custom.html":
+            content["formatted_body"] = trim_reply_fallback_html(content["formatted_body"])
     except KeyError:
         pass
+    content["body"] = trim_reply_fallback_text(content["body"])
+
+    message = DBMessage.get_by_mxid(event_id, room_id, tg_space)
+    if message:
+        return message.tgid
     return None
 
 
@@ -148,5 +150,5 @@ def init_mx(context: "Context") -> None:
     config = context.config
     dn_template = config.get("bridge.displayname_template", "{displayname} (Telegram)")
     dn_template = re.escape(dn_template).replace(re.escape("{displayname}"), "[^>]+")
-    plain_mention_regex = re.compile(f"(\s|^)({dn_template})")
+    plain_mention_regex = re.compile(f"^({dn_template})")
     should_bridge_plaintext_highlights = config["bridge.plaintext_highlights"] or False

mautrix_telegram/formatter/from_matrix/html_reader.py

@@ -1,4 +1,66 @@
-try:
-    from .html_reader_lxml import HTMLNode, read_html
-except ImportError:
-    from .html_reader_htmlparser import HTMLNode, read_html
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Dict, List, Tuple
+
+from html.parser import HTMLParser
+
+
+class HTMLNode(list):
+    def __init__(self, tag: str, attrs: List[Tuple[str, str]]):
+        super().__init__()
+        self.tag = tag  # type: str
+        self.text = ""  # type: str
+        self.tail = ""  # type: str
+        self.attrib = dict(attrs)  # type: Dict[str, str]
+
+
+class NodeifyingParser(HTMLParser):
+    # From https://www.w3.org/TR/html5/syntax.html#writing-html-documents-elements
+    void_tags = ("area", "base", "br", "col", "command", "embed", "hr", "img", "input", "link",
+                 "meta", "param", "source", "track", "wbr")
+
+    def __init__(self):
+        super().__init__()
+        self.stack = [HTMLNode("html", [])]  # type: List[HTMLNode]
+
+    def handle_starttag(self, tag, attrs):
+        node = HTMLNode(tag, attrs)
+        self.stack[-1].append(node)
+        if tag not in self.void_tags:
+            self.stack.append(node)
+
+    def handle_startendtag(self, tag, attrs):
+        self.stack[-1].append(HTMLNode(tag, attrs))
+
+    def handle_endtag(self, tag):
+        if tag == self.stack[-1].tag:
+            self.stack.pop()
+
+    def handle_data(self, data):
+        if len(self.stack[-1]) > 0:
+            self.stack[-1][-1].tail += data
+        else:
+            self.stack[-1].text += data
+
+    def error(self, message):
+        pass
+
+
+def read_html(data: str) -> HTMLNode:
+    parser = NodeifyingParser()
+    parser.feed(data)
+    return parser.stack[0]

mautrix_telegram/formatter/from_matrix/html_reader_htmlparser.py

@@ -1,58 +0,0 @@
-# -*- coding: future_fstrings -*-
-# mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Dict, List, Tuple
-
-from html.parser import HTMLParser
-
-
-class HTMLNode(list):
-    def __init__(self, tag: str, attrs: List[Tuple[str, str]]):
-        super().__init__()
-        self.tag = tag  # type: str
-        self.text = ""  # type: str
-        self.tail = ""  # type: str
-        self.attrib = dict(attrs)  # type: Dict[str, str]
-
-
-class NodeifyingParser(HTMLParser):
-    def __init__(self):
-        super().__init__()
-        self.stack = [HTMLNode("html", [])]  # type: List[HTMLNode]
-
-    def handle_starttag(self, tag, attrs):
-        node = HTMLNode(tag, attrs)
-        self.stack[-1].append(node)
-        self.stack.append(node)
-
-    def handle_endtag(self, tag):
-        if tag == self.stack[-1].tag:
-            self.stack.pop()
-
-    def handle_data(self, data):
-        if len(self.stack[-1]) > 0:
-            self.stack[-1][-1].tail += data
-        else:
-            self.stack[-1].text += data
-
-    def error(self, message):
-        pass
-
-
-def read_html(data: str) -> HTMLNode:
-    parser = NodeifyingParser()
-    parser.feed(data)
-    return parser.stack[0]

mautrix_telegram/formatter/from_matrix/parser.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -18,15 +18,15 @@ from typing import List, Tuple, Pattern
 import re
 
 from telethon.tl.types import (MessageEntityMention as Mention, MessageEntityBotCommand as Command,
-                               MessageEntityMentionName as MentionName, MessageEntityEmail as Email,
-                               MessageEntityUrl as URL, MessageEntityTextUrl as TextURL,
+                               MessageEntityMentionName as MentionName, MessageEntityUrl as URL,
+                               MessageEntityEmail as Email, MessageEntityTextUrl as TextURL,
                                MessageEntityBold as Bold, MessageEntityItalic as Italic,
                                MessageEntityCode as Code, MessageEntityPre as Pre,
-                               TypeMessageEntity)
+                               MessageEntityStrike as Strike, MessageEntityUnderline as Underline,
+                               MessageEntityBlockquote as Blockquote, TypeMessageEntity)
 
 from ... import user as u, puppet as pu, portal as po
 from ...types import MatrixUserID
-from ..util import html_to_unicode
 from .telegram_message import TelegramMessage, Entity, offset_length_multiply
 
 from .html_reader import HTMLNode, read_html
@@ -101,13 +101,6 @@ class MatrixParser:
             children.append(child)
         return TelegramMessage.join(children, "\n")
 
-    @classmethod
-    def blockquote_to_tmessage(cls, node: HTMLNode, ctx: RecursionContext) -> TelegramMessage:
-        msg = cls.tag_aware_parse_node(node, ctx)
-        children = msg.trim().split("\n")
-        children = [child.prepend("> ") for child in children]
-        return TelegramMessage.join(children, "\n")
-
     @classmethod
     def header_to_tmessage(cls, node: HTMLNode, ctx: RecursionContext) -> TelegramMessage:
         children = cls.node_to_tmessages(node, ctx)
@@ -122,15 +115,14 @@ class MatrixParser:
             msg.format(Bold)
         elif node.tag in ("i", "em"):
             msg.format(Italic)
+        elif node.tag in ("s", "strike", "del"):
+            msg.format(Strike)
+        elif node.tag in ("u", "ins"):
+            msg.format(Underline)
+        elif node == "blockquote":
+            msg.format(Blockquote)
         elif node.tag == "command":
             msg.format(Command)
-        elif node.tag in ("s", "strike", "del"):
-            msg.text = html_to_unicode(msg.text, "\u0336")
-        elif node.tag in ("u", "ins"):
-            msg.text = html_to_unicode(msg.text, "\u0332")
-
-        if node.tag in ("s", "strike", "del", "u", "ins"):
-            msg.entities = Entity.adjust(msg.entities, offset_length_multiply(2))
 
         return msg
 
@@ -169,10 +161,17 @@ class MatrixParser:
                 if msg.text == href
                 else msg.format(TextURL, url=href))
 
+    @classmethod
+    def blockquote_to_tmessage(cls, node: HTMLNode, ctx: RecursionContext) -> TelegramMessage:
+        msg = cls.tag_aware_parse_node(node, ctx)
+        children = msg.trim().split("\n")
+        children = [child.prepend("> ") for child in children]
+        return TelegramMessage.join(children, "\n")
+
     @classmethod
     def node_to_tmessage(cls, node: HTMLNode, ctx: RecursionContext) -> TelegramMessage:
-        if node.tag == "blockquote":
-            return cls.blockquote_to_tmessage(node, ctx)
+        if node.tag == "mx-reply":
+            return TelegramMessage("")
         elif node.tag == "ol":
             return cls.list_to_tmessage(node, ctx)
         elif node.tag == "ul":
@@ -183,6 +182,11 @@ class MatrixParser:
             return TelegramMessage("\n")
         elif node.tag in ("b", "strong", "i", "em", "s", "del", "u", "ins", "command"):
             return cls.basic_format_to_tmessage(node, ctx)
+        elif node.tag == "blockquote":
+            # Telegram already has blockquote entities in the protocol schema, but it strips them
+            # server-side and none of the official clients support them.
+            # TODO once Telegram changes that, use the above if block for blockquotes too.
+            return cls.blockquote_to_tmessage(node, ctx)
         elif node.tag == "a":
             return cls.link_to_tstring(node, ctx)
         elif node.tag == "p":

mautrix_telegram/formatter/from_matrix/telegram_message.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by

mautrix_telegram/formatter/from_telegram.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -24,7 +24,8 @@ from telethon.tl.types import (MessageEntityMention, MessageEntityMentionName, M
                                MessageEntityItalic, MessageEntityCode, MessageEntityPre,
                                MessageEntityBotCommand, MessageEntityHashtag, MessageEntityCashtag,
                                MessageEntityPhone, TypeMessageEntity, Message, PeerChannel,
-                               MessageFwdHeader, PeerUser)
+                               MessageEntityBlockquote, MessageEntityStrike, MessageFwdHeader,
+                               MessageEntityUnderline, PeerUser)
 
 from mautrix_appservice import MatrixRequestError
 from mautrix_appservice.intent_api import IntentAPI
@@ -33,20 +34,12 @@ from .. import user as u, puppet as pu, portal as po
 from ..types import TelegramID
 from ..db import Message as DBMessage
 from .util import (add_surrogates, remove_surrogates, trim_reply_fallback_html,
-                   trim_reply_fallback_text, unicode_to_html)
+                   trim_reply_fallback_text)
 
 if TYPE_CHECKING:
     from ..abstract_user import AbstractUser
-    from ..context import Context
-
-try:
-    from lxml.html.diff import htmldiff
-except ImportError:
-    htmldiff = None  # type: ignore
-
 
 log = logging.getLogger("mau.fmt.tg")  # type: logging.Logger
-should_highlight_edits = False  # type: bool
 
 
 def telegram_reply_to_matrix(evt: Message, source: 'AbstractUser') -> Dict:
@@ -54,13 +47,16 @@ def telegram_reply_to_matrix(evt: Message, source: 'AbstractUser') -> Dict:
         space = (evt.to_id.channel_id
                  if isinstance(evt, Message) and isinstance(evt.to_id, PeerChannel)
                  else source.tgid)
-        msg = DBMessage.get_by_tgid(evt.reply_to_msg_id, space)
+        msg = DBMessage.get_one_by_tgid(evt.reply_to_msg_id, space)
         if msg:
             return {
                 "m.in_reply_to": {
                     "event_id": msg.mxid,
                     "room_id": msg.mx_room,
-                }
+                },
+                "rel_type": "m.reference",
+                "event_id": msg.mxid,
+                "room_id": msg.mx_room,
             }
     return {}
 
@@ -71,30 +67,33 @@ async def _add_forward_header(source, text: str, html: Optional[str],
         html = escape(text)
     fwd_from_html, fwd_from_text = None, None
     if fwd_from.from_id:
-        user = u.User.get_by_tgid(fwd_from.from_id)
+        user = u.User.get_by_tgid(TelegramID(fwd_from.from_id))
         if user:
             fwd_from_text = user.displayname or user.mxid
-            fwd_from_html = f"<a href='https://matrix.to/#/{user.mxid}'>{fwd_from_text}</a>"
+            fwd_from_html = (f"<a href='https://matrix.to/#/{user.mxid}'>"
+                             f"{escape(fwd_from_text)}</a>")
 
         if not fwd_from_text:
             puppet = pu.Puppet.get(TelegramID(fwd_from.from_id), create=False)
             if puppet and puppet.displayname:
                 fwd_from_text = puppet.displayname or puppet.mxid
-                fwd_from_html = f"<a href='https://matrix.to/#/{puppet.mxid}'>{fwd_from_text}</a>"
+                fwd_from_html = (f"<a href='https://matrix.to/#/{puppet.mxid}'>"
+                                 f"{escape(fwd_from_text)}</a>")
 
         if not fwd_from_text:
             user = await source.client.get_entity(PeerUser(fwd_from.from_id))
             if user:
                 fwd_from_text = pu.Puppet.get_displayname(user, False)
-                fwd_from_html = f"<b>{fwd_from_text}</b>"
+                fwd_from_html = f"<b>{escape(fwd_from_text)}</b>"
     else:
         portal = po.Portal.get_by_tgid(TelegramID(fwd_from.channel_id))
         if portal:
             fwd_from_text = portal.title
             if portal.alias:
-                fwd_from_html = f"<a href='https://matrix.to/#/{portal.alias}'>{fwd_from_text}</a>"
+                fwd_from_html = (f"<a href='https://matrix.to/#/{portal.alias}'>"
+                                 f"{escape(fwd_from_text)}</a>")
             else:
-                fwd_from_html = f"<b>{fwd_from_text}</b>"
+                fwd_from_html = f"<b>{escape(fwd_from_text)}</b>"
         else:
             channel = await source.client.get_entity(PeerChannel(fwd_from.channel_id))
             if channel:
@@ -115,32 +114,19 @@ async def _add_forward_header(source, text: str, html: Optional[str],
     return text, html
 
 
-def highlight_edits(new_html: str, old_html: str) -> str:
-    # Don't include `Edit:` text in diff.
-    if old_html.startswith("<u>Edit:</u> "):
-        old_html = old_html[len("<u>Edit:</u> "):]
-
-    # Generate diff with lxml
-    new_html = htmldiff(old_html, new_html)
-
-    # Replace <ins> with <u> since Riot doesn't allow <ins>
-    new_html = new_html.replace("<ins>", "<u>").replace("</ins>", "</u>")
-    # Remove <del>s since we just want to hide deletions.
-    new_html = re.sub("<del>.+?</del>", "", new_html)
-    return new_html
-
-
 async def _add_reply_header(source: "AbstractUser", text: str, html: str, evt: Message,
-                            relates_to: Dict, main_intent: IntentAPI, is_edit: bool
-                            ) -> Tuple[str, str]:
+                            relates_to: Dict, main_intent: IntentAPI) -> Tuple[str, str]:
     space = (evt.to_id.channel_id
              if isinstance(evt, Message) and isinstance(evt.to_id, PeerChannel)
              else source.tgid)
 
-    msg = DBMessage.get_by_tgid(evt.reply_to_msg_id, space)
+    msg = DBMessage.get_one_by_tgid(evt.reply_to_msg_id, space)
     if not msg:
         return text, html
 
+    relates_to["rel_type"] = "m.reference"
+    relates_to["event_id"] = msg.mxid
+    relates_to["room_id"] = msg.mx_room
     relates_to["m.in_reply_to"] = {
         "event_id": msg.mxid,
         "room_id": msg.mx_room,
@@ -159,22 +145,14 @@ async def _add_reply_header(source: "AbstractUser", text: str, html: str, evt: M
 
         puppet = pu.Puppet.get_by_mxid(r_sender, create=False)
         r_displayname = puppet.displayname if puppet else r_sender
-        r_sender_link = f"<a href='https://matrix.to/#/{r_sender}'>{r_displayname}</a>"
-
-        if is_edit and should_highlight_edits:
-            html = highlight_edits(html or escape(text), r_html_body)
+        r_sender_link = f"<a href='https://matrix.to/#/{r_sender}'>{escape(r_displayname)}</a>"
     except (ValueError, KeyError, MatrixRequestError):
         r_sender_link = "unknown user"
         r_displayname = "unknown user"
         r_text_body = "Failed to fetch message"
         r_html_body = "<em>Failed to fetch message</em>"
 
-    if is_edit:
-        html = f"<u>Edit:</u> {html or escape(text)}"
-        text = f"Edit: {text}"
-
-    r_keyword = "In reply to" if not is_edit else "Edit to"
-    r_msg_link = f"<a href='https://matrix.to/#/{msg.mx_room}/{msg.mxid}'>{r_keyword}</a>"
+    r_msg_link = f"<a href='https://matrix.to/#/{msg.mx_room}/{msg.mxid}'>In reply to</a>"
     html = (
         f"<mx-reply><blockquote>{r_msg_link} {r_sender_link}\n{r_html_body}</blockquote></mx-reply>"
         + (html or escape(text)))
@@ -191,10 +169,10 @@ async def _add_reply_header(source: "AbstractUser", text: str, html: str, evt: M
 
 async def telegram_to_matrix(evt: Message, source: "AbstractUser",
                              main_intent: Optional[IntentAPI] = None,
-                             is_edit: bool = False, prefix_text: Optional[str] = None,
-                             prefix_html: Optional[str] = None, override_text: str = None,
-                             override_entities: List[TypeMessageEntity] = None
-                             ) -> Tuple[str, str, Dict]:
+                             prefix_text: Optional[str] = None, prefix_html: Optional[str] = None,
+                             override_text: str = None,
+                             override_entities: List[TypeMessageEntity] = None,
+                             no_reply_fallback: bool = False) -> Tuple[str, str, Dict]:
     text = add_surrogates(override_text or evt.message)
     entities = override_entities or evt.entities
     html = _telegram_entities_to_matrix_catch(text, entities) if entities else None
@@ -208,9 +186,8 @@ async def telegram_to_matrix(evt: Message, source: "AbstractUser",
     if evt.fwd_from:
         text, html = await _add_forward_header(source, text, html, evt.fwd_from)
 
-    if evt.reply_to_msg_id:
-        text, html = await _add_reply_header(source, text, html, evt, relates_to, main_intent,
-                                             is_edit)
+    if evt.reply_to_msg_id and not no_reply_fallback:
+        text, html = await _add_reply_header(source, text, html, evt, relates_to, main_intent)
 
     if isinstance(evt, Message) and evt.post and evt.post_author:
         if not html:
@@ -218,9 +195,6 @@ async def telegram_to_matrix(evt: Message, source: "AbstractUser",
         text += f"\n- {evt.post_author}"
         html += f"<br/><i>- <u>{evt.post_author}</u></i>"
 
-    html = unicode_to_html(text, html, "\u0336", "del")
-    html = unicode_to_html(text, html, "\u0332", "u")
-
     if html:
         html = html.replace("\n", "<br/>")
 
@@ -238,25 +212,39 @@ def _telegram_entities_to_matrix_catch(text: str, entities: List[TypeMessageEnti
     return "[failed conversion in _telegram_entities_to_matrix]"
 
 
-def _telegram_entities_to_matrix(text: str, entities: List[TypeMessageEntity]) -> str:
+def _telegram_entities_to_matrix(text: str, entities: List[TypeMessageEntity],
+                                 offset: int = 0, length: int = None) -> str:
     if not entities:
-        return text
+        return escape(text)
+    if length is None:
+        length = len(text)
     html = []
     last_offset = 0
-    for entity in entities:
-        if entity.offset > last_offset:
-            html.append(escape(text[last_offset:entity.offset]))
-        elif entity.offset < last_offset:
+    for i, entity in enumerate(entities):
+        if entity.offset > offset + length:
+            break
+        relative_offset = entity.offset - offset
+        if relative_offset > last_offset:
+            html.append(escape(text[last_offset:relative_offset]))
+        elif relative_offset < last_offset:
             continue
 
         skip_entity = False
-        entity_text = escape(text[entity.offset:entity.offset + entity.length])
+        entity_text = _telegram_entities_to_matrix(
+            text=text[relative_offset:relative_offset + entity.length],
+            entities=entities[i + 1:], offset=entity.offset, length=entity.length)
         entity_type = type(entity)
 
         if entity_type == MessageEntityBold:
             html.append(f"<strong>{entity_text}</strong>")
         elif entity_type == MessageEntityItalic:
             html.append(f"<em>{entity_text}</em>")
+        elif entity_type == MessageEntityUnderline:
+            html.append(f"<u>{entity_text}</u>")
+        elif entity_type == MessageEntityStrike:
+            html.append(f"<del>{entity_text}</del>")
+        elif entity_type == MessageEntityBlockquote:
+            html.append(f"<blockquote>{entity_text}</blockquote>")
         elif entity_type == MessageEntityCode:
             html.append(f"<pre><code>{entity_text}</code></pre>"
                         if "\n" in entity_text
@@ -278,8 +266,8 @@ def _telegram_entities_to_matrix(text: str, entities: List[TypeMessageEntity]) -
             html.append(f"<font color='blue'>{entity_text}</font>")
         else:
             skip_entity = True
-        last_offset = entity.offset + (0 if skip_entity else entity.length)
-    html.append(text[last_offset:])
+        last_offset = relative_offset + (0 if skip_entity else entity.length)
+    html.append(escape(text[last_offset:]))
 
     return "".join(html)
 
@@ -341,14 +329,9 @@ def _parse_url(html: List[str], entity_text: str, url: str) -> bool:
 
         portal = po.Portal.find_by_username(group)
         if portal:
-            message = DBMessage.get_by_tgid(TelegramID(msgid), portal.tgid)
+            message = DBMessage.get_one_by_tgid(TelegramID(msgid), portal.tgid)
             if message:
                 url = f"https://matrix.to/#/{portal.mxid}/{message.mxid}"
 
     html.append(f"<a href='{url}'>{entity_text}</a>")
     return False
-
-
-def init_tg(context: "Context") -> None:
-    global should_highlight_edits
-    should_highlight_edits = htmldiff and context.config["bridge.highlight_edits"]

mautrix_telegram/formatter/util.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -20,38 +20,6 @@ import struct
 import re
 
 
-def unicode_to_html(text: str, html: str, ctrl: str, tag: str) -> str:
-    if ctrl not in text:
-        return html
-    if not html:
-        html = escape(text)
-    tag_start = f"<{tag}>"
-    tag_end = f"</{tag}>"
-    characters = html.split(ctrl)
-    html = ""
-    in_tag = False
-    for char in characters:
-        if not in_tag:
-            if len(char) > 1:
-                html += char[0:-1]
-                char = char[-1]
-            html += tag_start
-            in_tag = True
-            html += char
-        else:
-            if len(char) > 1:
-                html += tag_end
-                in_tag = False
-            html += char
-    if in_tag:
-        html += tag_end
-    return html
-
-
-def html_to_unicode(text: str, ctrl: str) -> str:
-    return ctrl.join(text) + ctrl
-
-
 # add_surrogates and remove_surrogates are unicode surrogate utility functions from Telethon.
 # Licensed under the MIT license.
 # https://github.com/LonamiWebs/Telethon/blob/7cce7aa3e4c6c7019a55530391b1761d33e5a04e/telethon/helpers.py

mautrix_telegram/matrix.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -17,6 +17,7 @@
 from typing import Dict, List, Match, Optional, Set, Tuple, TYPE_CHECKING
 import logging
 import asyncio
+import time
 import re
 
 from mautrix_appservice import MatrixRequestError, IntentError
@@ -27,12 +28,21 @@ from . import user as u, portal as po, puppet as pu, commands as com
 if TYPE_CHECKING:
     from .context import Context
 
+try:
+    from prometheus_client import Histogram
+
+    EVENT_TIME = Histogram("matrix_event", "Time spent processing Matrix events",
+                           ["event_type"])
+except ImportError:
+    Histogram = None
+    EVENT_TIME = None
+
 
 class MatrixHandler:
     log = logging.getLogger("mau.mx")  # type: logging.Logger
 
     def __init__(self, context: 'Context') -> None:
-        self.az, self.db, self.config, _, self.tgbot = context.core
+        self.az, self.config, _, self.tgbot = context.core
         self.commands = com.CommandProcessor(context)  # type: com.CommandProcessor
         self.previously_typing = []  # type: List[MatrixUserID]
 
@@ -126,12 +136,31 @@ class MatrixHandler:
 
         if not inviter.whitelisted:
             await self.az.intent.send_notice(
-                room_id, text="",
-                html="You are not whitelisted to use this bridge.<br/><br/>"
+                room_id,
+                text="You are not whitelisted to use this bridge.\n\n"
                      "If you are the owner of this bridge, see the "
-                     "<code>bridge.permissions</code> section in your config file.")
+                     "`bridge.permissions` section in your config file.",
+                html="<p>You are not whitelisted to use this bridge.</p>"
+                     "<p>If you are the owner of this bridge, see the "
+                     "<code>bridge.permissions</code> section in your config file.</p>")
             await self.az.intent.leave_room(room_id)
 
+        try:
+            is_management = len(await self.az.intent.get_room_members(room_id)) == 2
+        except MatrixRequestError:
+            is_management = False
+        cmd_prefix = self.commands.command_prefix
+        text = html = "Hello, I'm a Telegram bridge bot. "
+        if is_management and inviter.puppet_whitelisted and not await inviter.is_logged_in():
+            text += f"Use `{cmd_prefix} help` for help or `{cmd_prefix} login` to log in."
+            html += (f"Use <code>{cmd_prefix} help</code> for help"
+                     f" or <code>{cmd_prefix} login</code> to log in.")
+            pass
+        else:
+            text += f"Use `{cmd_prefix} help` for help."
+            html += f"Use <code>{cmd_prefix} help</code> for help."
+        await self.az.intent.send_notice(room_id, text=text, html=html)
+
     async def handle_invite(self, room_id: MatrixRoomID, user_id: MatrixUserID,
                             inviter_mxid: MatrixUserID) -> None:
         self.log.debug(f"{inviter_mxid} invited {user_id} to {room_id}")
@@ -213,7 +242,7 @@ class MatrixHandler:
         prefix = self.config["bridge.command_prefix"]
         is_command = text.startswith(prefix)
         if is_command:
-            text = text[len(prefix) + 1:]
+            text = text[len(prefix) + 1:].lstrip()
         return is_command, text
 
     async def handle_message(self, room: MatrixRoomID, sender_id: MatrixUserID, message: Dict,
@@ -248,7 +277,7 @@ class MatrixHandler:
                 # Not enough values to unpack, i.e. no arguments
                 command = text
                 args = []
-            await self.commands.handle(room, sender, command, args, is_management,
+            await self.commands.handle(room, event_id, sender, command, args, is_management,
                                        is_portal=portal is not None)
 
     @staticmethod
@@ -370,63 +399,79 @@ class MatrixHandler:
         return (sender == self.az.bot_mxid
                 or pu.Puppet.get_id_from_mxid(sender) is not None)
 
-    async def try_handle_event(self, evt: MatrixEvent) -> None:
+    async def try_handle_ephemeral_event(self, evt: MatrixEvent) -> None:
         try:
-            await self.handle_event(evt)
+            await self.handle_ephemeral_event(evt)
         except Exception:
             self.log.exception("Error handling manually received Matrix event")
 
-    async def handle_event(self, evt: MatrixEvent) -> None:
-        if self.filter_matrix_event(evt):
-            return
-        self.log.debug("Received event: %s", evt)
+    async def handle_ephemeral_event(self, evt: MatrixEvent) -> None:
         evt_type = evt.get("type", "m.unknown")  # type: str
         room_id = evt.get("room_id", None)  # type: Optional[MatrixRoomID]
-        event_id = evt.get("event_id", None)  # type: Optional[MatrixEventID]
         sender = evt.get("sender", None)  # type: Optional[MatrixUserID]
         content = evt.get("content", {})  # type: Dict
-        if evt_type == "m.room.member":
-            state_key = evt["state_key"]  # type: MatrixUserID
-            prev_content = evt.get("unsigned", {}).get("prev_content", {})  # type: Dict
-            membership = content.get("membership", "")  # type: str
-            prev_membership = prev_content.get("membership", "leave")  # type: str
-            if membership == prev_membership:
-                match = re.compile("@(.+):(.+)").match(state_key)  # type: Match
-                mxid = match.group(0)  # type: str
-                displayname = content.get("displayname", None) or mxid  # type: str
-                prev_displayname = prev_content.get("displayname", None) or mxid  # type: str
-                if displayname != prev_displayname:
-                    await self.handle_name_change(room_id, state_key, displayname,
-                                                  prev_displayname, event_id)
-            elif membership == "invite":
-                await self.handle_invite(room_id, state_key, sender)
-            elif prev_membership == "join" and membership == "leave":
-                await self.handle_part(room_id, state_key, sender, event_id)
-            elif membership == "join":
-                await self.handle_join(room_id, state_key, event_id)
-        elif evt_type in ("m.room.message", "m.sticker"):
-            if evt_type != "m.room.message":
-                content["msgtype"] = evt_type
-            await self.handle_message(room_id, sender, content, event_id)
-        elif evt_type == "m.room.redaction":
-            await self.handle_redaction(room_id, sender, evt["redacts"])
-        elif evt_type == "m.room.power_levels":
-            prev_content = evt.get("unsigned", {}).get("prev_content", {})
-            await self.handle_power_levels(room_id, sender, evt["content"], prev_content)
-        elif evt_type in ("m.room.name", "m.room.avatar", "m.room.topic"):
-            await self.handle_room_meta(evt_type, room_id, sender, evt["content"])
-        elif evt_type == "m.room.pinned_events":
-            new_events = set(evt["content"]["pinned"])
-            try:
-                old_events = set(evt["unsigned"]["prev_content"]["pinned"])
-            except KeyError:
-                old_events = set()
-            await self.handle_room_pin(room_id, sender, new_events, old_events)
-        elif evt_type == "m.room.tombstone":
-            await self.handle_room_upgrade(room_id, evt["content"]["replacement_room"])
-        elif evt_type == "m.receipt":
+        if evt_type == "m.receipt":
             await self.handle_read_receipts(room_id, self.parse_read_receipts(content))
         elif evt_type == "m.presence":
             await self.handle_presence(sender, content.get("presence", "offline"))
         elif evt_type == "m.typing":
             await self.handle_typing(room_id, content.get("user_ids", []))
+
+    async def handle_event(self, evt: MatrixEvent) -> None:
+        if self.filter_matrix_event(evt):
+            return
+        start_time = time.time()
+        self.log.debug("Received event: %s", evt)
+        evt_type = evt.get("type", "m.unknown")  # type: str
+        room_id = evt.get("room_id", None)  # type: Optional[MatrixRoomID]
+        event_id = evt.get("event_id", None)  # type: Optional[MatrixEventID]
+        sender = evt.get("sender", None)  # type: Optional[MatrixUserID]
+        state_key = evt.get("state_key", None)
+        content = evt.get("content", {})  # type: Dict
+        if state_key is not None:
+            if evt_type == "m.room.member":
+                prev_content = evt.get("unsigned", {}).get("prev_content", {})  # type: Dict
+                membership = content.get("membership", "")  # type: str
+                prev_membership = prev_content.get("membership", "leave")  # type: str
+                if membership == prev_membership:
+                    match = re.compile("@(.+):(.+)").match(state_key)  # type: Match
+                    mxid = match.group(0)  # type: str
+                    displayname = content.get("displayname", None) or mxid  # type: str
+                    prev_displayname = prev_content.get("displayname", None) or mxid  # type: str
+                    if displayname != prev_displayname:
+                        await self.handle_name_change(room_id, state_key, displayname,
+                                                      prev_displayname, event_id)
+                elif membership == "invite":
+                    await self.handle_invite(room_id, state_key, sender)
+                elif prev_membership == "join" and membership == "leave":
+                    await self.handle_part(room_id, state_key, sender, event_id)
+                elif membership == "join":
+                    await self.handle_join(room_id, state_key, event_id)
+            elif evt_type == "m.room.power_levels":
+                prev_content = evt.get("unsigned", {}).get("prev_content", {})
+                await self.handle_power_levels(room_id, sender, evt["content"], prev_content)
+            elif evt_type in ("m.room.name", "m.room.avatar", "m.room.topic"):
+                await self.handle_room_meta(evt_type, room_id, sender, evt["content"])
+            elif evt_type == "m.room.pinned_events":
+                new_events = set(evt["content"]["pinned"])
+                try:
+                    old_events = set(evt["unsigned"]["prev_content"]["pinned"])
+                except KeyError:
+                    old_events = set()
+                await self.handle_room_pin(room_id, sender, new_events, old_events)
+            elif evt_type == "m.room.tombstone":
+                await self.handle_room_upgrade(room_id, evt["content"]["replacement_room"])
+            else:
+                return
+        else:
+            if evt_type in ("m.room.message", "m.sticker"):
+                if evt_type != "m.room.message":
+                    content["msgtype"] = evt_type
+                await self.handle_message(room_id, sender, content, event_id)
+            elif evt_type == "m.room.redaction":
+                await self.handle_redaction(room_id, sender, evt["redacts"])
+            else:
+                return
+
+        if EVENT_TIME:
+            EVENT_TIME.labels(event_type=evt_type).observe(time.time() - start_time)

mautrix_telegram/portal.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -38,36 +38,38 @@ from telethon.tl.functions.messages import (
     EditChatPhotoRequest, EditChatTitleRequest, ExportChatInviteRequest, GetFullChatRequest,
     UpdatePinnedMessageRequest, MigrateChatRequest, SetTypingRequest, EditChatAboutRequest)
 from telethon.tl.functions.channels import (
-    CreateChannelRequest, EditAdminRequest, EditBannedRequest, EditPhotoRequest,
-    EditTitleRequest, GetParticipantsRequest, InviteToChannelRequest,
-    JoinChannelRequest, LeaveChannelRequest, UpdateUsernameRequest)
+    CreateChannelRequest, EditAdminRequest, EditBannedRequest, EditPhotoRequest, EditTitleRequest,
+    GetParticipantsRequest, InviteToChannelRequest, JoinChannelRequest, LeaveChannelRequest,
+    UpdateUsernameRequest)
 from telethon.tl.functions.messages import ReadHistoryRequest as ReadMessageHistoryRequest
 from telethon.tl.functions.channels import ReadHistoryRequest as ReadChannelHistoryRequest
-from telethon.errors import ChatAdminRequiredError, ChatNotModifiedError
+from telethon.errors import (ChatAdminRequiredError, ChatNotModifiedError, PhotoExtInvalidError,
+                             PhotoInvalidDimensionsError, PhotoSaveFileInvalidError)
 from telethon.tl.patched import Message, MessageService
 from telethon.tl.types import (
-    Channel, ChatAdminRights, ChatBannedRights, ChannelFull, ChannelParticipantAdmin,
+    Channel, ChatAdminRights, ChatBannedRights, ChannelFull, ChannelParticipantAdmin, Document,
     ChannelParticipantCreator, ChannelParticipantsRecent, ChannelParticipantsSearch, Chat,
-    ChatFull, ChatInviteEmpty, ChatParticipantAdmin, ChatParticipantCreator, ChatPhoto,
-    DocumentAttributeFilename, DocumentAttributeImageSize, DocumentAttributeSticker,
-    DocumentAttributeVideo, FileLocation, GeoPoint, InputChannel, InputChatUploadedPhoto,
-    InputPeerChannel, InputPeerChat, InputPeerUser, InputUser, InputUserSelf,
-    MessageActionChannelCreate, MessageActionChatAddUser, MessageActionChatCreate,
+    ChatFull, ChatInviteEmpty, ChatParticipantAdmin, ChatParticipantCreator, ChatPhoto, Poll,
+    DocumentAttributeFilename, DocumentAttributeImageSize, DocumentAttributeSticker, PhotoEmpty,
+    DocumentAttributeVideo, GeoPoint, InputChannel, InputChatUploadedPhoto, InputPhotoFileLocation,
+    InputPeerChannel, InputPeerChat, InputPeerUser, InputUser, InputUserSelf, MessageMediaPoll,
+    MessageActionChannelCreate, MessageActionChatAddUser, MessageActionChatCreate, ChatPhotoEmpty,
     MessageActionChatDeletePhoto, MessageActionChatDeleteUser, MessageActionChatEditPhoto,
     MessageActionChatEditTitle, MessageActionChatJoinedByLink, MessageActionChatMigrateTo,
     MessageActionPinMessage, MessageActionGameScore, MessageMediaContact, MessageMediaDocument,
-    MessageMediaGeo, MessageMediaPhoto, MessageMediaUnsupported, MessageMediaGame, PeerChannel,
-    PeerChat, PeerUser, Photo, PhotoCachedSize, SendMessageCancelAction, SendMessageTypingAction,
-    TypeChannelParticipant, TypeChat, TypeChatParticipant, TypeDocumentAttribute, TypeInputPeer,
-    TypeMessageAction, TypeMessageEntity, TypePeer, TypePhotoSize, TypeUpdates, TypeUser, PhotoSize,
-    TypeUserFull, UpdateChatUserTyping, UpdateNewChannelMessage, UpdateNewMessage, UpdateUserTyping,
-    User, UserFull, MessageEntityPre)
+    MessageMediaGeo, MessageMediaPhoto, MessageMediaUnsupported, MessageMediaGame,
+    PeerChannel, PeerChat, PeerUser, Photo, PhotoCachedSize, SendMessageCancelAction,
+    SendMessageTypingAction, TypeChannelParticipant, TypeChat, TypeChatParticipant,
+    TypeDocumentAttribute, TypeInputPeer, TypeMessageAction, TypeMessageEntity, TypePeer,
+    TypePhotoSize, TypeUpdates, TypeUser, PhotoSize, TypeUserFull, UpdateChatUserTyping,
+    UpdateNewChannelMessage, UpdateNewMessage, UpdateUserTyping, User, UserFull, MessageEntityPre,
+    InputMediaUploadedDocument, InputPeerPhotoFileLocation)
 from mautrix_appservice import MatrixRequestError, IntentError, AppService, IntentAPI
 
 from .types import MatrixEventID, MatrixRoomID, MatrixUserID, TelegramID
 from .context import Context
 from .db import Portal as DBPortal, Message as DBMessage, TelegramFile as DBTelegramFile
-from .util import ignore_coro
+from .util import ignore_coro, sane_mimetypes
 from . import puppet as p, user as u, formatter, util
 
 if TYPE_CHECKING:
@@ -76,8 +78,6 @@ if TYPE_CHECKING:
     from .config import Config
     from .tgclient import MautrixTelegramClient
 
-mimetypes.init()
-
 config = None  # type: Config
 
 TypeMessage = Union[Message, MessageService]
@@ -346,7 +346,10 @@ class Portal:
                 await self.invite_to_matrix(invites or [])
             return self.mxid
         async with self._room_create_lock:
-            return await self._create_matrix_room(user, entity, invites)
+            try:
+                return await self._create_matrix_room(user, entity, invites)
+            except Exception:
+                self.log.exception("Fatal error creating Matrix room")
 
     async def _create_matrix_room(self, user: 'AbstractUser', entity: TypeChat, invites: InviteList
                                   ) -> Optional[MatrixRoomID]:
@@ -397,6 +400,11 @@ class Portal:
             "type": "m.room.power_levels",
             "content": power_levels,
         }]
+        if config["appservice.community_id"]:
+            initial_state.append({
+                "type": "m.room.related_groups",
+                "content": {"groups": [config["appservice.community_id"]]},
+            })
 
         room_id = await self.main_intent.create_room(alias=alias, is_public=public,
                                                      is_direct=direct, invitees=invites or [],
@@ -431,6 +439,10 @@ class Portal:
             levels["events_default"] = 0
         else:
             dbr = entity.default_banned_rights
+            if not dbr:
+                self.log.debug(f"default_banned_rights is None in {entity}")
+                dbr = ChatBannedRights(invite_users=True, change_info=True, pin_messages=True,
+                                       send_stickers=False, send_messages=False, until_date=0)
             levels["ban"] = 99
             levels["kick"] = 50
             levels["invite"] = 50 if dbr.invite_users else 0
@@ -440,7 +452,6 @@ class Portal:
             levels["events"]["m.room.topic"] = 50 if dbr.change_info else 0
             levels["events"][
                 "m.room.pinned_events"] = 50 if dbr.pin_messages else 0
-            levels["events"]["m.sticker"] = 50 if dbr.send_stickers else 0
             levels["events"]["m.room.power_levels"] = 75
             levels["events"]["m.room.history_visibility"] = 75
             levels["state_default"] = 50
@@ -448,6 +459,7 @@ class Portal:
             levels["events_default"] = (50 if (self.peer_type == "channel" and not entity.megagroup
                                                or entity.default_banned_rights.send_messages)
                                         else 0)
+            levels["events"]["m.sticker"] = 50 if dbr.send_stickers else levels["events_default"]
         if "users" not in levels:
             levels["users"] = {
                 self.main_intent.mxid: 100
@@ -569,7 +581,7 @@ class Portal:
         changed = await self.update_title(entity.title) or changed
 
         if isinstance(entity.photo, ChatPhoto):
-            changed = await self.update_avatar(user, entity.photo.photo_big) or changed
+            changed = await self.update_avatar(user, entity.photo) or changed
 
         if changed:
             self.save()
@@ -610,9 +622,23 @@ class Portal:
         return False
 
     @staticmethod
-    def _get_largest_photo_size(photo: Union[Photo, List[TypePhotoSize]]) -> TypePhotoSize:
-        return max(photo.sizes if isinstance(photo, Photo) else photo, key=(lambda photo2: (
-            len(photo2.bytes) if not isinstance(photo2, PhotoSize) else photo2.size)))
+    def _get_largest_photo_size(photo: Union[Photo, Document]
+                                ) -> Tuple[Optional[InputPhotoFileLocation],
+                                           Optional[TypePhotoSize]]:
+        if not photo:
+            return None, None
+        if isinstance(photo, Document) and not photo.thumbs:
+            return None, None
+        largest = max(photo.sizes if isinstance(photo, Photo) else photo.thumbs,
+                      key=(lambda photo2: (len(photo2.bytes)
+                                           if not isinstance(photo2, PhotoSize)
+                                           else photo2.size)))
+        return InputPhotoFileLocation(
+            id=photo.id,
+            access_hash=photo.access_hash,
+            file_reference=photo.file_reference,
+            thumb_size=largest.type,
+        ), largest
 
     async def remove_avatar(self, _: 'AbstractUser', save: bool = False) -> None:
         await self.main_intent.set_room_avatar(self.mxid, None)
@@ -620,11 +646,33 @@ class Portal:
         if save:
             self.save()
 
-    async def update_avatar(self, user: 'AbstractUser', photo: FileLocation,
+    async def update_avatar(self, user: 'AbstractUser',
+                            photo: Union[ChatPhoto, ChatPhotoEmpty, Photo, PhotoEmpty],
                             save: bool = False) -> bool:
-        photo_id = f"{photo.volume_id}-{photo.local_id}"
+        if isinstance(photo, ChatPhoto):
+            loc = InputPeerPhotoFileLocation(
+                peer=await self.get_input_entity(user),
+                local_id=photo.photo_big.local_id,
+                volume_id=photo.photo_big.volume_id,
+                big=True
+            )
+            photo_id = f"{loc.volume_id}-{loc.local_id}"
+        elif isinstance(photo, Photo):
+            loc, largest = self._get_largest_photo_size(photo)
+            photo_id = f"{largest.location.volume_id}-{largest.location.local_id}"
+        elif isinstance(photo, (ChatPhotoEmpty, PhotoEmpty)):
+            photo_id = ""
+            loc = None
+        else:
+            raise ValueError(f"Unknown photo type {type(photo)}")
         if self.photo_id != photo_id:
-            file = await util.transfer_file_to_matrix(user.client, self.main_intent, photo)
+            if not photo_id:
+                await self.main_intent.set_room_avatar(self.mxid, "")
+                self.photo_id = ""
+                if save:
+                    self.save()
+                return True
+            file = await util.transfer_file_to_matrix(user.client, self.main_intent, loc)
             if file:
                 await self.main_intent.set_room_avatar(self.mxid, file.mxc)
                 self.photo_id = photo_id
@@ -743,14 +791,9 @@ class Portal:
                 return body + current_extension
         except (ValueError, KeyError):
             pass
-        ext_override = {
-            "image/jpeg": ".jpg"
-        }
         if mime:
-            ext = ext_override.get(mime, mimetypes.guess_extension(mime))
-            return f"matrix_upload{ext}"
-        else:
-            return ""
+            return f"matrix_upload{sane_mimetypes.guess_extension(mime)}"
+        return ""
 
     def get_config(self, key: str) -> Any:
         local = util.recursive_get(self.local_config, key)
@@ -768,7 +811,7 @@ class Portal:
 
         tpl_args = dict(mxid=user.mxid,
                         username=user.mxid_localpart,
-                        displayname=displayname)
+                        displayname=escape_html(displayname))
         tpl_args = {**tpl_args, **(arguments or {})}
         message = Template(tpl).safe_substitute(tpl_args)
         return {
@@ -892,7 +935,7 @@ class Portal:
                                 ) -> None:
         if "formatted_body" not in message:
             message["format"] = "org.matrix.custom.html"
-            message["formatted_body"] = escape_html(message.get("body", ""))
+            message["formatted_body"] = escape_html(message.get("body", "")).replace("\n", "<br/>")
         body = message["formatted_body"]
 
         tpl = (self.get_config(f"message_formats.[{msgtype}]")
@@ -900,7 +943,7 @@ class Portal:
         displayname = await self.get_displayname(sender)
         tpl_args = dict(sender_mxid=sender.mxid,
                         sender_username=sender.mxid_localpart,
-                        sender_displayname=displayname,
+                        sender_displayname=escape_html(displayname),
                         message=body)
         message["formatted_body"] = Template(tpl).safe_substitute(tpl_args)
 
@@ -909,9 +952,14 @@ class Portal:
         msgtype = message.get("msgtype", "m.text")
         if msgtype == "m.emote":
             await self._apply_msg_format(sender, msgtype, message)
+            if "m.new_content" in message:
+                await self._apply_msg_format(sender, msgtype, message["m.new_content"])
+                message["m.new_content"]["msgtype"] = "m.text"
             message["msgtype"] = "m.text"
         elif use_relaybot:
             await self._apply_msg_format(sender, msgtype, message)
+            if "m.new_content" in message:
+                await self._apply_msg_format(sender, msgtype, message["m.new_content"])
 
     @staticmethod
     def _matrix_event_to_entities(event: Dict[str, Any]
@@ -943,15 +991,26 @@ class Portal:
             return None
 
     async def _handle_matrix_text(self, sender_id: TelegramID, event_id: MatrixEventID,
-                                  space: TelegramID, client: 'MautrixTelegramClient', message: Dict,
-                                  reply_to: TelegramID) -> None:
+                                  space: TelegramID, client: 'MautrixTelegramClient',
+                                  message: Dict, reply_to: TelegramID) -> None:
         lock = self.require_send_lock(sender_id)
         async with lock:
             lp = self.get_config("telegram_link_preview")
+            relates_to = message.get("m.relates_to", None) or {}
+            if relates_to.get("rel_type", None) == "m.replace":
+                orig_msg = DBMessage.get_by_mxid(relates_to.get("event_id", ""), self.mxid, space)
+                if orig_msg and "m.new_content" in message:
+                    message = message["m.new_content"]
+                    formatter.matrix_reply_to_telegram(message, space, room_id=self.mxid)
+                    response = await client.edit_message(self.peer, orig_msg.tgid, message,
+                                                         parse_mode=self._matrix_event_to_entities,
+                                                         link_preview=lp)
+                    self._add_telegram_message_to_db(event_id, space, -1, response)
+                    return
             response = await client.send_message(self.peer, message, reply_to=reply_to,
                                                  parse_mode=self._matrix_event_to_entities,
                                                  link_preview=lp)
-            self._add_telegram_message_to_db(event_id, space, response)
+            self._add_telegram_message_to_db(event_id, space, 0, response)
 
     async def _handle_matrix_file(self, msgtype: str, sender_id: TelegramID,
                                   event_id: MatrixEventID, space: TelegramID,
@@ -981,12 +1040,28 @@ class Portal:
 
         caption = message["body"] if message["body"].lower() != file_name.lower() else None
 
-        media = await client.upload_file_direct(file, mime, attributes, file_name)
+        media = await client.upload_file_direct(
+            file, mime, attributes, file_name,
+            max_image_size=config["bridge.image_as_file_size"] * 1000 ** 2)
         lock = self.require_send_lock(sender_id)
         async with lock:
-            response = await client.send_media(self.peer, media, reply_to=reply_to,
-                                               caption=caption)
-            self._add_telegram_message_to_db(event_id, space, response)
+            relates_to = message.get("m.relates_to", None) or {}
+            if relates_to.get("rel_type", None) == "m.replace":
+                orig_msg = DBMessage.get_by_mxid(relates_to.get("event_id", ""), self.mxid, space)
+                if orig_msg:
+                    response = await client.edit_message(self.peer, orig_msg.tgid,
+                                                         caption, file=media)
+                    self._add_telegram_message_to_db(event_id, space, -1, response)
+                    return
+            try:
+                response = await client.send_media(self.peer, media, reply_to=reply_to,
+                                                   caption=caption)
+            except (PhotoInvalidDimensionsError, PhotoSaveFileInvalidError, PhotoExtInvalidError):
+                media = InputMediaUploadedDocument(file=media.file, mime_type=mime,
+                                                   attributes=attributes)
+                response = await client.send_media(self.peer, media, reply_to=reply_to,
+                                                   caption=caption)
+            self._add_telegram_message_to_db(event_id, space, 0, response)
 
     async def _handle_matrix_location(self, sender_id: TelegramID, event_id: MatrixEventID,
                                       space: TelegramID, client: 'MautrixTelegramClient',
@@ -1002,19 +1077,31 @@ class Portal:
 
         lock = self.require_send_lock(sender_id)
         async with lock:
+            relates_to = message.get("m.relates_to", None) or {}
+            if relates_to.get("rel_type", None) == "m.replace":
+                orig_msg = DBMessage.get_by_mxid(relates_to.get("event_id", ""), self.mxid, space)
+                if orig_msg:
+                    response = await client.edit_message(self.peer, orig_msg.tgid,
+                                                         caption, file=media)
+                    self._add_telegram_message_to_db(event_id, space, -1, response)
+                    return
             response = await client.send_media(self.peer, media, reply_to=reply_to,
                                                caption=caption, entities=entities)
-            self._add_telegram_message_to_db(event_id, space, response)
+            self._add_telegram_message_to_db(event_id, space, 0, response)
 
     def _add_telegram_message_to_db(self, event_id: MatrixEventID, space: TelegramID,
-                                    response: TypeMessage) -> None:
+                                    edit_index: int, response: TypeMessage) -> None:
         self.log.debug("Handled Matrix message: %s", response)
-        self.is_duplicate(response, (event_id, space))
+        self.is_duplicate(response, (event_id, space), force_hash=edit_index != 0)
+        if edit_index < 0:
+            prev_edit = DBMessage.get_one_by_tgid(TelegramID(response.id), space, -1)
+            edit_index = prev_edit.edit_index + 1
         DBMessage(
             tgid=TelegramID(response.id),
             tg_space=space,
             mx_room=self.mxid,
-            mxid=event_id).insert()
+            mxid=event_id,
+            edit_index=edit_index).insert()
 
     async def handle_matrix_message(self, sender: 'u.User', message: Dict[str, Any],
                                     event_id: MatrixEventID) -> None:
@@ -1078,7 +1165,10 @@ class Portal:
         message = DBMessage.get_by_mxid(event_id, self.mxid, space)
         if not message:
             return
-        await real_deleter.client.delete_messages(self.peer, [message.tgid])
+        if message.edit_index == 0:
+            await real_deleter.client.delete_messages(self.peer, [message.tgid])
+        else:
+            self.log.debug(f"Ignoring deletion of edit event {message.mxid} in {message.mx_room}")
 
     async def _update_telegram_power_level(self, sender: 'u.User', user_id: TelegramID,
                                            level: int) -> None:
@@ -1142,7 +1232,7 @@ class Portal:
 
         file = await self.main_intent.download_file(url)
         mime = magic.from_buffer(file, mime=True)
-        ext = mimetypes.guess_extension(mime)
+        ext = sane_mimetypes.guess_extension(mime)
         uploaded = await sender.client.upload_file(file, file_name=f"avatar{ext}", use_cache=False)
         photo = InputChatUploadedPhoto(file=uploaded)
 
@@ -1157,8 +1247,8 @@ class Portal:
                                and isinstance(update.message, MessageService)
                                and isinstance(update.message.action, MessageActionChatEditPhoto))
             if is_photo_update:
-                loc = self._get_largest_photo_size(update.message.action.photo).location
-                self.photo_id = f"{loc.volume_id}-{loc.local_id}"
+                loc, size = self._get_largest_photo_size(update.message.action.photo)
+                self.photo_id = f"{size.location.volume_id}-{size.location.local_id}"
                 self.save()
                 break
 
@@ -1249,8 +1339,13 @@ class Portal:
 
         invites = await self._get_telegram_users_in_matrix_room()
         if len(invites) < 2:
-            raise ValueError("Not enough Telegram users to create a chat")
-
+            if self.bot is not None:
+                info, mxid = await self.bot.get_me()
+                raise ValueError("Not enough Telegram users to create a chat. "
+                                 "Invite more Telegram ghost users to the room, such as the "
+                                 f"relaybot ([{info.first_name}](https://matrix.to/#/{mxid})).")
+            raise ValueError("Not enough Telegram users to create a chat. "
+                             "Invite more Telegram ghost users to the room.")
         if self.peer_type == "chat":
             response = await source.client(CreateChatRequest(title=self.title, users=invites))
             entity = response.chats[0]
@@ -1302,12 +1397,14 @@ class Portal:
     def get_external_url(self, evt: Message) -> Optional[str]:
         if self.peer_type == "channel" and self.username is not None:
             return f"https://t.me/{self.username}/{evt.id}"
+        elif self.peer_type != "user":
+            return f"https://t.me/c/{self.tgid}/{evt.id}"
         return None
 
     async def handle_telegram_photo(self, source: 'AbstractUser', intent: IntentAPI, evt: Message,
                                     relates_to: Dict = None) -> Optional[Dict]:
-        largest_size = self._get_largest_photo_size(evt.media.photo)
-        file = await util.transfer_file_to_matrix(source.client, intent, largest_size.location)
+        loc, largest_size = self._get_largest_photo_size(evt.media.photo)
+        file = await util.transfer_file_to_matrix(source.client, intent, loc)
         if not file:
             return None
         if self.get_config("inline_images") and (evt.message
@@ -1328,18 +1425,16 @@ class Portal:
             "orientation": 0,
             "mimetype": file.mime_type,
         }
-        ext_override = {
-            "image/jpeg": ".jpg"
-        }
-        name = "image" + ext_override.get(file.mime_type, mimetypes.guess_extension(file.mime_type))
+        name = f"image{sane_mimetypes.guess_extension(file.mime_type)}"
         await intent.set_typing(self.mxid, is_typing=False)
         result = await intent.send_image(self.mxid, file.mxc, info=info, text=name,
                                          relates_to=relates_to, timestamp=evt.date,
                                          external_url=self.get_external_url(evt))
         if evt.message:
-            text, html, _ = await formatter.telegram_to_matrix(evt, source, self.main_intent)
-            await intent.send_text(self.mxid, text, html=html, timestamp=evt.date,
-                                   external_url=self.get_external_url(evt))
+            text, html, _ = await formatter.telegram_to_matrix(evt, source, self.main_intent,
+                                                               no_reply_fallback=True)
+            result = await intent.send_text(self.mxid, text, html=html, timestamp=evt.date,
+                                            external_url=self.get_external_url(evt))
         return result
 
     @staticmethod
@@ -1365,7 +1460,7 @@ class Portal:
 
     @staticmethod
     def _parse_telegram_document_meta(evt: Message, file: DBTelegramFile, attrs: Dict,
-                                      thumb: TypePhotoSize) -> Tuple[Dict, str]:
+                                      thumb_size: TypePhotoSize) -> Tuple[Dict, str]:
         document = evt.media.document
         name = evt.message or attrs["name"]
         if attrs["is_sticker"]:
@@ -1376,7 +1471,11 @@ class Portal:
                 except ValueError:
                     name = alt
 
-        mime_type = document.mime_type or file.mime_type
+        generic_types = ("text/plain", "application/octet-stream")
+        if file.mime_type in generic_types and document.mime_type not in generic_types:
+            mime_type = document.mime_type or file.mime_type
+        else:
+            mime_type = file.mime_type or document.mime_type
         info = {
             "size": file.size,
             "mimetype": mime_type,
@@ -1393,8 +1492,8 @@ class Portal:
             info["thumbnail_url"] = file.thumbnail.mxc
             info["thumbnail_info"] = {
                 "mimetype": file.thumbnail.mime_type,
-                "h": file.thumbnail.height or thumb.h,
-                "w": file.thumbnail.width or thumb.w,
+                "h": file.thumbnail.height or thumb_size.h,
+                "w": file.thumbnail.width or thumb_size.w,
                 "size": file.thumbnail.size,
             }
 
@@ -1403,18 +1502,25 @@ class Portal:
     async def handle_telegram_document(self, source: 'AbstractUser', intent: IntentAPI,
                                        evt: Message, relates_to: dict = None) -> Optional[Dict]:
         document = evt.media.document
+
         attrs = self._parse_telegram_document_attributes(document.attributes)
 
-        thumb = self._get_largest_photo_size(document.thumbs)
-        if not isinstance(thumb, (PhotoSize, PhotoCachedSize)):
-            self.log.debug(f"Unsupported thumbnail type {type(thumb)}")
-            thumb = None
-        file = await util.transfer_file_to_matrix(source.client, intent, document, thumb,
+        if document.size > config["bridge.max_document_size"] * 1000 ** 2:
+            name = attrs["name"] or ""
+            caption = f"\n{evt.message}" if evt.message else ""
+            return await intent.send_notice(self.mxid, f"Too large file {name}{caption}")
+
+        thumb_loc, thumb_size = self._get_largest_photo_size(document)
+        if thumb_size and not isinstance(thumb_size, (PhotoSize, PhotoCachedSize)):
+            self.log.debug(f"Unsupported thumbnail type {type(thumb_size)}")
+            thumb_loc = None
+            thumb_size = None
+        file = await util.transfer_file_to_matrix(source.client, intent, document, thumb_loc,
                                                   is_sticker=attrs["is_sticker"])
         if not file:
             return None
 
-        info, name = self._parse_telegram_document_meta(evt, file, attrs, thumb)
+        info, name = self._parse_telegram_document_meta(evt, file, attrs, thumb_size)
 
         await intent.set_typing(self.mxid, is_typing=False)
 
@@ -1481,7 +1587,7 @@ class Portal:
                                       external_url=self.get_external_url(evt))
 
     async def handle_telegram_unsupported(self, source: 'AbstractUser', intent: IntentAPI,
-                                          evt: Message, _: dict = None) -> dict:
+                                          evt: Message, relates_to: dict = None) -> dict:
         override_text = ("This message is not supported on your version of Mautrix-Telegram. "
                          "Please check https://github.com/tulir/mautrix-telegram or ask your "
                          "bridge administrator about possible updates.")
@@ -1497,33 +1603,65 @@ class Portal:
             "net.maunium.telegram.unsupported": True,
         }, timestamp=evt.date, external_url=self.get_external_url(evt))
 
+    async def handle_telegram_poll(self, source: 'AbstractUser', intent: IntentAPI, evt: Message,
+                                   relates_to: dict) -> dict:
+        poll = evt.media.poll  # type: Poll
+        poll_id = self._encode_msgid(source, evt)
+
+        _n = 0
+
+        def n() -> int:
+            nonlocal _n
+            _n += 1
+            return _n
+
+        text = (f"Poll: {poll.question}\n"
+                + "\n".join(f"{n()}. {answer.text}" for answer in poll.answers) +
+                "\n"
+                f"Vote with !tg vote {poll_id} <choice number>")
+
+        html = (f"<strong>Poll</strong>: {poll.question}<br/>\n"
+                f"<ol>"
+                + "\n".join(f"<li>{answer.text}</li>"
+                            for answer in poll.answers) +
+                "</ol>\n"
+                f"Vote with <code>!tg vote {poll_id} &lt;choice number&gt;</code>")
+        await intent.set_typing(self.mxid, is_typing=False)
+        return await intent.send_text(self.mxid, text, html=html, relates_to=relates_to,
+                                      msgtype="m.text", timestamp=evt.date,
+                                      external_url=self.get_external_url(evt))
+
     @staticmethod
     def _int_to_bytes(i: int) -> bytes:
         hex_value = "{0:010x}".format(i)
         return codecs.decode(hex_value, "hex_codec")
 
-    async def handle_telegram_game(self, source: 'AbstractUser', intent: IntentAPI,
-                                   evt: Message, _: dict = None):
-        game = evt.media.game
+    def _encode_msgid(self, source: 'AbstractUser', evt: Message) -> str:
         if self.peer_type == "channel":
-            play_id = base64.b64encode(b"c"
-                                       + self._int_to_bytes(self.tgid)
-                                       + self._int_to_bytes(evt.id))
+            play_id = (b"c"
+                       + self._int_to_bytes(self.tgid)
+                       + self._int_to_bytes(evt.id))
         elif self.peer_type == "chat":
-            play_id = base64.b64encode(b"g"
-                                       + self._int_to_bytes(self.tgid)
-                                       + self._int_to_bytes(evt.id)
-                                       + self._int_to_bytes(source.tgid))
+            play_id = (b"g"
+                       + self._int_to_bytes(self.tgid)
+                       + self._int_to_bytes(evt.id)
+                       + self._int_to_bytes(source.tgid))
         elif self.peer_type == "user":
-            play_id = base64.b64encode(b"u"
-                                       + self._int_to_bytes(self.tgid)
-                                       + self._int_to_bytes(evt.id))
+            play_id = (b"u"
+                       + self._int_to_bytes(self.tgid)
+                       + self._int_to_bytes(evt.id))
         else:
             raise ValueError("Portal has invalid peer type")
-        play_id = play_id.decode("utf-8").rstrip("=")
+        return base64.b64encode(play_id).decode("utf-8").rstrip("=")
+
+    async def handle_telegram_game(self, source: 'AbstractUser', intent: IntentAPI,
+                                   evt: Message, relates_to: dict = None):
+        game = evt.media.game
+        play_id = self._encode_msgid(source, evt)
         command = f"!tg play {play_id}"
         override_text = f"Run {command} in your bridge management room to play {game.title}"
-        override_entities = [MessageEntityPre(offset=len("Run "), length=len(command), language="")]
+        override_entities = [
+            MessageEntityPre(offset=len("Run "), length=len(command), language="")]
         text, html, relates_to = await formatter.telegram_to_matrix(
             evt, source, self.main_intent,
             override_text=override_text, override_entities=override_entities)
@@ -1541,9 +1679,6 @@ class Portal:
                                    evt: Message) -> None:
         if not self.mxid:
             return
-        elif not self.get_config("edits_as_replies"):
-            self.log.debug("Edits as replies disabled, ignoring edit event...")
-            return
         elif hasattr(evt, "media") and isinstance(evt.media, (MessageMediaGame,)):
             self.log.debug("Ignoring game message edit event")
             return
@@ -1561,28 +1696,52 @@ class Portal:
         if duplicate_found:
             mxid, other_tg_space = duplicate_found
             if tg_space != other_tg_space:
-                DBMessage.update_by_tgid(TelegramID(evt.id), tg_space,
-                                         mxid=mxid,
-                                         mx_room=self.mxid)
+                prev_edit_msg = DBMessage.get_one_by_tgid(TelegramID(evt.id), tg_space, -1)
+                if not prev_edit_msg:
+                    return
+                DBMessage(mxid=mxid, mx_room=self.mxid, tg_space=tg_space, tgid=evt.id,
+                          edit_index=prev_edit_msg.edit_index + 1).insert()
             return
 
-        evt.reply_to_msg_id = evt.id
-        text, html, relates_to = await formatter.telegram_to_matrix(evt, source, self.main_intent,
-                                                                    is_edit=True)
-        intent = sender.intent if sender else self.main_intent
-        await intent.set_typing(self.mxid, is_typing=False)
-        response = await intent.send_text(self.mxid, text, html=html, relates_to=relates_to,
-                                          external_url=self.get_external_url(evt))
-
-        mxid = response["event_id"]
-
-        msg = DBMessage.get_by_tgid(TelegramID(evt.id), tg_space)
-        if not msg:
+        text, html, _ = await formatter.telegram_to_matrix(evt, source, self.main_intent,
+                                                           no_reply_fallback=True)
+        editing_msg = DBMessage.get_one_by_tgid(TelegramID(evt.id), tg_space)
+        if not editing_msg:
             self.log.info(f"Didn't find edited message {evt.id}@{tg_space} (src {source.tgid}) "
                           "in database.")
-            # Oh crap
             return
-        msg.update(mxid=mxid, mx_room=self.mxid)
+
+        msgtype = ("m.notice"
+                   if sender and sender.is_bot and self.get_config("bot_messages_as_notices")
+                   else "m.text")
+        content = {
+            "body": f"Edit: {text}",
+            "msgtype": msgtype,
+            "format": "org.matrix.custom.html",
+            "formatted_body": (f"<a href='https://matrix.to/#/{editing_msg.mx_room}/"
+                               f"{editing_msg.mxid}'>Edit</a>: "
+                               f"{html or escape_html(text)}"),
+            "external_url": self.get_external_url(evt),
+            "m.new_content": {
+                "body": text,
+                "msgtype": "m.text",
+                **({"format": "org.matrix.custom.html",
+                    "formatted_body": html} if html else {}),
+            },
+            "m.relates_to": {
+                "rel_type": "m.replace",
+                "event_id": editing_msg.mxid,
+            },
+        }
+
+        intent = sender.intent if sender else self.main_intent
+        await intent.set_typing(self.mxid, is_typing=False)
+        response = await intent.send_message(self.mxid, content)
+        mxid = response["event_id"]
+
+        prev_edit_msg = DBMessage.get_one_by_tgid(TelegramID(evt.id), tg_space, -1) or editing_msg
+        DBMessage(mxid=mxid, mx_room=self.mxid, tg_space=tg_space, tgid=evt.id,
+                  edit_index=prev_edit_msg.edit_index + 1).insert()
         DBMessage.update_by_mxid(temporary_identifier, self.mxid, mxid=mxid)
 
     async def handle_telegram_message(self, source: 'AbstractUser', sender: p.Puppet,
@@ -1606,11 +1765,11 @@ class Portal:
                            f"as it was already handled (in space {other_tg_space})")
             if tg_space != other_tg_space:
                 DBMessage(tgid=TelegramID(evt.id), mx_room=self.mxid, mxid=mxid,
-                          tg_space=tg_space).insert()
+                          tg_space=tg_space, edit_index=0).insert()
             return
 
         if self.dedup_pre_db_check and self.peer_type == "channel":
-            msg = DBMessage.get_by_tgid(TelegramID(evt.id), tg_space)
+            msg = DBMessage.get_one_by_tgid(TelegramID(evt.id), tg_space)
             if msg:
                 self.log.debug(f"Ignoring message {evt.id} (src {source.tgid}) as it was already"
                                f"handled into {msg.mxid}. This duplicate was catched in the db "
@@ -1619,13 +1778,13 @@ class Portal:
                 return
 
         if sender and not sender.displayname:
-            self.log.debug(f"Telegram user {sender.tgid} sent a message, but doesn't have a"
+            self.log.debug(f"Telegram user {sender.tgid} sent a message, but doesn't have a "
                            "displayname, updating info...")
             entity = await source.client.get_entity(PeerUser(sender.tgid))
             await sender.update_info(source, entity)
 
-        allowed_media = (MessageMediaPhoto, MessageMediaDocument, MessageMediaGeo, MessageMediaGame,
-                         MessageMediaUnsupported)
+        allowed_media = (MessageMediaPhoto, MessageMediaDocument, MessageMediaGeo,
+                         MessageMediaGame, MessageMediaPoll, MessageMediaUnsupported)
         media = evt.media if hasattr(evt, "media") and isinstance(evt.media,
                                                                   allowed_media) else None
         intent = sender.intent if sender else self.main_intent
@@ -1637,6 +1796,7 @@ class Portal:
                 MessageMediaPhoto: self.handle_telegram_photo,
                 MessageMediaDocument: self.handle_telegram_document,
                 MessageMediaGeo: self.handle_telegram_location,
+                MessageMediaPoll: self.handle_telegram_poll,
                 MessageMediaUnsupported: self.handle_telegram_unsupported,
                 MessageMediaGame: self.handle_telegram_game,
             }[type(media)](source, intent, evt,
@@ -1664,7 +1824,7 @@ class Portal:
         self.log.debug("Handled Telegram message: %s", evt)
         try:
             DBMessage(tgid=TelegramID(evt.id), mx_room=self.mxid, mxid=mxid,
-                      tg_space=tg_space).insert()
+                      tg_space=tg_space, edit_index=0).insert()
             DBMessage.update_by_mxid(temporary_identifier, self.mxid, mxid=mxid)
         except IntegrityError as e:
             self.log.exception(f"{e.__class__.__name__} while saving message mapping. "
@@ -1697,8 +1857,7 @@ class Portal:
         if isinstance(action, MessageActionChatEditTitle):
             await self.update_title(action.title, save=True)
         elif isinstance(action, MessageActionChatEditPhoto):
-            largest_size = self._get_largest_photo_size(action.photo)
-            await self.update_avatar(source, largest_size.location, save=True)
+            await self.update_avatar(source, action.photo, save=True)
         elif isinstance(action, MessageActionChatDeletePhoto):
             await self.remove_avatar(source, save=True)
         elif isinstance(action, MessageActionChatAddUser):
@@ -1743,7 +1902,7 @@ class Portal:
         self._temp_pinned_message_id = None
         self._temp_pinned_message_sender = None
 
-        message = DBMessage.get_by_tgid(msg_id, self._temp_pinned_message_id_space)
+        message = DBMessage.get_one_by_tgid(msg_id, self._temp_pinned_message_id_space)
         if message:
             await intent.set_pinned_messages(self.mxid, [message.mxid])
         else:
@@ -1865,7 +2024,7 @@ class Portal:
             existing.delete()
         except KeyError:
             pass
-        self.db_instance.update(tgid=new_id, tg_receiver=new_id)
+        self.db_instance.update(tgid=new_id, tg_receiver=new_id, peer_type=self.peer_type)
         old_id = self.tgid
         self.tgid = new_id
         self.tg_receiver = new_id
@@ -2002,7 +2161,7 @@ class Portal:
 
 def init(context: Context) -> None:
     global config
-    Portal.az, _, config, Portal.loop, Portal.bot = context.core
+    Portal.az, config, Portal.loop, Portal.bot = context.core
     Portal.max_initial_member_sync = config["bridge.max_initial_member_sync"]
     Portal.sync_channel_members = config["bridge.sync_channel_members"]
     Portal.sync_matrix_state = config["bridge.sync_matrix_state"]

mautrix_telegram/puppet.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -14,8 +14,7 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import (Awaitable, Coroutine, Dict, List, Iterable, Optional, Pattern, Union,
-                    TYPE_CHECKING)
+from typing import Awaitable, Any, Dict, List, Iterable, Optional, Pattern, Union, TYPE_CHECKING
 from difflib import SequenceMatcher
 from enum import Enum
 from aiohttp import ServerDisconnectedError
@@ -23,9 +22,8 @@ import asyncio
 import logging
 import re
 
-from sqlalchemy import orm
-
-from telethon.tl.types import UserProfilePhoto, User, FileLocation, UpdateUserName, PeerUser
+from telethon.tl.types import (UserProfilePhoto, User, UpdateUserName, PeerUser, TypeInputPeer,
+                               InputPeerPhotoFileLocation, UserProfilePhotoEmpty)
 from mautrix_appservice import AppService, IntentAPI, IntentError, MatrixRequestError
 
 from .types import MatrixUserID, TelegramID
@@ -45,7 +43,6 @@ config = None  # type: Config
 
 class Puppet:
     log = logging.getLogger("mau.puppet")  # type: logging.Logger
-    db = None  # type: orm.Session
     az = None  # type: AppService
     mx = None  # type: MatrixHandler
     loop = None  # type: asyncio.AbstractEventLoop
@@ -65,6 +62,7 @@ class Puppet:
                  photo_id: Optional[str] = None,
                  is_bot: bool = False,
                  is_registered: bool = False,
+                 disable_updates: bool = False,
                  db_instance: Optional[DBPuppet] = None) -> None:
         self.id = id  # type: TelegramID
         self.access_token = access_token  # type: Optional[str]
@@ -77,6 +75,7 @@ class Puppet:
         self.photo_id = photo_id  # type: Optional[str]
         self.is_bot = is_bot  # type: bool
         self.is_registered = is_registered  # type: bool
+        self.disable_updates = disable_updates  # type: bool
         self._db_instance = db_instance  # type: Optional[DBPuppet]
 
         self.default_mxid_intent = self.az.intent.user(self.default_mxid)
@@ -115,6 +114,9 @@ class Puppet:
         match = regex.match(self.displayname)
         return match.group(1) or self.displayname
 
+    def get_input_entity(self, user: 'AbstractUser') -> Awaitable[TypeInputPeer]:
+        return user.client.get_input_entity(PeerUser(user_id=self.tgid))
+
     # region Custom puppet management
     def _fresh_intent(self) -> IntentAPI:
         return (self.az.intent.user(self.custom_mxid, self.access_token)
@@ -219,13 +221,13 @@ class Puppet:
         return new_events
 
     def handle_sync(self, presence: List, ephemeral: Dict) -> None:
-        presence_events = [self.mx.try_handle_event(event) for event in presence]
+        presence_events = [self.mx.try_handle_ephemeral_event(event) for event in presence]
 
         for room_id, events in ephemeral.items():
             for event in events:
                 event["room_id"] = room_id
 
-        ephemeral_events = [self.mx.try_handle_event(event)
+        ephemeral_events = [self.mx.try_handle_ephemeral_event(event)
                             for events in ephemeral.values()
                             for event in self.filter_events(events)]
 
@@ -285,23 +287,26 @@ class Puppet:
         return DBPuppet(id=self.id, access_token=self.access_token, custom_mxid=self.custom_mxid,
                         username=self.username, displayname=self.displayname,
                         displayname_source=self.displayname_source, photo_id=self.photo_id,
-                        is_bot=self.is_bot, matrix_registered=self.is_registered)
+                        is_bot=self.is_bot, matrix_registered=self.is_registered,
+                        disable_updates=self.disable_updates)
 
     @classmethod
     def from_db(cls, db_puppet: DBPuppet) -> 'Puppet':
         return Puppet(db_puppet.id, db_puppet.access_token, db_puppet.custom_mxid,
                       db_puppet.username, db_puppet.displayname, db_puppet.displayname_source,
                       db_puppet.photo_id, db_puppet.is_bot, db_puppet.matrix_registered,
-                      db_instance=db_puppet)
+                      db_puppet.disable_updates, db_instance=db_puppet)
 
     def save(self) -> None:
         self.db_instance.update(access_token=self.access_token, custom_mxid=self.custom_mxid,
                                 username=self.username, displayname=self.displayname,
                                 displayname_source=self.displayname_source, photo_id=self.photo_id,
-                                is_bot=self.is_bot, matrix_registered=self.is_registered)
+                                is_bot=self.is_bot, matrix_registered=self.is_registered,
+                                disable_updates=self.disable_updates)
 
     # endregion
     # region Info updating
+
     def similarity(self, query: str) -> int:
         username_similarity = (SequenceMatcher(None, self.username, query).ratio()
                                if self.username else 0)
@@ -338,6 +343,8 @@ class Puppet:
             displayname=name)
 
     async def update_info(self, source: 'AbstractUser', info: User) -> None:
+        if self.disable_updates:
+            return
         changed = False
         if self.username != info.username:
             self.username = info.username
@@ -345,7 +352,7 @@ class Puppet:
 
         changed = await self.update_displayname(source, info) or changed
         if isinstance(info.photo, UserProfilePhoto):
-            changed = await self.update_avatar(source, info.photo.photo_big) or changed
+            changed = await self.update_avatar(source, info.photo) or changed
 
         self.is_bot = info.bot
 
@@ -354,33 +361,68 @@ class Puppet:
 
     async def update_displayname(self, source: 'AbstractUser', info: Union[User, UpdateUserName]
                                  ) -> bool:
-        ignore_source = (not source.is_relaybot
-                         and self.displayname_source is not None
-                         and self.displayname_source != source.tgid)
-        if ignore_source:
+        if self.disable_updates:
             return False
-        if isinstance(info, UpdateUserName):
+        allow_source = (source.is_relaybot
+                        or self.displayname_source == source.tgid
+                        # No displayname source, so just trust anything
+                        or self.displayname_source is None
+                        # No phone -> not in contact list -> can't set custom name
+                        or (isinstance(info, User) and info.phone is None))
+        if not allow_source:
+            return False
+        elif isinstance(info, UpdateUserName):
             info = await source.client.get_entity(PeerUser(self.tgid))
 
         displayname = self.get_displayname(info)
         if displayname != self.displayname:
-            await self.default_mxid_intent.set_display_name(displayname)
             self.displayname = displayname
             self.displayname_source = source.tgid
+            try:
+                await self.default_mxid_intent.set_display_name(displayname[:100])
+            except MatrixRequestError:
+                self.log.exception("Failed to set displayname")
+                self.displayname = ""
+                self.displayname_source = None
             return True
         elif source.is_relaybot or self.displayname_source is None:
             self.displayname_source = source.tgid
             return True
         return False
 
-    async def update_avatar(self, source: 'AbstractUser', photo: FileLocation) -> bool:
-        photo_id = f"{photo.volume_id}-{photo.local_id}"
+    async def update_avatar(self, source: 'AbstractUser',
+                            photo: Union[UserProfilePhoto, UserProfilePhotoEmpty]) -> bool:
+        if self.disable_updates:
+            return False
+
+        if isinstance(photo, UserProfilePhotoEmpty):
+            photo_id = ""
+        else:
+            photo_id = str(photo.photo_id)
         if self.photo_id != photo_id:
-            file = await util.transfer_file_to_matrix(source.client, self.default_mxid_intent,
-                                                      photo)
+            if not photo_id:
+                self.photo_id = ""
+                try:
+                    await self.default_mxid_intent.set_avatar("")
+                except MatrixRequestError:
+                    self.log.exception("Failed to set avatar")
+                    self.photo_id = ""
+                return True
+
+            loc = InputPeerPhotoFileLocation(
+                peer=await self.get_input_entity(source),
+                local_id=photo.photo_big.local_id,
+                volume_id=photo.photo_big.volume_id,
+                big=True
+            )
+            file = await util.transfer_file_to_matrix(source.client, self.default_mxid_intent, loc)
             if file:
-                await self.default_mxid_intent.set_avatar(file.mxc)
                 self.photo_id = photo_id
+                try:
+                    await self.default_mxid_intent.set_avatar(file.mxc)
+                except MatrixRequestError:
+                    self.log.exception("Failed to set avatar")
+                    self.photo_id = ""
                 return True
         return False
 
@@ -400,8 +442,7 @@ class Puppet:
 
         if create:
             puppet = cls(tgid)
-            cls.db.add(puppet.db_instance)
-            cls.db.commit()
+            puppet.db_instance.insert()
             return puppet
 
         return None
@@ -481,9 +522,9 @@ class Puppet:
     # endregion
 
 
-def init(context: 'Context') -> List[Coroutine]:  # [None, None, PuppetError]
+def init(context: 'Context') -> List[Awaitable[Any]]:  # [None, None, PuppetError]
     global config
-    Puppet.az, Puppet.db, config, Puppet.loop, _ = context.core
+    Puppet.az, config, Puppet.loop, _ = context.core
     Puppet.mx = context.mx
     Puppet.username_template = config.get("bridge.username_template", "telegram_{userid}")
     Puppet.hs_domain = config["homeserver"]["domain"]

mautrix_telegram/scripts/dbms_migrate/__main__.py

@@ -11,12 +11,19 @@ parser.add_argument("-f", "--from-url", type=str, required=True, metavar="<url>"
                     help="the old database path")
 parser.add_argument("-t", "--to-url", type=str, required=True, metavar="<url>",
                     help="the new database path")
+parser.add_argument("-v", "--verbose", action="store_true", help="Verbose logs while migrating")
 args = parser.parse_args()
+verbose = args.verbose or False
+
+
+def log(message, end="\n"):
+    if verbose:
+        print(message, end=end, flush=True)
 
 
 def connect(to):
-    import mautrix_telegram.base as base
-    base.Base = declarative_base()
+    import mautrix_telegram.db.base as base
+    base.Base = declarative_base(cls=base.BaseBase)
     from mautrix_telegram.db import (Portal, Message, UserPortal, User, RoomState, UserProfile,
                                      Contact, Puppet, BotChat, TelegramFile)
     db_engine = sql.create_engine(to)
@@ -45,15 +52,30 @@ def connect(to):
         "TelegramFile": TelegramFile,
     }
 
-
+log("Connecting to old database")
 session, tables = connect(args.from_url)
 
 data = {}
 for name, table in tables.items():
+    log("Reading table {name}...".format(name=name), end=" ")
     data[name] = session.query(table).all()
+    log("Done!")
 
+log("Connecting to new database")
 session, tables = connect(args.to_url)
+
 for name, table in tables.items():
+    log("Writing table {name}".format(name=name), end="")
+    length = len(data[name])
+    n = 0
     for row in data[name]:
         session.merge(row)
+        n += 5
+        if n >= length:
+            log(".", end="")
+            n = 0
+    log(" Done!")
+
+log("Committing changes to database...", end=" ")
 session.commit()
+log("Done!")

mautrix_telegram/scripts/telematrix_import/__main__.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by

mautrix_telegram/sqlstatestore.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by

mautrix_telegram/tgclient.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -27,11 +27,11 @@ from telethon.tl.patched import Message
 class MautrixTelegramClient(TelegramClient):
     async def upload_file_direct(self, file: bytes, mime_type: str = None,
                                  attributes: List[TypeDocumentAttribute] = None,
-                                 file_name: str = None
+                                 file_name: str = None, max_image_size: float = 10 * 1000 ** 2,
                                  ) -> Union[InputMediaUploadedDocument, InputMediaUploadedPhoto]:
         file_handle = await super().upload_file(file, file_name=file_name, use_cache=False)
 
-        if mime_type == "image/png" or mime_type == "image/jpeg":
+        if (mime_type == "image/png" or mime_type == "image/jpeg") and len(file) < max_image_size:
             return InputMediaUploadedPhoto(file_handle)
         else:
             attributes = attributes or []

mautrix_telegram/user.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -65,7 +65,7 @@ class User(AbstractUser):
         self.db_portals = db_portals or []
         self._db_instance = db_instance  # type: Optional[DBUser]
 
-        self.command_status = None  # type: Dict
+        self.command_status = None  # type: Optional[Dict]
 
         (self.relaybot_whitelisted,
          self.whitelisted,
@@ -102,7 +102,9 @@ class User(AbstractUser):
 
     @property
     def db_contacts(self) -> Iterable[TelegramID]:
-        return (puppet.id for puppet in self.contacts)
+        return (puppet.id
+                for puppet in self.contacts
+                if puppet)
 
     @db_contacts.setter
     def db_contacts(self, contacts: Iterable[TelegramID]) -> None:
@@ -110,7 +112,9 @@ class User(AbstractUser):
 
     @property
     def db_portals(self) -> Iterable[Tuple[TelegramID, TelegramID]]:
-        return (portal.tgid_full for portal in self.portals.values() if not portal.deleted)
+        return (portal.tgid_full
+                for portal in self.portals.values()
+                if portal and not portal.deleted)
 
     @db_portals.setter
     def db_portals(self, portals: Iterable[Tuple[TelegramID, TelegramID]]) -> None:
@@ -129,12 +133,15 @@ class User(AbstractUser):
 
     def new_db_instance(self) -> DBUser:
         return DBUser(mxid=self.mxid, tgid=self.tgid, tg_username=self.username,
-                      contacts=self.db_contacts, saved_contacts=self.saved_contacts,
-                      portals=self.db_portals)
+                      saved_contacts=self.saved_contacts, portals=self.db_portals)
 
-    def save(self) -> None:
+    def save(self, contacts: bool = False, portals: bool = False) -> None:
         self.db_instance.update(tgid=self.tgid, tg_username=self.username, tg_phone=self.phone,
                                 saved_contacts=self.saved_contacts)
+        if contacts:
+            self.db_instance.contacts = self.db_contacts
+        if portals:
+            self.db_instance.portals = self.db_portals
 
     def delete(self, delete_db: bool = True) -> None:
         try:
@@ -154,6 +161,12 @@ class User(AbstractUser):
     # endregion
     # region Telegram connection management
 
+    async def try_ensure_started(self) -> None:
+        try:
+            await self.ensure_started()
+        except Exception:
+            self.log.exception("Exception in ensure_started")
+
     def ensure_started(self, even_if_no_session=False) -> Awaitable['User']:
         return super().ensure_started(even_if_no_session)
 
@@ -232,7 +245,7 @@ class User(AbstractUser):
         if puppet.is_real_user:
             await puppet.switch_mxid(None, None)
         for _, portal in self.portals.items():
-            if not portal.mxid or portal.has_bot:
+            if not portal or portal.deleted or not portal.mxid or portal.has_bot:
                 continue
             try:
                 await portal.main_intent.kick(portal.mxid, self.mxid, "Logged out of Telegram.")
@@ -240,7 +253,7 @@ class User(AbstractUser):
                 pass
         self.portals = {}
         self.contacts = []
-        self.save()
+        self.save(portals=True, contacts=True)
         if self.tgid:
             try:
                 del self.by_tgid[self.tgid]
@@ -295,7 +308,7 @@ class User(AbstractUser):
             creators.append(
                 portal.create_matrix_room(self, entity, invites=[self.mxid],
                                           synchronous=synchronous_create))
-        self.save()
+        self.save(portals=True)
         await asyncio.gather(*creators, loop=self.loop)
 
     def register_portal(self, portal: po.Portal) -> None:
@@ -305,12 +318,12 @@ class User(AbstractUser):
         except KeyError:
             pass
         self.portals[portal.tgid_full] = portal
-        self.save()
+        self.save(portals=True)
 
     def unregister_portal(self, portal: po.Portal) -> None:
         try:
             del self.portals[portal.tgid_full]
-            self.save()
+            self.save(portals=True)
         except KeyError:
             pass
 
@@ -335,7 +348,7 @@ class User(AbstractUser):
             puppet = pu.Puppet.get(user.id)
             await puppet.update_info(self, user)
             self.contacts.append(puppet)
-        self.save()
+        self.save(contacts=True)
 
     # endregion
     # region Class instance lookup
@@ -393,9 +406,9 @@ class User(AbstractUser):
     # endregion
 
 
-def init(context: 'Context') -> List[Awaitable['User']]:
+def init(context: 'Context') -> List[Awaitable[None]]:
     global config
     config = context.config
 
     users = [User.from_db(user) for user in DBUser.all()]
-    return [user.ensure_started() for user in users if user.tgid]
+    return [user.try_ensure_started() for user in users if user.tgid]

mautrix_telegram/util/__init__.py

@@ -3,6 +3,5 @@ from .format_duration import format_duration
 from .signed_token import sign_token, verify_token
 from .recursive_dict import recursive_del, recursive_set, recursive_get
 
-
 def ignore_coro(coro):
     pass

mautrix_telegram/util/file_transfer.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -23,14 +23,16 @@ import asyncio
 import magic
 from sqlalchemy.exc import IntegrityError, InvalidRequestError
 
-from telethon.tl.types import (Document, FileLocation, InputFileLocation, InputDocumentFileLocation,
-                               TypePhotoSize, PhotoSize, PhotoCachedSize)
+from telethon.tl.types import (Document, InputFileLocation, InputDocumentFileLocation,
+                               TypePhotoSize, PhotoSize, PhotoCachedSize, InputPhotoFileLocation,
+                               InputPeerPhotoFileLocation)
 from telethon.errors import (AuthBytesInvalidError, AuthKeyInvalidError, LocationInvalidError,
-                             SecurityError)
+                             SecurityError, FileIdInvalidError)
 from mautrix_appservice import IntentAPI
 
 from ..tgclient import MautrixTelegramClient
 from ..db import TelegramFile as DBTelegramFile
+from ..util import sane_mimetypes
 
 try:
     from PIL import Image
@@ -47,7 +49,8 @@ except ImportError:
 
 log = logging.getLogger("mau.util")  # type: logging.Logger
 
-TypeLocation = Union[Document, InputDocumentFileLocation, FileLocation, InputFileLocation]
+TypeLocation = Union[Document, InputDocumentFileLocation, InputPeerPhotoFileLocation,
+                     InputFileLocation, InputPhotoFileLocation]
 
 
 def convert_image(file: bytes, source_mime: str = "image/webp", target_type: str = "png",
@@ -99,9 +102,9 @@ def _read_video_thumbnail(data: bytes, video_ext: str = "mp4", frame_ext: str =
 
 
 def _location_to_id(location: TypeLocation) -> str:
-    if isinstance(location, (Document, InputDocumentFileLocation)):
+    if isinstance(location, (Document, InputDocumentFileLocation, InputPhotoFileLocation)):
         return f"{location.id}-{location.access_hash}"
-    elif isinstance(location, (FileLocation, InputFileLocation)):
+    elif isinstance(location, (InputFileLocation, InputPeerPhotoFileLocation)):
         return f"{location.volume_id}-{location.local_id}"
 
 
@@ -119,7 +122,7 @@ async def transfer_thumbnail_to_matrix(client: MautrixTelegramClient, intent: In
     if db_file:
         return db_file
 
-    video_ext = mimetypes.guess_extension(mime)
+    video_ext = sane_mimetypes.guess_extension(mime)
     if VideoFileClip and video_ext:
         try:
             file, width, height = _read_video_thumbnail(video, video_ext, frame_ext="png")
@@ -147,9 +150,11 @@ async def transfer_thumbnail_to_matrix(client: MautrixTelegramClient, intent: In
 
 transfer_locks = {}  # type: Dict[str, asyncio.Lock]
 
+TypeThumbnail = Optional[Union[TypeLocation, TypePhotoSize]]
+
 
 async def transfer_file_to_matrix(client: MautrixTelegramClient, intent: IntentAPI,
-                                  location: TypeLocation, thumbnail: Optional[Union[TypeLocation, TypePhotoSize]] = None,
+                                  location: TypeLocation, thumbnail: TypeThumbnail = None,
                                   is_sticker: bool = False) -> Optional[DBTelegramFile]:
     location_id = _location_to_id(location)
     if not location_id:
@@ -171,15 +176,15 @@ async def transfer_file_to_matrix(client: MautrixTelegramClient, intent: IntentA
 
 async def _unlocked_transfer_file_to_matrix(client: MautrixTelegramClient, intent: IntentAPI,
                                             loc_id: str, location: TypeLocation,
-                                            thumbnail: Optional[Union[TypeLocation, TypePhotoSize]],
-                                            is_sticker: bool) -> Optional[DBTelegramFile]:
+                                            thumbnail: TypeThumbnail, is_sticker: bool
+                                            ) -> Optional[DBTelegramFile]:
     db_file = DBTelegramFile.get(loc_id)
     if db_file:
         return db_file
 
     try:
         file = await client.download_file(location)
-    except LocationInvalidError:
+    except (LocationInvalidError, FileIdInvalidError):
         return None
     except (AuthBytesInvalidError, AuthKeyInvalidError, SecurityError) as e:
         log.exception(f"{e.__class__.__name__} while downloading a file.")

mautrix_telegram/util/format_duration.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by

mautrix_telegram/util/recursive_dict.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by

mautrix_telegram/util/sane_mimetypes.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -14,10 +14,25 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from lxml import html
+import mimetypes
 
-HTMLNode = html.HtmlElement
+mimetypes.init()
+
+sanity_overrides = {
+    "image/jpeg": ".jpeg",
+    "image/tiff": ".tiff",
+    "text/plain": ".txt",
+    "text/html": ".html",
+    "audio/mpeg": ".mp3",
+    "audio/ogg": ".ogg",
+    "application/xml": ".xml",
+    "application/octet-stream": "",
+    "application/x-msdos-program": ".exe",
+}
 
 
-def read_html(data: str) -> HTMLNode:
-    return html.fromstring(data)
+def guess_extension(mime: str) -> str:
+    try:
+        return sanity_overrides[mime]
+    except KeyError:
+        return mimetypes.guess_extension(mime)

mautrix_telegram/util/signed_token.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by

mautrix_telegram/web/common/auth_api.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -56,7 +56,7 @@ class AuthAPI(abc.ABC):
                                               error="You have already logged in with your Matrix "
                                                     "account.", errcode="already-logged-in")
 
-        resp = await puppet.switch_mxid(token, user.mxid)
+        resp = await puppet.switch_mxid(token.strip(), user.mxid)
         if resp == PuppetError.OnlyLoginSelf:
             return self.get_mx_login_response(status=403, errcode="only-login-self",
                                               error="You can only log in as your own Matrix user.")
@@ -72,10 +72,15 @@ class AuthAPI(abc.ABC):
                                           errcode="not-yet-implemented")
 
     async def post_login_phone(self, user: User, phone: str) -> web.Response:
+        if not phone or not phone.strip():
+            return self.get_login_response(mxid=user.mxid, state="request", status=400,
+                                           errcode="phone_number_invalid",
+                                           error="Phone number not given.")
         try:
-            await user.client.sign_in(phone or "+123")
+            await user.client.sign_in(phone.strip())
             return self.get_login_response(mxid=user.mxid, state="code", status=200,
-                                           message="Code requested successfully.")
+                                           message="Code requested successfully. Check your SMS "
+                                                   "or Telegram client and enter the code below.")
         except PhoneNumberInvalidError:
             return self.get_login_response(mxid=user.mxid, state="request", status=400,
                                            errcode="phone_number_invalid",
@@ -87,7 +92,8 @@ class AuthAPI(abc.ABC):
         except PhoneNumberAppSignupForbiddenError:
             return self.get_login_response(mxid=user.mxid, state="request", status=403,
                                            errcode="phone_number_app_signup_forbidden",
-                                           error="You have disabled 3rd party apps on your account.")
+                                           error="You have disabled 3rd party apps on your "
+                                                 "account.")
         except PhoneNumberUnoccupiedError:
             return self.get_login_response(mxid=user.mxid, state="request", status=404,
                                            errcode="phone_number_unoccupied",
@@ -116,10 +122,9 @@ class AuthAPI(abc.ABC):
         if user.command_status and user.command_status["action"] == "Login":
             user.command_status = None
 
-
     async def post_login_token(self, user: User, token: str) -> web.Response:
         try:
-            user_info = await user.client.sign_in(bot_token=token)
+            user_info = await user.client.sign_in(bot_token=token.strip())
             await self.postprocess_login(user, user_info)
             return self.get_login_response(mxid=user.mxid, state="logged-in", status=200,
                                            username=user_info.username, phone=None,
@@ -173,7 +178,7 @@ class AuthAPI(abc.ABC):
 
     async def post_login_password(self, user: User, password: str) -> web.Response:
         try:
-            user_info = await user.client.sign_in(password=password)
+            user_info = await user.client.sign_in(password=password.strip())
             await self.postprocess_login(user, user_info)
             human_tg_id = f"@{user_info.username}" if user_info.username else f"+{user_info.phone}"
             return self.get_login_response(mxid=user.mxid, state="logged-in", status=200,

mautrix_telegram/web/provisioning/__init__.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -247,7 +247,7 @@ class ProvisioningAPI(AuthAPI):
             "group": "chat",
         }[type]
 
-        portal = Portal(tgid=None, mxid=room_id, title=title, about=about, peer_type=type)
+        portal = Portal(tgid=TelegramID(0), mxid=room_id, title=title, about=about, peer_type=type)
         try:
             await portal.create_telegram_chat(user, supergroup=supergroup)
         except ValueError as e:
@@ -365,7 +365,7 @@ class ProvisioningAPI(AuthAPI):
 
     async def bridge_info(self, request: web.Request) -> web.Response:
         return web.json_response({
-            "relaybot_username": self.context.bot.username,
+            "relaybot_username": self.context.bot.username if self.context.bot is not None else None,
         }, status=200)
 
     @staticmethod

mautrix_telegram/web/public/__init__.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -87,7 +87,8 @@ class PublicBridgeWebsite(AuthAPI):
         return self.get_login_response(mxid=user.mxid, human_tg_id=user.human_tg_id)
 
     async def get_matrix_login(self, request: web.Request) -> web.Response:
-        mxid = self.verify_token(request.rel_url.query.get("token", None), endpoint="/matrix-login")
+        mxid = self.verify_token(request.rel_url.query.get("token", None),
+                                 endpoint="/matrix-login")
         if not mxid:
             return self.get_mx_login_response(status=401, state="invalid-token")
         user = User.get_by_mxid(mxid, create=False) if mxid else None
@@ -124,7 +125,8 @@ class PublicBridgeWebsite(AuthAPI):
                                                       error=error, message=message, mxid=mxid))
 
     async def post_matrix_login(self, request: web.Request) -> web.Response:
-        mxid = self.verify_token(request.rel_url.query.get("token", None), endpoint="/matrix-login")
+        mxid = self.verify_token(request.rel_url.query.get("token", None),
+                                 endpoint="/matrix-login")
         if not mxid:
             return self.get_mx_login_response(status=401, state="invalid-token")
 
@@ -167,7 +169,13 @@ class PublicBridgeWebsite(AuthAPI):
         elif "bot_token" in data:
             return await self.post_login_token(user, data["bot_token"])
         elif "code" in data:
-            resp = await self.post_login_code(user, data["code"],
+            try:
+                code = int(data["code"].strip())
+            except ValueError:
+                return self.get_login_response(mxid=user.mxid, state="code", status=400,
+                                               errcode="phone_code_invalid",
+                                               error="Phone code must be a number.")
+            resp = await self.post_login_code(user, code,
                                               password_in_data="password" in data)
             if resp or "password" not in data:
                 return resp

mautrix_telegram/web/public/login.css

@@ -1,6 +1,6 @@
 /*
  * mautrix-telegram - A Matrix-Telegram puppeting bridge
- * Copyright (C) 2018 Tulir Asokan
+ * Copyright (C) 2019 Tulir Asokan
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU Affero General Public License as published by

mautrix_telegram/web/public/login.html.mako

@@ -1,6 +1,6 @@
 <!--
 mautrix-telegram - A Matrix-Telegram puppeting bridge
-Copyright (C) 2018 Tulir Asokan
+Copyright (C) 2019 Tulir Asokan
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU Affero General Public License as published by
@@ -16,7 +16,7 @@ You should have received a copy of the GNU Affero General Public License
 along with this program.  If not, see <https://www.gnu.org/licenses/>.
 -->
 <!DOCTYPE html>
-<html>
+<html lang="en">
 <head>
 	<title>Login - Mautrix-Telegram bridge</title>
 	<link rel="icon" type="image/png" href="favicon.png"/>
@@ -25,9 +25,10 @@ along with this program.  If not, see <https://www.gnu.org/licenses/>.
 	<meta property="og:image" content="favicon.png">
 	<meta charset="utf-8">
 	<link rel="stylesheet" href="//fonts.googleapis.com/css?family=Roboto:300,700">
-	<link rel="stylesheet" href="//cdn.rawgit.com/necolas/normalize.css/master/normalize.css">
 	<link rel="stylesheet"
-		  href="//cdn.rawgit.com/milligram/milligram/master/dist/milligram.min.css">
+		  href="https://cdnjs.cloudflare.com/ajax/libs/normalize/8.0.1/normalize.min.css">
+	<link rel="stylesheet"
+		  href="https://cdnjs.cloudflare.com/ajax/libs/milligram/1.3.0/milligram.min.css">
 	<link rel="stylesheet" href="login.css"/>
 
 	<script>
@@ -94,13 +95,14 @@ along with this program.  If not, see <https://www.gnu.org/licenses/>.
 				% if state == "request":
 					<label for="value">Phone number</label>
 					<input type="tel" id="value" name="phone" placeholder="Enter phone number"/>
-					<button type="submit">Request code</button>
-					<button class="button-clear" type="button" onclick="switchToBotLogin()">
+					<button type="submit">Start</button>
+					<button class="button-clear float-right" type="button" onclick="switchToBotLogin()">
 						Use bot token
 					</button>
 				% elif state == "bot_token":
 					<label for="value">Bot token</label>
-					<input type="text" id="value" name="bot_token" placeholder="Enter bot API token"/>
+					<input type="text" id="value" name="bot_token"
+						   placeholder="Enter bot API token"/>
 					<button type="submit">Sign in</button>
 				% elif state == "code":
 					<label for="value">Phone code</label>

mautrix_telegram/web/public/matrix-login.html.mako

@@ -1,6 +1,6 @@
 <!--
 mautrix-telegram - A Matrix-Telegram puppeting bridge
-Copyright (C) 2018 Tulir Asokan
+Copyright (C) 2019 Tulir Asokan
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU Affero General Public License as published by
@@ -16,7 +16,7 @@ You should have received a copy of the GNU Affero General Public License
 along with this program.  If not, see <https://www.gnu.org/licenses/>.
 -->
 <!DOCTYPE html>
-<html>
+<html lang="en">
 <head>
 	<title>Matrix login - Mautrix-Telegram bridge</title>
 	<link rel="icon" type="image/png" href="favicon.png"/>
@@ -25,9 +25,10 @@ along with this program.  If not, see <https://www.gnu.org/licenses/>.
 	<meta property="og:image" content="favicon.png">
 	<meta charset="utf-8">
 	<link rel="stylesheet" href="//fonts.googleapis.com/css?family=Roboto:300,700">
-	<link rel="stylesheet" href="//cdn.rawgit.com/necolas/normalize.css/master/normalize.css">
 	<link rel="stylesheet"
-		  href="//cdn.rawgit.com/milligram/milligram/master/dist/milligram.min.css">
+		  href="https://cdnjs.cloudflare.com/ajax/libs/normalize/8.0.1/normalize.min.css">
+	<link rel="stylesheet"
+		  href="https://cdnjs.cloudflare.com/ajax/libs/milligram/1.3.0/milligram.min.css">
 	<link rel="stylesheet" href="login.css"/>
 </head>
 <body>

optional-requirements.txt

@@ -1,4 +1,4 @@
-lxml
 cryptg
 Pillow
 moviepy
+prometheus-client

setup.cfg

@@ -0,0 +1,2 @@
+[aliases]
+test=pytest

setup.py

@@ -3,11 +3,10 @@ import glob
 import mautrix_telegram
 
 extras = {
-    "highlight_edits": ["lxml>=4.1.1,<5"],
-    "better_formatter": ["lxml>=4.1.1,<5"],
-    "fast_crypto": ["cryptg>=0.1,<0.2"],
-    "webp_convert": ["Pillow>=4.3.0,<6"],
+    "fast_crypto": ["cryptg>=0.1,<0.3"],
+    "webp_convert": ["Pillow>=4.3.0,<7"],
     "hq_thumbnails": ["moviepy>=1.0,<2.0"],
+    "metrics": ["prometheus-client>=0.6.0,<0.8.0"],
 }
 extras["all"] = list({dep for deps in extras.values() for dep in deps})
 
@@ -32,18 +31,21 @@ setuptools.setup(
 
     install_requires=[
         "aiohttp>=3.0.1,<4",
-        "mautrix-appservice>=0.3.8,<0.4.0",
+        "mautrix-appservice>=0.3.11,<0.4.0",
         "SQLAlchemy>=1.2.3,<2",
         "alembic>=1.0.0,<2",
         "commonmark>=0.8.1,<1",
         "ruamel.yaml>=0.15.35,<0.16",
         "future-fstrings>=0.4.2",
         "python-magic>=0.4.15,<0.5",
-        "telethon>=1.5.5,<1.6",
-        "telethon-session-sqlalchemy>=0.2.8,<0.3",
+        "telethon>=1.9,<1.10",
+        "telethon-session-sqlalchemy>=0.2.14,<0.3",
     ],
     extras_require=extras,
 
+    setup_requires=["pytest-runner"],
+    tests_require=["pytest", "pytest-asyncio", "pytest-mock"],
+
     classifiers=[
         "Development Status :: 4 - Beta",
         "License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)",

tests/commands/test_handler.py

@@ -0,0 +1,370 @@
+from typing import Tuple
+from unittest.mock import Mock
+
+import pytest
+from _pytest.fixtures import FixtureRequest
+from pytest_mock import MockFixture
+
+import mautrix_telegram.commands.handler
+from mautrix_telegram.commands.handler import (CommandEvent, CommandHandler, CommandProcessor,
+                                               HelpSection)
+from mautrix_telegram.config import Config
+from mautrix_telegram.context import Context
+from mautrix_telegram.types import MatrixEventID, MatrixRoomID, MatrixUserID
+import mautrix_telegram.user as u
+
+from tests.utils.helpers import AsyncMock, list_true_once_each
+
+
+@pytest.fixture
+def context(request: FixtureRequest) -> Context:
+    """Returns a Context with mocked Attributes.
+
+    Uses the attribute cls.config as Config.
+    """
+    # Config(path, registration_path, base_path)
+    config = getattr(request.cls, 'config', Config("", "", ""))
+    return Context(az=Mock(), config=config, loop=Mock(), session_container=Mock(), bot=Mock())
+
+
+@pytest.fixture
+def command_processor(context: Context) -> CommandProcessor:
+    """Returns a mocked CommandProcessor."""
+    return CommandProcessor(context)
+
+
+class TestCommandEvent:
+    config = Config("", "", "")
+    config["bridge.command_prefix"] = "tg"
+    config["bridge.permissions"] = {"*": "noperm"}
+
+    def test_reply(
+        self, command_processor: CommandProcessor, mocker: MockFixture
+    ) -> None:
+        mocker.patch("mautrix_telegram.user.config", self.config)
+
+        evt = CommandEvent(
+            processor=command_processor,
+            room=MatrixRoomID("#mock_room:example.org"),
+            event=MatrixEventID("$H45H:example.org"),
+            sender=u.User(MatrixUserID("@sender:example.org")),
+            command="help",
+            args=[],
+            is_management=True,
+            is_portal=False,
+        )
+
+        mock_az = command_processor.az
+
+        message = "**This** <i>was</i><br/><strong>all</strong>fun*!"
+
+        # html, no markdown
+        evt.reply(message, allow_html=True, render_markdown=False)
+        mock_az.intent.send_notice.assert_called_with(
+            MatrixRoomID("#mock_room:example.org"),
+            "**This** <i>was</i><br/><strong>all</strong>fun*!",
+            html="**This** <i>was</i><br/><strong>all</strong>fun*!\n",
+        )
+
+        # html, markdown (default)
+        evt.reply(message, allow_html=True, render_markdown=True)
+        mock_az.intent.send_notice.assert_called_with(
+            MatrixRoomID("#mock_room:example.org"),
+            "**This** <i>was</i><br/><strong>all</strong>fun*!",
+            html=(
+                "<p><strong>This</strong> <i>was</i><br/>"
+                "<strong>all</strong>fun*!</p>\n"
+            ),
+        )
+
+        # no html, no markdown
+        evt.reply(message, allow_html=False, render_markdown=False)
+        mock_az.intent.send_notice.assert_called_with(
+            MatrixRoomID("#mock_room:example.org"),
+            "**This** <i>was</i><br/><strong>all</strong>fun*!",
+            html=None,
+        )
+
+        # no html, markdown
+        evt.reply(message, allow_html=False, render_markdown=True)
+        mock_az.intent.send_notice.assert_called_with(
+            MatrixRoomID("#mock_room:example.org"),
+            "**This** <i>was</i><br/><strong>all</strong>fun*!",
+            html="<p><strong>This</strong> &lt;i&gt;was&lt;/i&gt;&lt;br/&gt;"
+                 "&lt;strong&gt;all&lt;/strong&gt;fun*!</p>\n"
+        )
+
+    def test_reply_with_cmdprefix(self, command_processor: CommandProcessor, mocker: MockFixture
+                                  ) -> None:
+        mocker.patch("mautrix_telegram.user.config", self.config)
+
+        evt = CommandEvent(
+            processor=command_processor,
+            room=MatrixRoomID("#mock_room:example.org"),
+            event=MatrixEventID("$H45H:example.org"),
+            sender=u.User(MatrixUserID("@sender:example.org")),
+            command="help",
+            args=[],
+            is_management=False,
+            is_portal=False,
+        )
+
+        mock_az = command_processor.az
+
+        evt.reply("$cmdprefix+sp ....$cmdprefix+sp...$cmdprefix $cmdprefix", allow_html=False,
+                  render_markdown=False)
+
+        mock_az.intent.send_notice.assert_called_with(
+            MatrixRoomID("#mock_room:example.org"),
+            "tg ....tg+sp...tg tg",
+            html=None,
+        )
+
+    def test_reply_with_cmdprefix_in_management_room(self, command_processor: CommandProcessor,
+                                                     mocker: MockFixture) -> None:
+        mocker.patch("mautrix_telegram.user.config", self.config)
+
+        evt = CommandEvent(
+            processor=command_processor,
+            room=MatrixRoomID("#mock_room:example.org"),
+            event=MatrixEventID("$H45H:example.org"),
+            sender=u.User(MatrixUserID("@sender:example.org")),
+            command="help",
+            args=[],
+            is_management=True,
+            is_portal=False,
+        )
+
+        mock_az = command_processor.az
+
+        evt.reply(
+            "$cmdprefix+sp ....$cmdprefix+sp...$cmdprefix $cmdprefix",
+            allow_html=True,
+            render_markdown=True,
+        )
+
+        mock_az.intent.send_notice.assert_called_with(
+            MatrixRoomID("#mock_room:example.org"),
+            "....tg+sp...tg tg",
+            html="<p>....tg+sp...tg tg</p>\n",
+        )
+
+
+class TestCommandHandler:
+    config = Config("", "", "")
+    config["bridge.permissions"] = {"*": "noperm"}
+
+    @pytest.mark.parametrize(
+        (
+            "needs_auth,"
+            "needs_puppeting,"
+            "needs_matrix_puppeting,"
+            "needs_admin,"
+            "management_only,"
+        ),
+        [l for l in list_true_once_each(length=5)]
+    )
+    @pytest.mark.asyncio
+    async def test_permissions_denied(
+        self,
+        needs_auth: bool,
+        needs_puppeting: bool,
+        needs_matrix_puppeting: bool,
+        needs_admin: bool,
+        management_only: bool,
+        command_processor: CommandProcessor,
+        boolean: bool,
+        mocker: MockFixture,
+    ) -> None:
+        mocker.patch("mautrix_telegram.user.config", self.config)
+
+        command = "testcmd"
+
+        mock_handler = Mock()
+
+        command_handler = CommandHandler(
+            handler=mock_handler,
+            needs_auth=needs_auth,
+            needs_puppeting=needs_puppeting,
+            needs_matrix_puppeting=needs_matrix_puppeting,
+            needs_admin=needs_admin,
+            management_only=management_only,
+            name=command,
+            help_text="No real command",
+            help_args="mock mockmock",
+            help_section=HelpSection("Mock Section", 42, ""),
+        )
+
+        sender = u.User(MatrixUserID("@sender:example.org"))
+        sender.puppet_whitelisted = False
+        sender.matrix_puppet_whitelisted = False
+        sender.is_admin = False
+
+        event = CommandEvent(
+            processor=command_processor,
+            room=MatrixRoomID("#mock_room:example.org"),
+            event=MatrixEventID("$H45H:example.org"),
+            sender=sender,
+            command=command,
+            args=[],
+            is_management=False,
+            is_portal=boolean,
+        )
+
+        assert await command_handler.get_permission_error(event)
+        assert not command_handler.has_permission(False, False, False, False, False)
+
+    @pytest.mark.parametrize(
+        (
+            "is_management,"
+            "puppet_whitelisted,"
+            "matrix_puppet_whitelisted,"
+            "is_admin,"
+            "is_logged_in,"
+        ),
+        [l for l in list_true_once_each(length=5)]
+    )
+    @pytest.mark.asyncio
+    async def test_permission_granted(
+        self,
+        is_management: bool,
+        puppet_whitelisted: bool,
+        matrix_puppet_whitelisted: bool,
+        is_admin: bool,
+        is_logged_in: bool,
+        command_processor: CommandProcessor,
+        boolean: bool,
+        mocker: MockFixture,
+    ) -> None:
+        mocker.patch("mautrix_telegram.user.config", self.config)
+
+        command = "testcmd"
+
+        mock_handler = Mock()
+
+        command_handler = CommandHandler(
+            handler=mock_handler,
+            needs_auth=False,
+            needs_puppeting=False,
+            needs_matrix_puppeting=False,
+            needs_admin=False,
+            management_only=False,
+            name=command,
+            help_text="No real command",
+            help_args="mock mockmock",
+            help_section=HelpSection("Mock Section", 42, ""),
+        )
+
+        sender = u.User(MatrixUserID("@sender:example.org"))
+        sender.puppet_whitelisted = puppet_whitelisted
+        sender.matrix_puppet_whitelisted = matrix_puppet_whitelisted
+        sender.is_admin = is_admin
+        mocker.patch.object(u.User, 'is_logged_in', return_value=is_logged_in)
+
+        event = CommandEvent(
+            processor=command_processor,
+            room=MatrixRoomID("#mock_room:example.org"),
+            event=MatrixEventID("$H45H:example.org"),
+            sender=sender,
+            command=command,
+            args=[],
+            is_management=is_management,
+            is_portal=boolean,
+        )
+
+        assert not await command_handler.get_permission_error(event)
+        assert command_handler.has_permission(
+            is_management=is_management,
+            puppet_whitelisted=puppet_whitelisted,
+            matrix_puppet_whitelisted=matrix_puppet_whitelisted,
+            is_admin=is_admin,
+            is_logged_in=is_logged_in,
+        )
+
+
+class TestCommandProcessor:
+    config = Config("", "", "")
+    config["bridge.command_prefix"] = "tg"
+    config["bridge.permissions"] = {"*": "relaybot"}
+
+    @pytest.mark.asyncio
+    async def test_handle(self, command_processor: CommandProcessor, boolean2: Tuple[bool, bool],
+                          mocker: MockFixture) -> None:
+        mocker.patch('mautrix_telegram.user.config', self.config)
+        mocker.patch(
+            'mautrix_telegram.commands.handler.command_handlers',
+            {"help": AsyncMock(), "unknown-command": AsyncMock()}
+        )
+
+        sender = u.User(MatrixUserID("@sender:example.org"))
+
+        result = await command_processor.handle(
+            room=MatrixRoomID("#mock_room:example.org"),
+            event_id=MatrixEventID("$H45H:example.org"),
+            sender=sender,
+            command="hElp",
+            args=[],
+            is_management=boolean2[0],
+            is_portal=boolean2[1],
+        )
+
+        assert result is None
+        command_handlers = mautrix_telegram.commands.handler.command_handlers
+        command_handlers["help"].mock.assert_called_once()  # type: ignore
+
+    @pytest.mark.asyncio
+    async def test_handle_unknown_command(self, command_processor: CommandProcessor,
+                                          boolean2: Tuple[bool, bool], mocker: MockFixture) -> None:
+        mocker.patch('mautrix_telegram.user.config', self.config)
+        mocker.patch(
+            'mautrix_telegram.commands.handler.command_handlers',
+            {"help": AsyncMock(), "unknown-command": AsyncMock()}
+        )
+
+        sender = u.User(MatrixUserID("@sender:example.org"))
+        sender.command_status = {}
+
+        result = await command_processor.handle(
+            room=MatrixRoomID("#mock_room:example.org"),
+            event_id=MatrixEventID("$H45H:example.org"),
+            sender=sender,
+            command="foo",
+            args=[],
+            is_management=boolean2[0],
+            is_portal=boolean2[1],
+        )
+
+        assert result is None
+        command_handlers = mautrix_telegram.commands.handler.command_handlers
+        command_handlers["help"].mock.assert_not_called()  # type: ignore
+        command_handlers["unknown-command"].mock.assert_called_once()  # type: ignore
+
+    @pytest.mark.asyncio
+    async def test_handle_delegated_handler(self, command_processor: CommandProcessor,
+                                            boolean2: Tuple[bool, bool],
+                                            mocker: MockFixture) -> None:
+        mocker.patch('mautrix_telegram.user.config', self.config)
+        mocker.patch(
+            'mautrix_telegram.commands.handler.command_handlers',
+            {"help": AsyncMock(), "unknown-command": AsyncMock()}
+        )
+
+        sender = u.User(MatrixUserID("@sender:example.org"))
+        sender.command_status = {"foo": AsyncMock(), "next": AsyncMock()}
+
+        result = await command_processor.handle(
+            room=MatrixRoomID("#mock_room:example.org"),
+            event_id=MatrixEventID("$H45H:example.org"),
+            sender=sender,  # u.User
+            command="foo",
+            args=[],
+            is_management=boolean2[0],
+            is_portal=boolean2[1]
+        )
+
+        assert result is None
+        command_handlers = mautrix_telegram.commands.handler.command_handlers
+        command_handlers["help"].mock.assert_not_called()  # type: ignore
+        command_handlers["unknown-command"].mock.assert_not_called()  # type: ignore
+        sender.command_status["foo"].mock.assert_not_called()  # type: ignore
+        sender.command_status["next"].mock.assert_called_once()  # type: ignore

tests/conftest.py

@@ -0,0 +1,3 @@
+pytest_plugins = [
+    "tests.utils.fixtures",
+]

tests/utils/fixtures.py

@@ -0,0 +1,27 @@
+"""This module provides utility fixtures for testing."""
+from typing import Tuple
+
+from _pytest.fixtures import FixtureRequest
+import pytest
+
+
+@pytest.fixture(params=[True, False])
+def boolean(request: FixtureRequest) -> bool:
+    return request.param
+
+
+@pytest.fixture
+def boolean1(boolean: bool) -> Tuple[bool]:
+    return boolean,
+
+
+@pytest.fixture(params=[True, False])
+def boolean2(request: FixtureRequest, boolean: bool) -> Tuple[bool, bool]:
+    return boolean, request.param
+
+
+@pytest.fixture(params=[True, False])
+def boolean3(request: FixtureRequest, boolean2: Tuple[bool, bool]) -> Tuple[bool, bool, bool]:
+    return boolean2[0], boolean2[1], request.param
+
+# …

tests/utils/helpers.py

@@ -0,0 +1,24 @@
+"""This module provides utility functions for testing."""
+from typing import Generator, Tuple
+from unittest.mock import Mock
+
+
+def AsyncMock(*args, **kwargs):
+    """Mocks a asyncronous coroutine which can be called with 'await'."""
+    m = Mock(*args, **kwargs)
+
+    async def mock_coro(*args, **kwargs):
+        return m(*args, **kwargs)
+
+    mock_coro.mock = m
+    return mock_coro
+
+
+def list_true_once_each(length: int) -> Generator[Tuple[bool, ...], None, None]:
+    """Yields tuples of bools with exactly one entry being True, starting left.
+
+    Args:
+        length: Length of the resulting tuples
+    """
+    for i in range(length):
+        yield tuple(i == j for j in range(length))