Bump version to 0.6.0rc1

Allow config key override through env var

.gitignore

@@ -1,9 +1,14 @@
 .idea/
 
 .venv
+env/
 pip-selfcheck.json
 *.pyc
 __pycache__
+build
+dist
+*.egg-info
+.eggs
 
 config.yaml
 registration.yaml

Dockerfile

@@ -1,4 +1,4 @@
-FROM docker.io/alpine:3.8
+FROM docker.io/alpine:3.9
 
 ENV UID=1337 \
     GID=1337 \
@@ -7,22 +7,37 @@ ENV UID=1337 \
 COPY . /opt/mautrix-telegram
 WORKDIR /opt/mautrix-telegram
 RUN apk add --no-cache \
-      python3-dev \
-      build-base \
       py3-virtualenv \
       py3-pillow \
       py3-aiohttp \
-      py3-lxml \
       py3-magic \
-      py3-numpy \
-      py3-asn1crypto \
       py3-sqlalchemy \
       py3-markdown \
       py3-psycopg2 \
+      # Not yet in stable repos:
+      #py3-ruamel \
+      # Indirect dependencies
+      #commonmark
+        py3-future \
+      #alembic
+        py3-mako \
+        py3-dateutil \
+        py3-markupsafe \
+      #moviepy
+        py3-decorator \
+        #py3-tqdm \
+        py3-requests \
+        #imageio
+          py3-numpy \
+      #telethon
+        py3-rsa \
+      # Other dependencies
+      python3-dev \
+      build-base \
       ffmpeg \
       ca-certificates \
       su-exec \
- && pip3 install -r requirements.txt -r optional-requirements.txt
+ && pip3 install .[all]
 
 VOLUME /data
 

ROADMAP.md

@@ -3,6 +3,7 @@
 * Matrix → Telegram
   * [x] Message content (text, formatting, files, etc..)
   * [x] Message redactions
+  * [x] Message edits
   * [ ] ‡ Message history
   * [x] Presence
   * [x] Typing notifications
@@ -21,6 +22,10 @@
     * [ ] ‡ Changes to displayname/avatar
 * Telegram → Matrix
   * [x] Message content (text, formatting, files, etc..)
+  * [ ] Advanced message content/media
+    * [x] Polls
+	* [x] Games
+	* [ ] Buttons
   * [x] Message deletions
   * [x] Message edits
   * [ ] Message history
@@ -48,6 +53,8 @@
   * [x] Option to use bot to relay messages for unauthenticated Matrix users
   * [x] Option to use own Matrix account for messages sent from other Telegram clients
   * [ ] ‡ Calls (hard, not yet supported by Telethon)
+  * [ ] ‡ Secret chats (not yet supported by Telethon)
+  * [ ] ‡ E2EE in Matrix rooms (not yet supported 
 
 † Information not automatically sent from source, i.e. implementation may not be possible  
 ‡ Maybe, i.e. this feature may or may not be implemented at some point

alembic/env.py

@@ -7,10 +7,9 @@ from os.path import abspath, dirname
 
 sys.path.insert(0, dirname(dirname(abspath(__file__))))
 
-from mautrix_telegram.base import Base
+from mautrix_telegram.db import Base
 from mautrix_telegram.config import Config
 from alchemysession import AlchemySessionContainer
-import mautrix_telegram.db
 
 # this is the Alembic Config object, which provides
 # access to the values within the .ini file in use.

alembic/versions/17574c57f3f8_add_disable_updates_field_for_puppets.py

@@ -0,0 +1,27 @@
+"""Add disable_updates field for puppets
+
+Revision ID: 17574c57f3f8
+Revises: a9119be92164
+Create Date: 2019-05-15 00:24:46.967529
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '17574c57f3f8'
+down_revision = 'a9119be92164'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    with op.batch_alter_table("puppet") as batch_op:
+        batch_op.add_column(sa.Column("disable_updates", sa.Boolean(), nullable=False,
+                                      server_default=sa.sql.expression.false()))
+
+
+def downgrade():
+    with op.batch_alter_table("puppet") as batch_op:
+        batch_op.drop_column("disable_updates")

alembic/versions/1fa46383a9d3_add_is_bot_field_to_puppets.py

@@ -17,7 +17,8 @@ depends_on = None
 
 
 def upgrade():
-    op.add_column('puppet', sa.Column('is_bot', sa.Boolean(), nullable=True))
+    with op.batch_alter_table("puppet") as batch_op:
+        batch_op.add_column(sa.Column('is_bot', sa.Boolean(), nullable=True))
 
 
 def downgrade():

alembic/versions/30eca60587f1_add_megagroup_field_to_portals.py

@@ -16,7 +16,8 @@ depends_on = None
 
 
 def upgrade():
-    op.add_column('portal', sa.Column('megagroup', sa.Boolean()))
+    with op.batch_alter_table("portal") as batch_op:
+        batch_op.add_column(sa.Column('megagroup', sa.Boolean()))
 
 
 def downgrade():

alembic/versions/6ca3d74d51e4_move_state_store_to_main_database.py

@@ -12,7 +12,7 @@ import json
 import re
 
 from mautrix_telegram.config import Config
-from mautrix_telegram.base import Base
+from mautrix_telegram.db import Base
 
 # revision identifiers, used by Alembic.
 revision = "6ca3d74d51e4"
@@ -57,7 +57,8 @@ class Puppet(Base):
 
 
 def upgrade():
-    op.add_column("puppet", sa.Column("matrix_registered", sa.Boolean(), nullable=False,
+    with op.batch_alter_table("puppet") as batch_op:
+        batch_op.add_column(sa.Column("matrix_registered", sa.Boolean(), nullable=False,
                                       server_default=sa.sql.expression.false()))
     op.create_table("mx_room_state",
                     sa.Column("room_id", sa.String(), nullable=False),

alembic/versions/9e9c89b0b877_add_edit_index_to_messages.py

@@ -0,0 +1,48 @@
+"""Add edit index to messages
+
+Revision ID: 9e9c89b0b877
+Revises: 17574c57f3f8
+Create Date: 2019-05-29 15:28:23.128377
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = '9e9c89b0b877'
+down_revision = '17574c57f3f8'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table('_message_temp',
+                    sa.Column('mxid', sa.String),
+                    sa.Column('mx_room', sa.String),
+                    sa.Column('tgid', sa.Integer),
+                    sa.Column('tg_space', sa.Integer),
+                    sa.Column('edit_index', sa.Integer),
+                    sa.PrimaryKeyConstraint('tgid', 'tg_space', 'edit_index'),
+                    sa.UniqueConstraint("mxid", "mx_room", "tg_space", name="_mx_id_room_2"))
+    c = op.get_bind()
+    c.execute("INSERT INTO _message_temp (mxid, mx_room, tgid, tg_space, edit_index) "
+              "SELECT message.mxid, message.mx_room, message.tgid, message.tg_space, 0 "
+              "FROM message")
+    c.execute("DROP TABLE message")
+    c.execute("ALTER TABLE _message_temp RENAME TO message")
+
+
+def downgrade():
+    op.create_table('_message_temp',
+                    sa.Column('mxid', sa.String),
+                    sa.Column('mx_room', sa.String),
+                    sa.Column('tgid', sa.Integer),
+                    sa.Column('tg_space', sa.Integer),
+                    sa.PrimaryKeyConstraint('tgid', 'tg_space'),
+                    sa.UniqueConstraint("mxid", "mx_room", "tg_space", name="_mx_id_room"))
+    c = op.get_bind()
+    c.execute("INSERT INTO _message_temp (mxid, mx_room, tgid, tg_space) "
+              "SELECT message.mxid, message.mx_room, message.tgid, message.tg_space "
+              "FROM message")
+    c.execute("DROP TABLE message")
+    c.execute("ALTER TABLE _message_temp RENAME TO message")

example-config.yaml

@@ -27,9 +27,6 @@ appservice:
     #   SQLite:   sqlite:///filename.db
     #   Postgres: postgres://username:password@hostname/dbname
     database: sqlite:///mautrix-telegram.db
-    # Whether or not to use SQLAlchemy Core for common database actions. Use if the bridge is
-    # being bottlenecked on ORM commits. Only supported with PostgreSQL.
-    sqlalchemy_core_mode: false
 
     # Public part of web server for out-of-Matrix interaction with the bridge.
     # Used for things like login if the user wants to make sure the 2FA password isn't stored in
@@ -67,6 +64,11 @@ appservice:
     as_token: "This value is generated when generating the registration"
     hs_token: "This value is generated when generating the registration"
 
+# Prometheus telemetry config. Requires prometheus-aio to be installed.
+metrics:
+    enabled: false
+    listen_port: 8000
+
 # Bridge config
 bridge:
     # Localpart template of MXIDs for Telegram users.
@@ -107,9 +109,21 @@ bridge:
     # If no channel admins have logged into the bridge, the bridge won't be able to sync the member
     # list regardless of this setting.
     sync_channel_members: true
+    # Whether or not to skip deleted members when syncing members.
+    skip_deleted_members: true
+    # Whether or not to automatically synchronize contacts and chats of Matrix users logged into
+    # their Telegram account at startup.
+    startup_sync: true
+    # Number of most recently active dialogs to check when syncing chats.
+    # Dialogs include groups and private chats, but only groups are synced.
+    # Set to 0 to remove limit.
+    sync_dialog_limit: 30
     # The maximum number of simultaneous Telegram deletions to handle.
     # A large number of simultaneous redactions could put strain on your homeserver.
     max_telegram_delete: 10
+    # Whether or not to automatically sync the Matrix room state (mostly unpuppeted displaynames)
+    # at startup and when creating a bridge.
+    sync_matrix_state: true
     # Allow logging in within Matrix. If false, the only way to log in is using the out-of-Matrix
     # login website (see appservice.public config section)
     allow_matrix_login: true
@@ -117,16 +131,35 @@ bridge:
     # Only enable this if your displayname_template has some static part that the bridge can use to
     # reliably identify what is a plaintext highlight.
     plaintext_highlights: false
-    # Highlight changed/added parts in edits. Requires lxml.
-    highlight_edits: false
     # Whether or not to make portals of publicly joinable channels/supergroups publicly joinable on Matrix.
     public_portals: true
     # Whether or not to fetch and handle Telegram updates at startup from the time the bridge was down.
     # Currently only works for private chats and normal groups.
+    # WARNING: This feature seems to be broken in the telegram library.
     catch_up: false
     # Whether or not to use /sync to get presence, read receipts and typing notifications when using
     # your own Matrix account as the Matrix puppet for your Telegram account.
     sync_with_custom_puppets: true
+    # Set to false to disable link previews in messages sent to Telegram.
+    telegram_link_preview: true
+    # Use inline images instead of a separate message for the caption.
+    # N.B. Inline images are not supported on all clients (e.g. Riot iOS).
+    inline_images: false
+    # Maximum size of image in megabytes before sending to Telegram as a document.
+    image_as_file_size: 10
+    # Maximum size of Telegram documents in megabytes to bridge.
+    max_document_size: 100
+
+    # Whether to bridge Telegram bot messages as m.notices or m.texts.
+    bot_messages_as_notices: true
+    bridge_notices:
+        # Whether or not Matrix bot messages (type m.notice) should be bridged.
+        default: false
+        # List of user IDs for whom the previous flag is flipped.
+        # e.g. if bridge_notices.default is false, notices from other users will not be bridged, but
+        #      notices from users listed here will be bridged.
+        exceptions:
+        - "@importantbot:example.com"
 
     # Some config options related to Telegram message deduplication.
     # The default values are usually fine, but some debug messages/warnings might recommend you
@@ -138,26 +171,6 @@ bridge:
         # You might need to increase this on high-traffic bridge instances.
         cache_queue_length: 20
 
-    # Show message editing as a reply to the original message.
-    # If this is false, message edits are not shown at all, as Matrix does not support editing yet.
-    edits_as_replies: false
-    bridge_notices:
-        # Whether or not Matrix bot messages (type m.notice) should be bridged.
-        default: false
-        # List of user IDs for whom the previous flag is flipped.
-        # e.g. if bridge_notices.default is false, notices from other users will not be bridged, but
-        #      notices from users listed here will be bridged.
-        exceptions:
-        - "@importantbot:example.com"
-    # Whether to bridge Telegram bot messages as m.notices or m.texts.
-    bot_messages_as_notices: true
-    # Use inline images instead of a separate message for the caption.
-    # N.B. Inline images are not supported on all clients (e.g. Riot iOS).
-    inline_images: false
-    # Whether to send stickers as the new native m.sticker type or normal m.images.
-    # Old versions of Riot don't support the new type at all.
-    # Remember that proper sticker support always requires Pillow to convert webp into png.
-    native_stickers: true
 
     # The formats to use when sending messages to Telegram via the relay bot.
     #
@@ -241,6 +254,52 @@ telegram:
     api_hash: tjyd5yge35lbodk1xwzw2jstp90k55qz
     # (Optional) Create your own bot at https://t.me/BotFather
     bot_token: disabled
+
+    # Telethon connection options.
+    connection:
+        # The timeout in seconds to be used when connecting.
+        timeout: 120
+        # How many times the reconnection should retry, either on the initial connection or when
+        # Telegram disconnects us. May be set to a negative or null value for infinite retries, but
+        # this is not recommended, since the program can get stuck in an infinite loop.
+        retries: 5
+        # The delay in seconds to sleep between automatic reconnections.
+        retry_delay: 1
+        # The threshold below which the library should automatically sleep on flood wait errors
+        # (inclusive). For instance, if a FloodWaitError for 17s occurs and flood_sleep_threshold
+        # is 20s, the library will sleep automatically. If the error was for 21s, it would raise
+        # the error instead. Values larger than a day (86400) will be changed to a day.
+        flood_sleep_threshold: 60
+        # How many times a request should be retried. Request are retried when Telegram is having
+        # internal issues, when there is a FloodWaitError less than flood_sleep_threshold, or when
+        # there's a migrate error. May take a negative or null value for infinite retries, but this
+        # is not recommended, since some requests can always trigger a call fail (such as searching
+        # for messages).
+        request_retries: 5
+
+    # Device info sent to Telegram.
+    device_info:
+        # "auto" = OS name+version.
+        device_model: auto
+        # "auto" = Telethon version.
+        system_version: auto
+        # "auto" = mautrix-telegram version.
+        app_version: auto
+        lang_code: en
+        system_lang_code: en
+
+    # Custom server to connect to.
+    server:
+        # Set to true to use these server settings. If false, will automatically
+        # use production server assigned by Telegram. Set to false in production.
+        enabled: false
+        # The DC ID to connect to.
+        dc: 2
+        # The IP to connect to.
+        ip: 149.154.167.40
+        # The port to connect to. 443 may not work, 80 is better and both are equally secure.
+        port: 80
+
     # Telethon proxy configuration.
     # You must install PySocks from pip for proxies to work.
     proxy:

mautrix_telegram/__init__.py

@@ -1,2 +1,2 @@
-__version__ = "0.4.0rc2"
+__version__ = "0.6.0rc1"
 __author__ = "Tulir Asokan <tulir@maunium.net>"

mautrix_telegram/__main__.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -14,15 +14,16 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Coroutine, List
+from typing import Awaitable, List, Any
+from time import time
 import argparse
 import asyncio
 import logging.config
 import sys
 import copy
 import signal
+import os
 
-from sqlalchemy import orm
 import sqlalchemy as sql
 
 from mautrix_appservice import AppService
@@ -31,11 +32,10 @@ from alchemysession import AlchemySessionContainer
 from .web.provisioning import ProvisioningAPI
 from .web.public import PublicBridgeWebsite
 from .abstract_user import init as init_abstract_user
-from .base import Base
 from .bot import init as init_bot
 from .config import Config
 from .context import Context
-from .db import init as init_db
+from .db import Base, init as init_db
 from .formatter import init as init_formatter
 from .matrix import MatrixHandler
 from .portal import init as init_portal
@@ -44,6 +44,11 @@ from .sqlstatestore import SQLStateStore
 from .user import User, init as init_user
 from . import __version__
 
+try:
+    import prometheus_client as prometheus
+except ImportError:
+    prometheus = None
+
 parser = argparse.ArgumentParser(
     description="A Matrix-Telegram puppeting bridge.",
     prog="python -m mautrix-telegram")
@@ -58,7 +63,7 @@ parser.add_argument("-r", "--registration", type=str, default="registration.yaml
                     metavar="<path>", help="the path to save the generated registration to")
 args = parser.parse_args()
 
-config = Config(args.config, args.registration, args.base_config)
+config = Config(args.config, args.registration, args.base_config, os.environ)
 config.load()
 config.update()
 
@@ -73,22 +78,23 @@ log = logging.getLogger("mau.init")  # type: logging.Logger
 log.debug(f"Initializing mautrix-telegram {__version__}")
 
 db_engine = sql.create_engine(config["appservice.database"] or "sqlite:///mautrix-telegram.db")
-db_factory = orm.sessionmaker(bind=db_engine)
-db_session = orm.scoping.scoped_session(db_factory)
 Base.metadata.bind = db_engine
 
-session_container = AlchemySessionContainer(engine=db_engine, session=db_session,
-                                            table_base=Base, table_prefix="telethon_",
-                                            manage_tables=False)
-if config["appservice.sqlalchemy_core_mode"]:
-    try:
-        session_container.core_mode = True
-    except AttributeError:
-        log.error("Current version of teleton-session-sqlalchemy does not support core mode")
+session_container = AlchemySessionContainer(engine=db_engine, table_base=Base, session=False,
+                                            table_prefix="telethon_", manage_tables=False)
+session_container.core_mode = True
+
+try:
+    import uvloop
+
+    asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
+    log.debug("Using uvloop for asyncio")
+except ImportError:
+    pass
 
 loop = asyncio.get_event_loop()  # type: asyncio.AbstractEventLoop
 
-state_store = SQLStateStore(db_session)
+state_store = SQLStateStore()
 mebibyte = 1024 ** 2
 appserv = AppService(config["homeserver.address"], config["homeserver.domain"],
                      config["appservice.as_token"], config["appservice.hs_token"],
@@ -98,8 +104,8 @@ appserv = AppService(config["homeserver.address"], config["homeserver.domain"],
                      aiohttp_params={
                          "client_max_size": config["appservice.max_body_size"] * mebibyte
                      })
-
-context = Context(appserv, db_session, config, loop, session_container)
+bot = init_bot(config)
+context = Context(appserv, config, loop, session_container, bot)
 
 if config["appservice.public.enabled"]:
     public_website = PublicBridgeWebsite(loop)
@@ -112,17 +118,23 @@ if config["appservice.provisioning.enabled"]:
                            provisioning_api.app)
     context.provisioning_api = provisioning_api
 
+context.mx = MatrixHandler(context)
+
+if config["metrics.enabled"]:
+    if prometheus:
+        prometheus.start_http_server(config["metrics.listen_port"])
+    else:
+        log.warn("Metrics are enabled in the config, but prometheus-async is not installed.")
+
 with appserv.run(config["appservice.hostname"], config["appservice.port"]) as start:
-    init_db(db_session, db_engine)
+    start_ts = time()
+    init_db(db_engine)
     init_abstract_user(context)
-    context.bot = init_bot(context)
-    context.mx = MatrixHandler(context)
     init_formatter(context)
     init_portal(context)
     startup_actions = (init_puppet(context) +
                        init_user(context) +
-                       [start,
-                        context.mx.init_as_bot()])  # type: List[Coroutine]
+                       [start, context.mx.init_as_bot()])  # type: List[Awaitable[Any]]
 
     if context.bot:
         startup_actions.append(context.bot.start())
@@ -130,10 +142,15 @@ with appserv.run(config["appservice.hostname"], config["appservice.port"]) as st
     signal.signal(signal.SIGINT, signal.default_int_handler)
     signal.signal(signal.SIGTERM, signal.default_int_handler)
 
+    end_ts = time()
     try:
-        log.debug("Initialization complete, running startup actions")
+        log.debug(f"Initialization complete in {round(end_ts - start_ts, 2)} seconds,"
+                  " running startup actions")
+        start_ts = time()
         loop.run_until_complete(asyncio.gather(*startup_actions, loop=loop))
-        log.debug("Startup actions complete, now running forever")
+        end_ts = time()
+        log.debug(f"Startup actions complete in {round(end_ts - start_ts, 2)} seconds,"
+                  " now running forever")
         loop.run_forever()
     except KeyboardInterrupt:
         log.debug("Interrupt received, stopping clients")
@@ -141,3 +158,6 @@ with appserv.run(config["appservice.hostname"], config["appservice.port"]) as st
             asyncio.gather(*[user.stop() for user in User.by_tgid.values()], loop=loop))
         log.debug("Clients stopped, shutting down")
         sys.exit(0)
+    except Exception as e:
+        log.exception("Unexpected error")
+        sys.exit(1)

mautrix_telegram/abstract_user.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -14,21 +14,21 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Tuple, Optional, List, Union, TYPE_CHECKING
+from typing import Tuple, Optional, List, Union, Dict, TYPE_CHECKING
 from abc import ABC, abstractmethod
 import asyncio
 import logging
 import platform
+import time
 
-from sqlalchemy import orm
-from telethon.tl.types import Channel, ChannelForbidden, Chat, ChatForbidden, Message, \
-    MessageActionChannelMigrateFrom, MessageService, PeerUser, TypeUpdate, \
-    UpdateChannelPinnedMessage, UpdateChatAdmins, UpdateChatParticipantAdmin, \
-    UpdateChatParticipants, UpdateChatUserTyping, UpdateDeleteChannelMessages, \
-    UpdateDeleteMessages, UpdateEditChannelMessage, UpdateEditMessage, UpdateNewChannelMessage, \
-    UpdateNewMessage, UpdateReadHistoryOutbox, UpdateShortChatMessage, UpdateShortMessage, \
-    UpdateUserName, UpdateUserPhoto, UpdateUserStatus, UpdateUserTyping, User, UserStatusOffline, \
-    UserStatusOnline
+from telethon.tl.patched import MessageService, Message
+from telethon.tl.types import (
+    Channel, ChannelForbidden, Chat, ChatForbidden, MessageActionChannelMigrateFrom, PeerUser,
+    TypeUpdate, UpdateChannelPinnedMessage, UpdateChatPinnedMessage, UpdateChatParticipantAdmin,
+    UpdateChatParticipants, UpdateChatUserTyping, UpdateDeleteChannelMessages, UpdateDeleteMessages,
+    UpdateEditChannelMessage, UpdateEditMessage, UpdateNewChannelMessage, UpdateNewMessage,
+    UpdateReadHistoryOutbox, UpdateShortChatMessage, UpdateShortMessage, UpdateUserName,
+    UpdateUserPhoto, UpdateUserStatus, UpdateUserTyping, User, UserStatusOffline, UserStatusOnline)
 
 from mautrix_appservice import MatrixRequestError, AppService
 from alchemysession import AlchemySessionContainer
@@ -51,12 +51,19 @@ UpdateMessage = Union[UpdateShortChatMessage, UpdateShortMessage, UpdateNewChann
                       UpdateNewMessage, UpdateEditMessage, UpdateEditChannelMessage]
 UpdateMessageContent = Union[UpdateShortMessage, UpdateShortChatMessage, Message, MessageService]
 
+try:
+    from prometheus_client import Histogram
+
+    UPDATE_TIME = Histogram("telegram_update", "Time spent processing Telegram updates",
+                            ["update_type"])
+except ImportError:
+    Histogram = None
+    UPDATE_TIME = None
 
 class AbstractUser(ABC):
     session_container = None  # type: AlchemySessionContainer
     loop = None  # type: asyncio.AbstractEventLoop
     log = None  # type: logging.Logger
-    db = None  # type: orm.Session
     az = None  # type: AppService
     bot = None  # type: Bot
     ignore_incoming_bot_events = True  # type: bool
@@ -97,18 +104,43 @@ class AbstractUser(ABC):
 
     def _init_client(self) -> None:
         self.log.debug(f"Initializing client for {self.name}")
-        device = f"{platform.system()} {platform.release()}"
-        sysversion = MautrixTelegramClient.__version__
+
         self.session = self.session_container.new_session(self.name)
-        self.client = MautrixTelegramClient(session=self.session,
-                                            api_id=config["telegram.api_id"],
-                                            api_hash=config["telegram.api_hash"],
-                                            loop=self.loop,
-                                            app_version=__version__,
-                                            system_version=sysversion,
-                                            device_model=device,
-                                            timeout=120,
-                                            proxy=self._proxy_settings)
+        if config["telegram.server.enabled"]:
+            self.session.set_dc(config["telegram.server.dc"],
+                                config["telegram.server.ip"],
+                                config["telegram.server.port"])
+
+        if self.is_relaybot:
+            base_logger = logging.getLogger("telethon.relaybot")
+        else:
+            base_logger = logging.getLogger(f"telethon.{self.tgid or -hash(self.mxid)}")
+
+        device = config["telegram.device_info.device_model"]
+        sysversion = config["telegram.device_info.system_version"]
+        appversion = config["telegram.device_info.app_version"]
+
+        self.client = MautrixTelegramClient(
+            session=self.session,
+
+            api_id=config["telegram.api_id"],
+            api_hash=config["telegram.api_hash"],
+
+            app_version=__version__ if appversion == "auto" else appversion,
+            system_version=MautrixTelegramClient.__version__ if sysversion == "auto" else sysversion,
+            device_model=f"{platform.system()} {platform.release()}" if device == "auto" else device,
+
+            timeout=config["telegram.connection.timeout"],
+            connection_retries=config["telegram.connection.retries"],
+            retry_delay=config["telegram.connection.retry_delay"],
+            flood_sleep_threshold=config["telegram.connection.flood_sleep_threshold"],
+            request_retries=config["telegram.connection.request_retries"],
+
+            proxy=self._proxy_settings,
+
+            loop=self.loop,
+            base_logger=base_logger
+        )
         self.client.add_event_handler(self._update_catch)
 
     @abstractmethod
@@ -128,11 +160,14 @@ class AbstractUser(ABC):
         raise NotImplementedError()
 
     async def _update_catch(self, update: TypeUpdate) -> None:
+        start_time = time.time()
         try:
             if not await self.update(update):
                 await self._update(update)
         except Exception:
             self.log.exception("Failed to handle Telegram update")
+        if UPDATE_TIME:
+            UPDATE_TIME.labels(update_type=type(update).__name__).observe(time.time() - start_time)
 
     async def get_dialogs(self, limit: int = None) -> List[Union[Chat, Channel]]:
         if self.is_bot:
@@ -164,16 +199,10 @@ class AbstractUser(ABC):
         return self
 
     async def ensure_started(self, even_if_no_session=False) -> 'AbstractUser':
-        if not self.puppet_whitelisted:
+        if not self.puppet_whitelisted or self.connected:
             return self
-        self.log.debug("ensure_started(%s, connected=%s, even_if_no_session=%s, session_count=%s)",
-                       self.mxid, self.connected, even_if_no_session,
-                       self.session_container.Session.query.filter(
-                           self.session_container.Session.session_id == self.mxid).count())
-        should_connect = (even_if_no_session or
-                          self.session_container.Session.query.filter(
-                              self.session_container.Session.session_id == self.mxid).count() > 0)
-        if not self.connected and should_connect:
+        self.log.debug("ensure_started(%s, even_if_no_session=%s)", self.mxid, even_if_no_session)
+        if even_if_no_session or self.session_container.has_session(self.mxid):
             await self.start(delete_unless_authenticated=not even_if_no_session)
         return self
 
@@ -184,6 +213,8 @@ class AbstractUser(ABC):
     # region Telegram update handling
 
     async def _update(self, update: TypeUpdate) -> None:
+        asyncio.ensure_future(self._handle_entity_updates(getattr(update, "_entities", {})),
+                              loop=self.loop)
         if isinstance(update, (UpdateShortChatMessage, UpdateShortMessage, UpdateNewChannelMessage,
                                UpdateNewMessage, UpdateEditMessage, UpdateEditChannelMessage)):
             await self.update_message(update)
@@ -195,11 +226,11 @@ class AbstractUser(ABC):
             await self.update_typing(update)
         elif isinstance(update, UpdateUserStatus):
             await self.update_status(update)
-        elif isinstance(update, (UpdateChatAdmins, UpdateChatParticipantAdmin)):
+        elif isinstance(update, UpdateChatParticipantAdmin):
             await self.update_admin(update)
         elif isinstance(update, UpdateChatParticipants):
             await self.update_participants(update)
-        elif isinstance(update, UpdateChannelPinnedMessage):
+        elif isinstance(update, (UpdateChannelPinnedMessage, UpdateChatPinnedMessage)):
             await self.update_pinned_messages(update)
         elif isinstance(update, (UpdateUserName, UpdateUserPhoto)):
             await self.update_others_info(update)
@@ -208,11 +239,14 @@ class AbstractUser(ABC):
         else:
             self.log.debug("Unhandled update: %s", update)
 
-    @staticmethod
-    async def update_pinned_messages(update: UpdateChannelPinnedMessage) -> None:
-        portal = po.Portal.get_by_tgid(TelegramID(update.channel_id))
+    async def update_pinned_messages(self, update: Union[UpdateChannelPinnedMessage,
+                                                         UpdateChatPinnedMessage]) -> None:
+        if isinstance(update, UpdateChatPinnedMessage):
+            portal = po.Portal.get_by_tgid(TelegramID(update.chat_id))
+        else:
+            portal = po.Portal.get_by_tgid(TelegramID(update.channel_id))
         if portal and portal.mxid:
-            await portal.receive_telegram_pin_id(update.id)
+            await portal.receive_telegram_pin_id(update.id, self.tgid)
 
     @staticmethod
     async def update_participants(update: UpdateChatParticipants) -> None:
@@ -230,26 +264,20 @@ class AbstractUser(ABC):
             return
 
         # We check that these are user read receipts, so tg_space is always the user ID.
-        message = DBMessage.get_by_tgid(update.max_id, self.tgid)
+        message = DBMessage.get_one_by_tgid(TelegramID(update.max_id), self.tgid, edit_index=-1)
         if not message:
             return
 
         puppet = pu.Puppet.get(TelegramID(update.peer.user_id))
         await puppet.intent.mark_read(portal.mxid, message.mxid)
 
-    async def update_admin(self,
-                           update: Union[UpdateChatAdmins, UpdateChatParticipantAdmin]) -> None:
+    async def update_admin(self, update: UpdateChatParticipantAdmin) -> None:
         # TODO duplication not checked
         portal = po.Portal.get_by_tgid(TelegramID(update.chat_id), peer_type="chat")
         if not portal or not portal.mxid:
             return
 
-        if isinstance(update, UpdateChatAdmins):
-            await portal.set_telegram_admins_enabled(update.enabled)
-        elif isinstance(update, UpdateChatParticipantAdmin):
-            await portal.set_telegram_admin(TelegramID(update.user_id))
-        else:
-            self.log.warning("Unexpected admin status update: %s", update)
+        await portal.set_telegram_admin(TelegramID(update.user_id))
 
     async def update_typing(self, update: Union[UpdateUserTyping, UpdateChatUserTyping]) -> None:
         if isinstance(update, UpdateUserTyping):
@@ -263,14 +291,25 @@ class AbstractUser(ABC):
         sender = pu.Puppet.get(TelegramID(update.user_id))
         await portal.handle_telegram_typing(sender, update)
 
+    async def _handle_entity_updates(self, entities: Dict[int, Union[User, Chat, Channel]]
+                                     ) -> None:
+        try:
+            users = (entity for entity in entities.values() if isinstance(entity, User))
+            puppets = ((pu.Puppet.get(TelegramID(user.id)), user) for user in users)
+            await asyncio.gather(*[puppet.update_info(self, info)
+                                   for puppet, info in puppets if puppet])
+        except Exception:
+            self.log.exception("Failed to handle entity updates")
+
     async def update_others_info(self, update: Union[UpdateUserName, UpdateUserPhoto]) -> None:
         # TODO duplication not checked
         puppet = pu.Puppet.get(TelegramID(update.user_id))
         if isinstance(update, UpdateUserName):
+            puppet.username = update.username
             if await puppet.update_displayname(self, update):
                 puppet.save()
         elif isinstance(update, UpdateUserPhoto):
-            if await puppet.update_avatar(self, update.photo.photo_big):
+            if await puppet.update_avatar(self, update.photo):
                 puppet.save()
         else:
             self.log.warning("Unexpected other user info update: %s", update)
@@ -310,7 +349,8 @@ class AbstractUser(ABC):
         return update, sender, portal
 
     @staticmethod
-    async def _try_redact(portal: po.Portal, message: DBMessage) -> None:
+    async def _try_redact(message: DBMessage) -> None:
+        portal = po.Portal.get_by_mxid(message.mx_room)
         if not portal:
             return
         try:
@@ -322,32 +362,26 @@ class AbstractUser(ABC):
         if len(update.messages) > MAX_DELETIONS:
             return
 
-        for message in update.messages:
-            message = DBMessage.get_by_tgid(TelegramID(message), self.tgid)
-            if not message:
-                continue
-            message.delete()
-            number_left = DBMessage.count_spaces_by_mxid(message.mxid, message.mx_room)
-            if number_left == 0:
-                portal = po.Portal.get_by_mxid(message.mx_room)
-                await self._try_redact(portal, message)
-        self.db.commit()
+        for message_id in update.messages:
+            messages = DBMessage.get_all_by_tgid(TelegramID(message_id), self.tgid)
+            for message in messages:
+                message.delete()
+                number_left = DBMessage.count_spaces_by_mxid(message.mxid, message.mx_room)
+                if number_left == 0:
+                    portal = po.Portal.get_by_mxid(message.mx_room)
+                    await self._try_redact(message)
 
     async def delete_channel_message(self, update: UpdateDeleteChannelMessages) -> None:
         if len(update.messages) > MAX_DELETIONS:
             return
 
-        portal = po.Portal.get_by_tgid(TelegramID(update.channel_id))
-        if not portal:
-            return
+        channel_id = TelegramID(update.channel_id)
 
-        for message in update.messages:
-            message = DBMessage.get_by_tgid(TelegramID(message), portal.tgid)
-            if not message:
-                continue
-            message.delete()
-            await self._try_redact(portal, message)
-        self.db.commit()
+        for message_id in update.messages:
+            messages = DBMessage.get_all_by_tgid(TelegramID(message_id), channel_id)
+            for message in messages:
+                message.delete()
+                await self._try_redact(message)
 
     async def update_message(self, original_update: UpdateMessage) -> None:
         update, sender, portal = self.get_message_details(original_update)
@@ -373,10 +407,7 @@ class AbstractUser(ABC):
 
         user = sender.tgid if sender else "admin"
         if isinstance(original_update, (UpdateEditMessage, UpdateEditChannelMessage)):
-            if config["bridge.edits_as_replies"]:
-                self.log.debug("Handling edit %s to %s by %s", update, portal.tgid_log, user)
-                return await portal.handle_telegram_edit(self, sender, update)
-            return
+            return await portal.handle_telegram_edit(self, sender, update)
 
         self.log.debug("Handling message %s to %s by %s", update, portal.tgid_log, user)
         return await portal.handle_telegram_message(self, sender, update)
@@ -386,7 +417,7 @@ class AbstractUser(ABC):
 
 def init(context: "Context") -> None:
     global config, MAX_DELETIONS
-    AbstractUser.az, AbstractUser.db, config, AbstractUser.loop, AbstractUser.relaybot = context.core
+    AbstractUser.az, config, AbstractUser.loop, AbstractUser.relaybot = context.core
     AbstractUser.ignore_incoming_bot_events = config["bridge.relaybot.ignore_own_incoming_events"]
     AbstractUser.session_container = context.session_container
     MAX_DELETIONS = config.get("bridge.max_telegram_delete", 10)

mautrix_telegram/base.py

@@ -1,2 +0,0 @@
-from sqlalchemy.ext.declarative import declarative_base
-Base = declarative_base()  # type: declarative_base

mautrix_telegram/bot.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -14,15 +14,16 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Awaitable, Callable, Dict, List, Optional, Pattern, TYPE_CHECKING
+from typing import Awaitable, Callable, Dict, List, Optional, Pattern, Tuple, TYPE_CHECKING
 import logging
 import re
 
+from telethon.tl.patched import Message, MessageService
 from telethon.tl.types import (
     ChannelParticipantAdmin, ChannelParticipantCreator, ChatForbidden, ChatParticipantAdmin,
-    ChatParticipantCreator, InputChannel, InputUser, Message, MessageActionChatAddUser,
-    MessageActionChatDeleteUser, MessageEntityBotCommand, MessageService, PeerChannel, PeerChat,
-    TypePeer, UpdateNewChannelMessage, UpdateNewMessage)
+    ChatParticipantCreator, InputChannel, InputUser, MessageActionChatAddUser,
+    MessageActionChatDeleteUser, MessageEntityBotCommand, PeerChannel, PeerChat, TypePeer,
+    UpdateNewChannelMessage, UpdateNewMessage, MessageActionChatMigrateTo, User)
 from telethon.tl.functions.messages import GetChatsRequest, GetFullChatRequest
 from telethon.tl.functions.channels import GetChannelsRequest, GetParticipantRequest
 from telethon.errors import ChannelInvalidError, ChannelPrivateError
@@ -30,6 +31,7 @@ from telethon.errors import ChannelInvalidError, ChannelPrivateError
 from .types import MatrixUserID
 from .abstract_user import AbstractUser
 from .db import BotChat
+from .types import TelegramID
 from . import puppet as pu, portal as po, user as u
 
 if TYPE_CHECKING:
@@ -54,10 +56,18 @@ class Bot(AbstractUser):
         self.username = None  # type: str
         self.is_relaybot = True  # type: bool
         self.is_bot = True  # type: bool
-        self.chats = {chat.id: chat.type for chat in BotChat.query.all()}  # type: Dict[int, str]
+        self.chats = {}  # type: Dict[int, str]
         self.tg_whitelist = []  # type: List[int]
         self.whitelist_group_admins = (config["bridge.relaybot.whitelist_group_admins"]
                                        or False)  # type: bool
+        self._me_info = None  # type: Optional[User]
+        self._me_mxid = None  # type: Optional[MatrixUserID]
+
+    async def get_me(self, use_cache: bool = True) -> Tuple[User, MatrixUserID]:
+        if not use_cache or not self._me_mxid:
+            self._me_info = await self.client.get_me()
+            self._me_mxid = pu.Puppet.get_mxid_from_id(TelegramID(self._me_info.id))
+        return self._me_info, self._me_mxid
 
     async def init_permissions(self) -> None:
         whitelist = config["bridge.relaybot.whitelist"] or []
@@ -72,6 +82,7 @@ class Bot(AbstractUser):
                 self.tg_whitelist.append(user_id)
 
     async def start(self, delete_unless_authenticated: bool = False) -> 'Bot':
+        self.chats = {chat.id: chat.type for chat in BotChat.all()}
         await super().start(delete_unless_authenticated)
         if not await self.is_logged_in():
             await self.client.sign_in(bot_token=self.token)
@@ -89,7 +100,7 @@ class Bot(AbstractUser):
         response = await self.client(GetChatsRequest(chat_ids))
         for chat in response.chats:
             if isinstance(chat, ChatForbidden) or chat.left or chat.deactivated:
-                self.remove_chat(chat.id)
+                self.remove_chat(TelegramID(chat.id))
 
         channel_ids = [InputChannel(chat_id, 0)
                        for chat_id, chat_type in self.chats.items()
@@ -98,7 +109,7 @@ class Bot(AbstractUser):
             try:
                 await self.client(GetChannelsRequest([channel_id]))
             except (ChannelPrivateError, ChannelInvalidError):
-                self.remove_chat(channel_id.channel_id)
+                self.remove_chat(TelegramID(channel_id.channel_id))
 
         if config["bridge.catch_up"]:
             try:
@@ -112,23 +123,19 @@ class Bot(AbstractUser):
     def unregister_portal(self, portal: po.Portal) -> None:
         self.remove_chat(portal.tgid)
 
-    def add_chat(self, chat_id: int, chat_type: str) -> None:
+    def add_chat(self, chat_id: TelegramID, chat_type: str) -> None:
         if chat_id not in self.chats:
             self.chats[chat_id] = chat_type
-            self.db.add(BotChat(id=chat_id, type=chat_type))
-            self.db.commit()
+            BotChat(id=TelegramID(chat_id), type=chat_type).insert()
 
-    def remove_chat(self, chat_id: int) -> None:
+    def remove_chat(self, chat_id: TelegramID) -> None:
         try:
             del self.chats[chat_id]
         except KeyError:
             pass
-        existing_chat = BotChat.query.get(chat_id)
-        if existing_chat:
-            self.db.delete(existing_chat)
-            self.db.commit()
+        BotChat.delete(chat_id)
 
-    async def _can_use_commands(self, chat: TypePeer, tgid: int) -> bool:
+    async def _can_use_commands(self, chat: TypePeer, tgid: TelegramID) -> bool:
         if tgid in self.tg_whitelist:
             return True
 
@@ -155,7 +162,7 @@ class Bot(AbstractUser):
             return False
         return True
 
-    async def handle_command_portal(self, portal: po.Portal, reply: ReplyFunc) -> None:
+    async def handle_command_portal(self, portal: po.Portal, reply: ReplyFunc) -> Message:
         if not config["bridge.relaybot.authless_portals"]:
             return await reply("This bridge doesn't allow portal creation from Telegram.")
 
@@ -221,7 +228,8 @@ class Bot(AbstractUser):
         text = message.message
 
         if self.match_command(text, "id"):
-            return await self.handle_command_id(message, reply)
+            await self.handle_command_id(message, reply)
+            return
 
         portal = po.Portal.get_by_entity(message.to_id)
 
@@ -239,7 +247,7 @@ class Bot(AbstractUser):
             await self.handle_command_invite(portal, reply, mxid_input=mxid)
 
     def handle_service_message(self, message: MessageService) -> None:
-        to_id = message.to_id
+        to_id = message.to_id  # type: TelegramID
         if isinstance(to_id, PeerChannel):
             to_id = to_id.channel_id
             chat_type = "channel"
@@ -254,6 +262,9 @@ class Bot(AbstractUser):
             self.add_chat(to_id, chat_type)
         elif isinstance(action, MessageActionChatDeleteUser) and action.user_id == self.tgid:
             self.remove_chat(to_id)
+        elif isinstance(action, MessageActionChatMigrateTo):
+            self.remove_chat(to_id)
+            self.add_chat(TelegramID(action.channel_id), "channel")
 
     async def update(self, update) -> bool:
         if not isinstance(update, (UpdateNewMessage, UpdateNewChannelMessage)):
@@ -278,9 +289,9 @@ class Bot(AbstractUser):
         return "bot"
 
 
-def init(context: 'Context') -> Optional[Bot]:
+def init(cfg: 'Config') -> Optional[Bot]:
     global config
-    config = context.config
+    config = cfg
     token = config["telegram.bot_token"]
     if token and not token.lower().startswith("disable"):
         return Bot(token)

mautrix_telegram/commands/__init__.py

@@ -2,4 +2,4 @@ from .handler import (command_handler, command_handlers as _command_handlers,
                       CommandHandler, CommandProcessor, CommandEvent,
                       SECTION_GENERAL, SECTION_AUTH, SECTION_CREATING_PORTALS,
                       SECTION_PORTAL_MANAGEMENT, SECTION_MISC, SECTION_ADMIN)
-from . import clean_rooms, auth, meta, telegram, portal
+from . import portal, telegram, clean_rooms, matrix_auth, meta

mautrix_telegram/commands/clean_rooms.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -70,7 +70,7 @@ async def clean_rooms(evt: CommandEvent) -> Optional[Dict]:
                for n, (room, other_member) in enumerate(management_rooms)]
               or ["No management rooms found."])
     reply.append("#### Active portal rooms (A)")
-    reply += ([f"{n+1}. [P{n+1}](https://matrix.to/#/{portal.mxid}) "
+    reply += ([f"{n+1}. [A{n+1}](https://matrix.to/#/{portal.mxid}) "
                f"(to Telegram chat \"{portal.title}\")"
                for n, portal in enumerate(portals)]
               or ["No active portal rooms found."])
@@ -79,7 +79,7 @@ async def clean_rooms(evt: CommandEvent) -> Optional[Dict]:
                for n, room in enumerate(unidentified_rooms)]
               or ["No unidentified rooms found."])
     reply.append("#### Inactive portal rooms (I)")
-    reply += ([f"{n}. [E{n}](https://matrix.to/#/{portal.mxid}) "
+    reply += ([f"{n}. [I{n}](https://matrix.to/#/{portal.mxid}) "
                f"(to Telegram chat \"{portal.title}\")"
                for n, portal in enumerate(empty_portals)]
               or ["No inactive portal rooms found."])
@@ -93,7 +93,7 @@ async def clean_rooms(evt: CommandEvent) -> Optional[Dict]:
               "",
               ("To clean specific rooms, type `$cmdprefix+sp clean-range <range>` "
                "where `range` is the range (e.g. `5-21`) prefixed with the first letter of"
-               "the group name."),
+               "the group name. (e.g. `I2-6`)"),
               "",
               ("Please note that you will have to re-run `$cmdprefix+sp clean-rooms` "
                "between each use of the commands above.")]
@@ -118,7 +118,7 @@ async def set_rooms_to_clean(evt, management_rooms: List[ManagementRoom],
     elif command == "clean-groups":
         if len(evt.args) < 2:
             return await evt.reply("**Usage:** `$cmdprefix+sp clean-groups [M][A][U][I]")
-        groups_to_clean = evt.args[1]
+        groups_to_clean = evt.args[1].upper()
         if "M" in groups_to_clean:
             rooms_to_clean += [room_id for (room_id, user_id) in management_rooms]
         if "A" in groups_to_clean:

mautrix_telegram/commands/handler.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -14,13 +14,16 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
+"""This module contains classes handling commands issued by Matrix users."""
 from typing import Awaitable, Callable, Dict, List, NamedTuple, Optional
-import commonmark
 import logging
+import traceback
+
+import commonmark
 
 from telethon.errors import FloodWaitError
 
-from ..types import MatrixRoomID
+from ..types import MatrixRoomID, MatrixEventID
 from ..util import format_duration
 from .. import user as u, context as c
 
@@ -57,9 +60,32 @@ md_parser = commonmark.Parser()
 md_renderer = HtmlEscapingRenderer()
 
 
+def ensure_trailing_newline(s: str) -> str:
+    """Returns the passed string, but with a guaranteed trailing newline."""
+    return s + ("" if s[-1] == "\n" else "\n")
+
+
 class CommandEvent:
-    def __init__(self, processor: 'CommandProcessor', room: MatrixRoomID, sender: u.User,
-                 command: str, args: List[str], is_management: bool, is_portal: bool) -> None:
+    """Holds information about a command issued in a Matrix room.
+
+    When a Matrix command was issued to the bot, CommandEvent will hold
+    information regarding the event.
+
+    Attributes:
+        room_id: The id of the Matrix room in which the command was issued.
+        event_id: The id of the matrix event which contained the command.
+        sender: The user who issued the command.
+        command: The issued command.
+        args: Arguments given with the issued command.
+        is_management: Determines whether the room in which the command wa
+            issued is a management room.
+        is_portal: Determines whether the room in which the command was issued
+            is a portal.
+    """
+
+    def __init__(self, processor: 'CommandProcessor', room: MatrixRoomID, event: MatrixEventID,
+                 sender: u.User, command: str, args: List[str], is_management: bool,
+                 is_portal: bool) -> None:
         self.az = processor.az
         self.log = processor.log
         self.loop = processor.loop
@@ -68,6 +94,7 @@ class CommandEvent:
         self.public_website = processor.public_website
         self.command_prefix = processor.command_prefix
         self.room_id = room
+        self.event_id = event
         self.sender = sender
         self.command = command
         self.args = args
@@ -76,23 +103,102 @@ class CommandEvent:
 
     def reply(self, message: str, allow_html: bool = False, render_markdown: bool = True
               ) -> Awaitable[Dict]:
-        message = message.replace("$cmdprefix+sp ",
-                                  "" if self.is_management else f"{self.command_prefix} ")
-        message = message.replace("$cmdprefix", self.command_prefix)
-        html = None
+        """Write a reply to the room in which the command was issued.
+
+        Replaces occurences of "$cmdprefix" in the message with the command
+        prefix and replaces occurences of "$cmdprefix+sp " with the command
+        prefix if the command was not issued in a management room.
+        If allow_html and render_markdown are both False, the message will not
+        be rendered to html and sending of html is disabled.
+
+        Args:
+            message: The message to post in the room.
+            allow_html: Escape html in the message or don't render html at all
+                if markdown is disabled.
+            render_markdown: Use markdown formatting to render the passed
+                message to html.
+
+        Returns:
+            Handler for the message sending function.
+        """
+        message_cmd = self._replace_command_prefix(message)
+        html = self._render_message(message_cmd, allow_html=allow_html,
+                                    render_markdown=render_markdown)
+
+        return self.az.intent.send_notice(self.room_id, message_cmd, html=html)
+
+    def mark_read(self) -> Awaitable[Dict]:
+        """Marks the command as read by the bot."""
+        return self.az.intent.mark_read(self.room_id, self.event_id)
+
+    def _replace_command_prefix(self, message: str) -> str:
+        """Returns the string with the proper command prefix entered."""
+        message = message.replace(
+            "$cmdprefix+sp ", "" if self.is_management else f"{self.command_prefix} "
+        )
+        return message.replace("$cmdprefix", self.command_prefix)
+
+    @staticmethod
+    def _render_message(message: str, allow_html: bool, render_markdown: bool) -> Optional[str]:
+        """Renders the message as HTML.
+
+        Args:
+            allow_html: Flag to allow custom HTML in the message.
+            render_markdown: If true, markdown styling is applied to the message.
+
+        Returns:
+            The message rendered as HTML.
+            None is returned if no styled output is required.
+        """
+        html = ""
         if render_markdown:
             md_renderer.allow_html = allow_html
             html = md_renderer.render(md_parser.parse(message))
         elif allow_html:
             html = message
-        return self.az.intent.send_notice(self.room_id, message, html=html)
+        return ensure_trailing_newline(html) if html else None
 
 
 class CommandHandler:
+    """A command which can be executed from a Matrix room.
+
+    The command manages its permission and help texts.
+    When called, it will check the permission of the command event and execute
+    the command or, in case of error, report back to the user.
+
+    Attributes:
+        needs_auth: Flag indicating if the sender is required to be logged in.
+        needs_puppeting: Flag indicating if the sender is required to use
+            Telegram puppeteering for this command.
+        needs_matrix_puppeting: Flag indicating if the sender is required to use
+            Matrix pupeteering.
+        needs_admin: Flag for whether only admin users can issue this command.
+        management_only: Whether the command can exclusively be issued in a
+            management room.
+        name: The name of this command.
+        help_section: Section of the help in which this command will appear.
+    """
+
     def __init__(self, handler: Callable[[CommandEvent], Awaitable[Dict]], needs_auth: bool,
                  needs_puppeting: bool, needs_matrix_puppeting: bool, needs_admin: bool,
                  management_only: bool, name: str, help_text: str, help_args: str,
                  help_section: HelpSection) -> None:
+        """
+        Args:
+            handler: The function handling the execution of this command.
+            needs_auth: Flag indicating if the sender is required to be logged in.
+            needs_puppeting: Flag indicating if the sender is required to use
+                Telegram puppeteering for this command.
+            needs_matrix_puppeting: Flag indicating if the sender is required to
+                use Matrix pupeteering.
+            needs_admin: Flag for whether only admin users can issue this command.
+            management_only: Whether the command can exclusively be issued
+                in a management room.
+            name: The name of this command.
+            help_text: The text displayed in the help for this command.
+            help_args: Help text for the arguments of this command.
+            help_section: Section of the help in which this command will appear.
+        """
         self._handler = handler
         self.needs_auth = needs_auth
         self.needs_puppeting = needs_puppeting
@@ -105,6 +211,14 @@ class CommandHandler:
         self.help_section = help_section
 
     async def get_permission_error(self, evt: CommandEvent) -> Optional[str]:
+        """Returns the reason why the command could not be issued.
+
+        Args:
+            evt: The event for which to get the error information.
+
+        Returns:
+            A string describing the error or None if there was no error.
+        """
         if self.management_only and not evt.is_management:
             return (f"`{evt.command}` is a restricted command: "
                     "you may only run it in management rooms.")
@@ -120,14 +234,40 @@ class CommandHandler:
 
     def has_permission(self, is_management: bool, puppet_whitelisted: bool,
                        matrix_puppet_whitelisted: bool, is_admin: bool, is_logged_in: bool) -> bool:
+        """Checks the permission for this command with the given status.
+
+        Args:
+            is_management: If the room in which the command will be issued is a
+                management room.
+            puppet_whitelisted: If the connected Telegram account puppet is
+                allowed to issue the command.
+            matrix_puppet_whitelisted: If the connected Matrix account puppet is
+                allowed to issue the command.
+            is_admin: If the issuing user is an admin.
+            is_logged_in: If the issuing user is logged in.
+
+        Returns:
+            True if a user with the given state is allowed to issue the
+            command.
+        """
         return ((not self.management_only or is_management) and
                 (not self.needs_puppeting or puppet_whitelisted) and
                 (not self.needs_matrix_puppeting or matrix_puppet_whitelisted) and
                 (not self.needs_admin or is_admin) and
                 (not self.needs_auth or is_logged_in))
 
-    async def __call__(self, evt: CommandEvent
-                       ) -> Dict:
+    async def __call__(self, evt: CommandEvent) -> Dict:
+        """Executes the command if evt was issued with proper rights.
+
+        Args:
+            evt: The CommandEvent for which to check permissions.
+
+        Returns:
+            The result of the command or the error message function.
+
+        Raises:
+            FloodWaitError
+        """
         error = await self.get_permission_error(evt)
         if error is not None:
             return await evt.reply(error)
@@ -135,31 +275,26 @@ class CommandHandler:
 
     @property
     def has_help(self) -> bool:
+        """Returns true if this command has a help text."""
         return bool(self.help_section) and bool(self._help_text)
 
     @property
     def help(self) -> str:
+        """Returns the help text to this command."""
         return f"**{self.name}** {self._help_args} - {self._help_text}"
 
 
 def command_handler(_func: Optional[Callable[[CommandEvent], Awaitable[Dict]]] = None, *,
-                    needs_auth: bool = True,
-                    needs_puppeting: bool = True,
-                    needs_matrix_puppeting: bool = False,
-                    needs_admin: bool = False,
-                    management_only: bool = False,
-                    name: Optional[str] = None,
-                    help_text: str = "",
-                    help_args: str = "",
-                    help_section: HelpSection = None
+                    needs_auth: bool = True, needs_puppeting: bool = True,
+                    needs_matrix_puppeting: bool = False, needs_admin: bool = False,
+                    management_only: bool = False, name: Optional[str] = None,
+                    help_text: str = "", help_args: str = "", help_section: HelpSection = None
                     ) -> Callable[[Callable[[CommandEvent], Awaitable[Optional[Dict]]]],
                                   CommandHandler]:
-    input_name = name
-
     def decorator(func: Callable[[CommandEvent], Awaitable[Optional[Dict]]]) -> CommandHandler:
-        name = input_name or func.__name__.replace("_", "-")
+        actual_name = name or func.__name__.replace("_", "-")
         handler = CommandHandler(func, needs_auth, needs_puppeting, needs_matrix_puppeting,
-                                 needs_admin, management_only, name, help_text, help_args,
+                                 needs_admin, management_only, actual_name, help_text, help_args,
                                  help_section)
         command_handlers[handler.name] = handler
         return handler
@@ -168,16 +303,40 @@ def command_handler(_func: Optional[Callable[[CommandEvent], Awaitable[Dict]]] =
 
 
 class CommandProcessor:
+    """Handles the raw commands issued by a user to the Matrix bot."""
     log = logging.getLogger("mau.commands")
 
     def __init__(self, context: c.Context) -> None:
-        self.az, self.db, self.config, self.loop, self.tgbot = context.core
+        self.az, self.config, self.loop, self.tgbot = context.core
         self.public_website = context.public_website
         self.command_prefix = self.config["bridge.command_prefix"]
 
-    async def handle(self, room: MatrixRoomID, sender: u.User, command: str, args: List[str],
-                     is_management: bool, is_portal: bool) -> Optional[Dict]:
-        evt = CommandEvent(self, room, sender, command, args, is_management, is_portal)
+    async def handle(self, room: MatrixRoomID, event_id: MatrixEventID, sender: u.User,
+                     command: str, args: List[str], is_management: bool, is_portal: bool
+                     ) -> Optional[Dict]:
+        """Handles the raw commands issued by a user to the Matrix bot.
+
+        If the command is not known, it might be a followup command and is
+        delegated to a command handler registered for that purpose in the
+        senders command_status as "next".
+
+        Args:
+            room: ID of the Matrix room in which the command was issued.
+            event_id: ID of the event by which the command was issued.
+            sender: The sender who issued the command.
+            command: The issued command, case insensitive.
+            args: Arguments given with the command.
+            is_management: Whether the room is a management room.
+            is_portal: Whether the room is a portal.
+
+        Returns:
+            The result of the error message function or None if no error
+            occured. Unknown and delegated commands do not count as errors.
+        """
+        if not command_handlers or "unknown-command" not in command_handlers:
+            raise ValueError("command_handlers are not properly initialized.")
+
+        evt = CommandEvent(self, room, event_id, sender, command, args, is_management, is_portal)
         orig_command = command
         command = command.lower()
         try:
@@ -196,6 +355,11 @@ class CommandProcessor:
         except Exception:
             self.log.exception("Unhandled error while handling command "
                                f"{evt.command} {' '.join(args)} from {sender.mxid}")
+            if evt.sender.is_admin and evt.is_management:
+                return await evt.reply("Unhandled error while handling command:\n\n"
+                                       "```traceback\n"
+                                       f"{traceback.format_exc()}"
+                                       "```")
             return await evt.reply("Unhandled error while handling command. "
                                    "Check logs for more details.")
         return None

mautrix_telegram/commands/matrix_auth.py

@@ -0,0 +1,101 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Dict, Optional
+
+from . import command_handler, CommandEvent, SECTION_AUTH
+from .. import puppet as pu
+
+
+@command_handler(needs_auth=True, needs_matrix_puppeting=True,
+                 help_section=SECTION_AUTH,
+                 help_text="Revert your Telegram account's Matrix puppet to use the default Matrix "
+                           "account.")
+async def logout_matrix(evt: CommandEvent) -> Optional[Dict]:
+    puppet = pu.Puppet.get(evt.sender.tgid)
+    if not puppet.is_real_user:
+        return await evt.reply("You are not logged in with your Matrix account.")
+    await puppet.switch_mxid(None, None)
+    return await evt.reply("Reverted your Telegram account's Matrix puppet back to the default.")
+
+
+@command_handler(needs_auth=True, management_only=True, needs_matrix_puppeting=True,
+                 help_section=SECTION_AUTH,
+                 help_text="Replace your Telegram account's Matrix puppet with your own Matrix "
+                           "account.")
+async def login_matrix(evt: CommandEvent) -> Optional[Dict]:
+    puppet = pu.Puppet.get(evt.sender.tgid)
+    if puppet.is_real_user:
+        return await evt.reply("You have already logged in with your Matrix account. "
+                               "Log out with `$cmdprefix+sp logout-matrix` first.")
+    allow_matrix_login = evt.config.get("bridge.allow_matrix_login", True)
+    if allow_matrix_login:
+        evt.sender.command_status = {
+            "next": enter_matrix_token,
+            "action": "Matrix login",
+        }
+    if evt.config["appservice.public.enabled"]:
+        prefix = evt.config["appservice.public.external"]
+        token = evt.public_website.make_token(evt.sender.mxid, "/matrix-login")
+        url = f"{prefix}/matrix-login?token={token}"
+        if allow_matrix_login:
+            return await evt.reply(
+                "This bridge instance allows you to log in inside or outside Matrix.\n\n"
+                "If you would like to log in within Matrix, please send your Matrix access token "
+                "here.\n"
+                f"If you would like to log in outside of Matrix, [click here]({url}).\n\n"
+                "Logging in outside of Matrix is recommended, because in-Matrix login would save "
+                "your access token in the message history.")
+        return await evt.reply("This bridge instance does not allow logging in inside Matrix.\n\n"
+                               f"Please visit [the login page]({url}) to log in.")
+    elif allow_matrix_login:
+        return await evt.reply(
+            "This bridge instance does not allow you to log in outside of Matrix.\n\n"
+            "Please send your Matrix access token here to log in.")
+    return await evt.reply("This bridge instance has been configured to not allow logging in.")
+
+
+@command_handler(needs_auth=True, needs_matrix_puppeting=True,
+                 help_section=SECTION_AUTH,
+                 help_text="Pings the server with the stored matrix authentication.")
+async def ping_matrix(evt: CommandEvent) -> Optional[Dict]:
+    puppet = pu.Puppet.get(evt.sender.tgid)
+    if not puppet.is_real_user:
+        return await evt.reply("You are not logged in with your Matrix account.")
+    resp = await puppet.init_custom_mxid()
+    if resp == pu.PuppetError.InvalidAccessToken:
+        return await evt.reply("Your access token is invalid.")
+    elif resp == pu.PuppetError.Success:
+        return await evt.reply("Your Matrix login is working.")
+    return await evt.reply(f"Unknown response while checking your Matrix login: {resp}.")
+
+
+async def enter_matrix_token(evt: CommandEvent) -> Dict:
+    evt.sender.command_status = None
+
+    puppet = pu.Puppet.get(evt.sender.tgid)
+    if puppet.is_real_user:
+        return await evt.reply("You have already logged in with your Matrix account. "
+                               "Log out with `$cmdprefix+sp logout-matrix` first.")
+
+    resp = await puppet.switch_mxid(" ".join(evt.args), evt.sender.mxid)
+    if resp == pu.PuppetError.OnlyLoginSelf:
+        return await evt.reply("You can only log in as your own Matrix user.")
+    elif resp == pu.PuppetError.InvalidAccessToken:
+        return await evt.reply("Failed to verify access token.")
+    assert resp == pu.PuppetError.Success, "Encountered an unhandled PuppetError."
+    return await evt.reply(
+        f"Replaced your Telegram account's Matrix puppet with {puppet.custom_mxid}.")

mautrix_telegram/commands/meta.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -51,8 +51,8 @@ async def _get_help_text(evt: CommandEvent) -> str:
                 help_sections.setdefault(handler.help_section, [])
                 help_sections[handler.help_section].append(handler.help + "  ")
         help_sorted = sorted(help_sections.items(), key=lambda item: item[0].order)
-        help = ["#### {}\n{}\n".format(key.name, "\n".join(value)) for key, value in help_sorted]
-        help_cache[cache_key] = "\n".join(help)
+        helps = ["#### {}\n{}\n".format(key.name, "\n".join(value)) for key, value in help_sorted]
+        help_cache[cache_key] = "\n".join(helps)
     return help_cache[cache_key]
 
 
@@ -65,8 +65,8 @@ def _get_management_status(evt: CommandEvent) -> str:
     return "**This is not a management room**: you must prefix commands with `$cmdprefix`."
 
 
-@command_handler(needs_auth=False, needs_puppeting=False,
+@command_handler(name="help", needs_auth=False, needs_puppeting=False,
                  help_section=SECTION_GENERAL,
                  help_text="Show this help message.")
-async def help(evt: CommandEvent) -> Optional[Dict]:
+async def help_cmd(evt: CommandEvent) -> Optional[Dict]:
     return await evt.reply(_get_management_status(evt) + "\n" + await _get_help_text(evt))

mautrix_telegram/commands/portal.py

@@ -1,614 +0,0 @@
-# -*- coding: future_fstrings -*-
-# mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Dict, Callable, Optional, Tuple, Coroutine, Awaitable
-from io import StringIO
-import asyncio
-
-from telethon.errors import (ChatAdminRequiredError, UsernameInvalidError,
-                             UsernameNotModifiedError, UsernameOccupiedError)
-from telethon.tl.types import ChatForbidden, ChannelForbidden
-from mautrix_appservice import MatrixRequestError, IntentAPI
-
-from ..types import MatrixRoomID, TelegramID
-from ..config import yaml
-from .. import portal as po, user as u, util
-from . import (command_handler, CommandEvent,
-               SECTION_ADMIN, SECTION_CREATING_PORTALS, SECTION_PORTAL_MANAGEMENT)
-
-
-@command_handler(needs_admin=True, needs_auth=False, name="set-pl",
-                 help_section=SECTION_ADMIN,
-                 help_args="<_level_> [_mxid_]",
-                 help_text="Set a temporary power level without affecting Telegram.")
-async def set_power_level(evt: CommandEvent) -> Dict:
-    try:
-        level = int(evt.args[0])
-    except KeyError:
-        return await evt.reply("**Usage:** `$cmdprefix+sp set-power <level> [mxid]`")
-    except ValueError:
-        return await evt.reply("The level must be an integer.")
-    levels = await evt.az.intent.get_power_levels(evt.room_id)
-    mxid = evt.args[1] if len(evt.args) > 1 else evt.sender.mxid
-    levels["users"][mxid] = level
-    try:
-        await evt.az.intent.set_power_levels(evt.room_id, levels)
-    except MatrixRequestError:
-        evt.log.exception("Failed to set power level.")
-        return await evt.reply("Failed to set power level.")
-    return {}
-
-
-@command_handler(help_section=SECTION_PORTAL_MANAGEMENT,
-                 help_text="Get a Telegram invite link to the current chat.")
-async def invite_link(evt: CommandEvent) -> Dict:
-    portal = po.Portal.get_by_mxid(evt.room_id)
-    if not portal:
-        return await evt.reply("This is not a portal room.")
-
-    if portal.peer_type == "user":
-        return await evt.reply("You can't invite users to private chats.")
-
-    try:
-        link = await portal.get_invite_link(evt.sender)
-        return await evt.reply(f"Invite link to {portal.title}: {link}")
-    except ValueError as e:
-        return await evt.reply(e.args[0])
-    except ChatAdminRequiredError:
-        return await evt.reply("You don't have the permission to create an invite link.")
-
-
-async def user_has_power_level(room: str, intent, sender: u.User, event: str, default: int = 50
-                               ) -> bool:
-    if sender.is_admin:
-        return True
-    # Make sure the state store contains the power levels.
-    try:
-        await intent.get_power_levels(room)
-    except MatrixRequestError:
-        return False
-    return intent.state_store.has_power_level(room, sender.mxid,
-                                              event=f"net.maunium.telegram.{event}",
-                                              default=default)
-
-
-async def _get_portal_and_check_permission(evt: CommandEvent, permission: str,
-                                           action: Optional[str] = None
-                                           ) -> Optional[po.Portal]:
-    room_id = MatrixRoomID(evt.args[0]) if len(evt.args) > 0 else evt.room_id
-
-    portal = po.Portal.get_by_mxid(room_id)
-    if not portal:
-        that_this = "This" if room_id == evt.room_id else "That"
-        await evt.reply(f"{that_this} is not a portal room.")
-        return None
-
-    if not await user_has_power_level(portal.mxid, evt.az.intent, evt.sender, permission):
-        action = action or f"{permission.replace('_', ' ')}s"
-        await evt.reply(f"You do not have the permissions to {action} that portal.")
-        return None
-    return portal
-
-
-def _get_portal_murder_function(action: str, room_id: str, function: Callable, command: str,
-                                completed_message: str) -> Dict:
-    async def post_confirm(confirm) -> Optional[Dict]:
-        confirm.sender.command_status = None
-        if len(confirm.args) > 0 and confirm.args[0] == f"confirm-{command}":
-            await function()
-            if confirm.room_id != room_id:
-                return await confirm.reply(completed_message)
-        else:
-            return await confirm.reply(f"{action} cancelled.")
-        return None
-
-    return {
-        "next": post_confirm,
-        "action": action,
-    }
-
-
-@command_handler(needs_auth=False, needs_puppeting=False,
-                 help_section=SECTION_PORTAL_MANAGEMENT,
-                 help_text="Remove all users from the current portal room and forget the portal. "
-                           "Only works for group chats; to delete a private chat portal, simply "
-                           "leave the room.")
-async def delete_portal(evt: CommandEvent) -> Optional[Dict]:
-    portal = await _get_portal_and_check_permission(evt, "unbridge")
-    if not portal:
-        return None
-
-    evt.sender.command_status = _get_portal_murder_function("Portal deletion", portal.mxid,
-                                                            portal.cleanup_and_delete, "delete",
-                                                            "Portal successfully deleted.")
-    return await evt.reply("Please confirm deletion of portal "
-                           f"[{portal.alias or portal.mxid}](https://matrix.to/#/{portal.mxid}) "
-                           f"to Telegram chat \"{portal.title}\" "
-                           "by typing `$cmdprefix+sp confirm-delete`"
-                           "\n\n"
-                           "**WARNING:** If the bridge bot has the power level to do so, **this "
-                           "will kick ALL users** in the room. If you just want to remove the "
-                           "bridge, use `$cmdprefix+sp unbridge` instead.")
-
-
-@command_handler(needs_auth=False, needs_puppeting=False,
-                 help_section=SECTION_PORTAL_MANAGEMENT,
-                 help_text="Remove puppets from the current portal room and forget the portal.")
-async def unbridge(evt: CommandEvent) -> Optional[Dict]:
-    portal = await _get_portal_and_check_permission(evt, "unbridge")
-    if not portal:
-        return None
-
-    evt.sender.command_status = _get_portal_murder_function("Room unbridging", portal.mxid,
-                                                            portal.unbridge, "unbridge",
-                                                            "Room successfully unbridged.")
-    return await evt.reply(f"Please confirm unbridging chat \"{portal.title}\" from room "
-                           f"[{portal.alias or portal.mxid}](https://matrix.to/#/{portal.mxid}) "
-                           "by typing `$cmdprefix+sp confirm-unbridge`")
-
-
-@command_handler(needs_auth=False, needs_puppeting=False,
-                 help_section=SECTION_PORTAL_MANAGEMENT,
-                 help_args="[_id_]",
-                 help_text="Bridge the current Matrix room to the Telegram chat with the given "
-                           "ID. The ID must be the prefixed version that you get with the `/id` "
-                           "command of the Telegram-side bot.")
-async def bridge(evt: CommandEvent) -> Dict:
-    if len(evt.args) == 0:
-        return await evt.reply("**Usage:** "
-                               "`$cmdprefix+sp bridge <Telegram chat ID> [Matrix room ID]`")
-    room_id = MatrixRoomID(evt.args[1]) if len(evt.args) > 1 else evt.room_id
-    that_this = "This" if room_id == evt.room_id else "That"
-
-    portal = po.Portal.get_by_mxid(room_id)
-    if portal:
-        return await evt.reply(f"{that_this} room is already a portal room.")
-
-    if not await user_has_power_level(room_id, evt.az.intent, evt.sender, "bridge"):
-        return await evt.reply(f"You do not have the permissions to bridge {that_this} room.")
-
-    # The /id bot command provides the prefixed ID, so we assume
-    tgid_str = evt.args[0]
-    if tgid_str.startswith("-100"):
-        tgid = TelegramID(int(tgid_str[4:]))
-        peer_type = "channel"
-    elif tgid_str.startswith("-"):
-        tgid = TelegramID(-int(tgid_str))
-        peer_type = "chat"
-    else:
-        return await evt.reply("That doesn't seem like a prefixed Telegram chat ID.\n\n"
-                               "If you did not get the ID using the `/id` bot command, please "
-                               "prefix channel IDs with `-100` and normal group IDs with `-`.\n\n"
-                               "Bridging private chats to existing rooms is not allowed.")
-
-    portal = po.Portal.get_by_tgid(tgid, peer_type=peer_type)
-    if not portal.allow_bridging():
-        return await evt.reply("This bridge doesn't allow bridging that Telegram chat.\n"
-                               "If you're the bridge admin, try "
-                               "`$cmdprefix+sp filter whitelist <Telegram chat ID>` first.")
-    if portal.mxid:
-        has_portal_message = (
-            "That Telegram chat already has a portal at "
-            f"[{portal.alias or portal.mxid}](https://matrix.to/#/{portal.mxid}). ")
-        if not await user_has_power_level(portal.mxid, evt.az.intent, evt.sender, "unbridge"):
-            return await evt.reply(f"{has_portal_message}"
-                                   "Additionally, you do not have the permissions to unbridge "
-                                   "that room.")
-        evt.sender.command_status = {
-            "next": confirm_bridge,
-            "action": "Room bridging",
-            "mxid": portal.mxid,
-            "bridge_to_mxid": room_id,
-            "tgid": portal.tgid,
-            "peer_type": portal.peer_type,
-        }
-        return await evt.reply(f"{has_portal_message}"
-                               "However, you have the permissions to unbridge that room.\n\n"
-                               "To delete that portal completely and continue bridging, use "
-                               "`$cmdprefix+sp delete-and-continue`. To unbridge the portal "
-                               "without kicking Matrix users, use `$cmdprefix+sp unbridge-and-"
-                               "continue`. To cancel, use `$cmdprefix+sp cancel`")
-    evt.sender.command_status = {
-        "next": confirm_bridge,
-        "action": "Room bridging",
-        "bridge_to_mxid": room_id,
-        "tgid": portal.tgid,
-        "peer_type": portal.peer_type,
-    }
-    return await evt.reply("That Telegram chat has no existing portal. To confirm bridging the "
-                           "chat to this room, use `$cmdprefix+sp continue`")
-
-
-async def cleanup_old_portal_while_bridging(evt: CommandEvent, portal: "po.Portal"
-                                            ) -> Tuple[bool, Optional[Coroutine[None, None, None]]]:
-    if not portal.mxid:
-        await evt.reply("The portal seems to have lost its Matrix room between you"
-                        "calling `$cmdprefix+sp bridge` and this command.\n\n"
-                        "Continuing without touching previous Matrix room...")
-        return True, None
-    elif evt.args[0] == "delete-and-continue":
-        return True, portal.cleanup_room(portal.main_intent, portal.mxid,
-                                         message="Portal deleted (moving to another room)")
-    elif evt.args[0] == "unbridge-and-continue":
-        return True, portal.cleanup_room(portal.main_intent, portal.mxid,
-                                         message="Room unbridged (portal moving to another room)",
-                                         puppets_only=True)
-    else:
-        await evt.reply(
-            "The chat you were trying to bridge already has a Matrix portal room.\n\n"
-            "Please use `$cmdprefix+sp delete-and-continue` or `$cmdprefix+sp unbridge-and-"
-            "continue` to either delete or unbridge the existing room (respectively) and "
-            "continue with the bridging.\n\n"
-            "If you changed your mind, use `$cmdprefix+sp cancel` to cancel.")
-        return False, None
-
-
-async def confirm_bridge(evt: CommandEvent) -> Optional[Dict]:
-    status = evt.sender.command_status
-    try:
-        portal = po.Portal.get_by_tgid(status["tgid"], peer_type=status["peer_type"])
-        bridge_to_mxid = status["bridge_to_mxid"]
-    except KeyError:
-        evt.sender.command_status = None
-        return await evt.reply("Fatal error: tgid or peer_type missing from command_status. "
-                               "This shouldn't happen unless you're messing with the command "
-                               "handler code.")
-    if "mxid" in status:
-        ok, coro = await cleanup_old_portal_while_bridging(evt, portal)
-        if not ok:
-            return None
-        elif coro:
-            asyncio.ensure_future(coro, loop=evt.loop)
-            await evt.reply("Cleaning up previous portal room...")
-    elif portal.mxid:
-        evt.sender.command_status = None
-        return await evt.reply("The portal seems to have created a Matrix room between you "
-                               "calling `$cmdprefix+sp bridge` and this command.\n\n"
-                               "Please start over by calling the bridge command again.")
-    elif evt.args[0] != "continue":
-        return await evt.reply("Please use `$cmdprefix+sp continue` to confirm the bridging or "
-                               "`$cmdprefix+sp cancel` to cancel.")
-
-    evt.sender.command_status = None
-    is_logged_in = await evt.sender.is_logged_in()
-    user = evt.sender if is_logged_in else evt.tgbot
-    try:
-        entity = await user.client.get_entity(portal.peer)
-    except Exception:
-        evt.log.exception("Failed to get_entity(%s) for manual bridging.", portal.peer)
-        if is_logged_in:
-            return await evt.reply("Failed to get info of telegram chat. "
-                                   "You are logged in, are you in that chat?")
-        else:
-            return await evt.reply("Failed to get info of telegram chat. "
-                                   "You're not logged in, is the relay bot in the chat?")
-    if isinstance(entity, (ChatForbidden, ChannelForbidden)):
-        if is_logged_in:
-            return await evt.reply("You don't seem to be in that chat.")
-        else:
-            return await evt.reply("The bot doesn't seem to be in that chat.")
-
-    direct = False
-
-    portal.mxid = bridge_to_mxid
-    portal.title, portal.about, levels = await get_initial_state(evt.az.intent, evt.room_id)
-    portal.photo_id = ""
-    portal.save()
-
-    asyncio.ensure_future(portal.update_matrix_room(user, entity, direct, levels=levels),
-                          loop=evt.loop)
-
-    return await evt.reply("Bridging complete. Portal synchronization should begin momentarily.")
-
-
-async def get_initial_state(intent: IntentAPI, room_id: str) -> Tuple[str, str, Dict]:
-    state = await intent.get_room_state(room_id)
-    title = None
-    about = None
-    levels = None
-    for event in state:
-        try:
-            if event["type"] == "m.room.name":
-                title = event["content"]["name"]
-            elif event["type"] == "m.room.topic":
-                about = event["content"]["topic"]
-            elif event["type"] == "m.room.power_levels":
-                levels = event["content"]
-            elif event["type"] == "m.room.canonical_alias":
-                title = title or event["content"]["alias"]
-        except KeyError:
-            # Some state event probably has empty content
-            pass
-    return title, about, levels
-
-
-@command_handler(help_section=SECTION_CREATING_PORTALS,
-                 help_args="[_type_]",
-                 help_text="Create a Telegram chat of the given type for the current Matrix room. "
-                           "The type is either `group`, `supergroup` or `channel` (defaults to "
-                           "`group`).")
-async def create(evt: CommandEvent) -> Dict:
-    type = evt.args[0] if len(evt.args) > 0 else "group"
-    if type not in {"chat", "group", "supergroup", "channel"}:
-        return await evt.reply(
-            "**Usage:** `$cmdprefix+sp create ['group'/'supergroup'/'channel']`")
-
-    if po.Portal.get_by_mxid(evt.room_id):
-        return await evt.reply("This is already a portal room.")
-
-    if not await user_has_power_level(evt.room_id, evt.az.intent, evt.sender, "bridge"):
-        return await evt.reply("You do not have the permissions to bridge this room.")
-
-    title, about, levels = await get_initial_state(evt.az.intent, evt.room_id)
-    if not title:
-        return await evt.reply("Please set a title before creating a Telegram chat.")
-
-    supergroup = type == "supergroup"
-    type = {
-        "supergroup": "channel",
-        "channel": "channel",
-        "chat": "chat",
-        "group": "chat",
-    }[type]
-
-    portal = po.Portal(tgid=None, mxid=evt.room_id, title=title, about=about, peer_type=type)
-    try:
-        await portal.create_telegram_chat(evt.sender, supergroup=supergroup)
-    except ValueError as e:
-        portal.delete()
-        return await evt.reply(e.args[0])
-    return await evt.reply(f"Telegram chat created. ID: {portal.tgid}")
-
-
-@command_handler(help_section=SECTION_PORTAL_MANAGEMENT,
-                 help_text="Upgrade a normal Telegram group to a supergroup.")
-async def upgrade(evt: CommandEvent) -> Dict:
-    portal = po.Portal.get_by_mxid(evt.room_id)
-    if not portal:
-        return await evt.reply("This is not a portal room.")
-    elif portal.peer_type == "channel":
-        return await evt.reply("This is already a supergroup or a channel.")
-    elif portal.peer_type == "user":
-        return await evt.reply("You can't upgrade private chats.")
-
-    try:
-        await portal.upgrade_telegram_chat(evt.sender)
-        return await evt.reply(f"Group upgraded to supergroup. New ID: {portal.tgid}")
-    except ChatAdminRequiredError:
-        return await evt.reply("You don't have the permission to upgrade this group.")
-    except ValueError as e:
-        return await evt.reply(e.args[0])
-
-
-@command_handler(help_section=SECTION_PORTAL_MANAGEMENT,
-                 help_text="View or change per-portal settings.",
-                 help_args="<`help`|_subcommand_> [...]")
-async def config(evt: CommandEvent) -> None:
-    cmd = evt.args[0].lower() if len(evt.args) > 0 else "help"
-    if cmd not in ("view", "defaults", "set", "unset", "add", "del"):
-        await config_help(evt)
-        return
-    elif cmd == "defaults":
-        await config_defaults(evt)
-        return
-
-    portal = po.Portal.get_by_mxid(evt.room_id)
-    if not portal:
-        await evt.reply("This is not a portal room.")
-        return
-    elif cmd == "view":
-        await config_view(evt, portal)
-        return
-
-    key = evt.args[1] if len(evt.args) > 1 else None
-    value = yaml.load(" ".join(evt.args[2:])) if len(evt.args) > 2 else None
-    if cmd == "set":
-        await config_set(evt, portal, key, value)
-    elif cmd == "unset":
-        await config_unset(evt, portal, key)
-    elif cmd == "add" or cmd == "del":
-        await config_add_del(evt, portal, key, value, cmd)
-    else:
-        return
-    portal.save()
-
-
-def config_help(evt: CommandEvent) -> Awaitable[Dict]:
-    return evt.reply("""**Usage:** `$cmdprefix config <subcommand> [...]`. Subcommands:
-
-* **help** - View this help text.
-* **view** - View the current config data.
-* **defaults** - View the default config values.
-* **set** <_key_> <_value_> - Set a config value.
-* **unset** <_key_> - Remove a config value.
-* **add** <_key_> <_value_> - Add a value to an array.
-* **del** <_key_> <_value_> - Remove a value from an array.
-""")
-
-
-def config_view(evt: CommandEvent, portal: po.Portal) -> Awaitable[Dict]:
-    stream = StringIO()
-    yaml.dump(portal.local_config, stream)
-    return evt.reply(f"Room-specific config:\n\n```yaml\n{stream.getvalue()}```")
-
-
-def config_defaults(evt: CommandEvent) -> Awaitable[Dict]:
-    stream = StringIO()
-    yaml.dump({
-        "edits_as_replies": evt.config["bridge.edits_as_replies"],
-        "bridge_notices": {
-            "default": evt.config["bridge.bridge_notices.default"],
-            "exceptions": evt.config["bridge.bridge_notices.exceptions"],
-        },
-        "bot_messages_as_notices": evt.config["bridge.bot_messages_as_notices"],
-        "inline_images": evt.config["bridge.inline_images"],
-        "native_stickers": evt.config["bridge.native_stickers"],
-        "message_formats": evt.config["bridge.message_formats"],
-        "state_event_formats": evt.config["bridge.state_event_formats"],
-    }, stream)
-    return evt.reply(f"Bridge instance wide config:\n\n```yaml\n{stream.getvalue()}```")
-
-
-def config_set(evt: CommandEvent, portal: po.Portal, key: str, value: str) -> Awaitable[Dict]:
-    if not key or value is None:
-        return evt.reply(f"**Usage:** `$cmdprefix+sp config set <key> <value>`")
-    elif util.recursive_set(portal.local_config, key, value):
-        return evt.reply(f"Successfully set the value of `{key}` to `{value}`.")
-    else:
-        return evt.reply(f"Failed to set value of `{key}`. "
-                         "Does the path contain non-map types?")
-
-
-def config_unset(evt: CommandEvent, portal: po.Portal, key: str) -> Awaitable[Dict]:
-    if not key:
-        return evt.reply(f"**Usage:** `$cmdprefix+sp config unset <key>`")
-    elif util.recursive_del(portal.local_config, key):
-        return evt.reply(f"Successfully deleted `{key}` from config.")
-    else:
-        return evt.reply(f"`{key}` not found in config.")
-
-
-def config_add_del(evt: CommandEvent, portal: po.Portal, key: str, value: str, cmd: str
-                   ) -> Awaitable[Dict]:
-    if not key or value is None:
-        return evt.reply(f"**Usage:** `$cmdprefix+sp config {cmd} <key> <value>`")
-
-    arr = util.recursive_get(portal.local_config, key)
-    if not arr:
-        return evt.reply(f"`{key}` not found in config. "
-                         f"Maybe do `$cmdprefix+sp config set {key} []` first?")
-    elif not isinstance(arr, list):
-        return evt.reply("`{key}` does not seem to be an array.")
-    elif cmd == "add":
-        if value in arr:
-            return evt.reply(f"The array at `{key}` already contains `{value}`.")
-        arr.append(value)
-        return evt.reply(f"Successfully added `{value}` to the array at `{key}`")
-    else:
-        if value not in arr:
-            return evt.reply(f"The array at `{key}` does not contain `{value}`.")
-        arr.remove(value)
-        return evt.reply(f"Successfully removed `{value}` from the array at `{key}`")
-
-
-@command_handler(help_section=SECTION_PORTAL_MANAGEMENT,
-                 help_args="<_name_|`-`>",
-                 help_text="Change the username of a supergroup/channel. "
-                           "To disable, use a dash (`-`) as the name.")
-async def group_name(evt: CommandEvent) -> Dict:
-    if len(evt.args) == 0:
-        return await evt.reply("**Usage:** `$cmdprefix+sp group-name <name/->`")
-
-    portal = po.Portal.get_by_mxid(evt.room_id)
-    if not portal:
-        return await evt.reply("This is not a portal room.")
-    elif portal.peer_type != "channel":
-        return await evt.reply("Only channels and supergroups have usernames.")
-
-    try:
-        await portal.set_telegram_username(evt.sender,
-                                           evt.args[0] if evt.args[0] != "-" else "")
-        if portal.username:
-            return await evt.reply(f"Username of channel changed to {portal.username}.")
-        else:
-            return await evt.reply(f"Channel is now private.")
-    except ChatAdminRequiredError:
-        return await evt.reply(
-            "You don't have the permission to set the username of this channel.")
-    except UsernameNotModifiedError:
-        if portal.username:
-            return await evt.reply("That is already the username of this channel.")
-        else:
-            return await evt.reply("This channel is already private")
-    except UsernameOccupiedError:
-        return await evt.reply("That username is already in use.")
-    except UsernameInvalidError:
-        return await evt.reply("Invalid username")
-
-
-@command_handler(needs_admin=True,
-                 help_section=SECTION_ADMIN,
-                 help_args="<`whitelist`|`blacklist`>",
-                 help_text="Change whether the bridge will allow or disallow bridging rooms by "
-                           "default.")
-async def filter_mode(evt: CommandEvent) -> Dict:
-    try:
-        mode = evt.args[0]
-        if mode not in ("whitelist", "blacklist"):
-            raise ValueError()
-    except (IndexError, ValueError):
-        return await evt.reply("**Usage:** `$cmdprefix+sp filter-mode <whitelist/blacklist>`")
-
-    evt.config["bridge.filter.mode"] = mode
-    evt.config.save()
-    po.Portal.filter_mode = mode
-    if mode == "whitelist":
-        return await evt.reply("The bridge will now disallow bridging chats by default.\n"
-                               "To allow bridging a specific chat, use"
-                               "`!filter whitelist <chat ID>`.")
-    else:
-        return await evt.reply("The bridge will now allow bridging chats by default.\n"
-                               "To disallow bridging a specific chat, use"
-                               "`!filter blacklist <chat ID>`.")
-
-
-@command_handler(needs_admin=True,
-                 help_section=SECTION_ADMIN,
-                 help_args="<`whitelist`|`blacklist`> <_chat ID_>",
-                 help_text="Allow or disallow bridging a specific chat.")
-async def filter(evt: CommandEvent) -> Optional[Dict]:
-    try:
-        action = evt.args[0]
-        if action not in ("whitelist", "blacklist", "add", "remove"):
-            raise ValueError()
-
-        id_str = evt.args[1]
-        if id_str.startswith("-100"):
-            id = int(id_str[4:])
-        elif id_str.startswith("-"):
-            id = int(id_str[1:])
-        else:
-            id = int(id_str)
-    except (IndexError, ValueError):
-        return await evt.reply("**Usage:** `$cmdprefix+sp filter <whitelist/blacklist> <chat ID>`")
-
-    mode = evt.config["bridge.filter.mode"]
-    if mode not in ("blacklist", "whitelist"):
-        return await evt.reply(f"Unknown filter mode \"{mode}\". Please fix the bridge config.")
-
-    list = evt.config["bridge.filter.list"]
-
-    if action in ("blacklist", "whitelist"):
-        action = "add" if mode == action else "remove"
-
-    def save() -> None:
-        evt.config["bridge.filter.list"] = list
-        evt.config.save()
-        po.Portal.filter_list = list
-
-    if action == "add":
-        if id in list:
-            return await evt.reply(f"That chat is already {mode}ed.")
-        list.append(id)
-        save()
-        return await evt.reply(f"Chat ID added to {mode}.")
-    elif action == "remove":
-        if id not in list:
-            return await evt.reply(f"That chat is not {mode}ed.")
-        list.remove(id)
-        save()
-        return await evt.reply(f"Chat ID removed from {mode}.")
-    return None

mautrix_telegram/commands/portal/__init__.py

@@ -0,0 +1 @@
+from . import admin, bridge, config, create_chat, filter, misc, unbridge

mautrix_telegram/commands/portal/admin.py

@@ -0,0 +1,101 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Dict
+import asyncio
+
+from mautrix_appservice import MatrixRequestError
+
+from ... import portal as po, puppet as pu, user as u
+from .. import command_handler, CommandEvent, SECTION_ADMIN
+
+
+@command_handler(needs_admin=True, needs_auth=False, name="set-pl",
+                 help_section=SECTION_ADMIN,
+                 help_args="<_level_> [_mxid_]",
+                 help_text="Set a temporary power level without affecting Telegram.")
+async def set_power_level(evt: CommandEvent) -> Dict:
+    try:
+        level = int(evt.args[0])
+    except KeyError:
+        return await evt.reply("**Usage:** `$cmdprefix+sp set-pl <level> [mxid]`")
+    except ValueError:
+        return await evt.reply("The level must be an integer.")
+    levels = await evt.az.intent.get_power_levels(evt.room_id)
+    mxid = evt.args[1] if len(evt.args) > 1 else evt.sender.mxid
+    levels["users"][mxid] = level
+    try:
+        await evt.az.intent.set_power_levels(evt.room_id, levels)
+    except MatrixRequestError:
+        evt.log.exception("Failed to set power level.")
+        return await evt.reply("Failed to set power level.")
+    return {}
+
+
+@command_handler(needs_admin=True, needs_auth=False,
+                 help_section=SECTION_ADMIN,
+                 help_args="<`portal`|`puppet`|`user`>",
+                 help_text="Clear internal bridge caches")
+async def clear_db_cache(evt: CommandEvent) -> Dict:
+    try:
+        section = evt.args[0].lower()
+    except IndexError:
+        return await evt.reply("**Usage:** `$cmdprefix+sp clear-db-cache <section>`")
+    if section == "portal":
+        po.Portal.by_tgid = {}
+        po.Portal.by_mxid = {}
+        await evt.reply("Cleared portal cache")
+    elif section == "puppet":
+        pu.Puppet.cache = {}
+        for puppet in pu.Puppet.by_custom_mxid.values():
+            puppet.sync_task.cancel()
+        pu.Puppet.by_custom_mxid = {}
+        await asyncio.gather(
+            *[puppet.init_custom_mxid() for puppet in pu.Puppet.all_with_custom_mxid()],
+            loop=evt.loop)
+        await evt.reply("Cleared puppet cache and restarted custom puppet syncers")
+    elif section == "user":
+        u.User.by_mxid = {
+            user.mxid: user
+            for user in u.User.by_tgid.values()
+        }
+        await evt.reply("Cleared non-logged-in user cache")
+    else:
+        return await evt.reply("**Usage:** `$cmdprefix+sp clear-db-cache <section>`")
+
+
+@command_handler(needs_admin=True, needs_auth=False,
+                 help_section=SECTION_ADMIN,
+                 help_args="[_mxid_]",
+                 help_text="Reload and reconnect a user")
+async def reload_user(evt: CommandEvent) -> Dict:
+    if len(evt.args) > 0:
+        mxid = evt.args[0]
+    else:
+        mxid = evt.sender.mxid
+    user = u.User.get_by_mxid(mxid, create=False)
+    if not user:
+        return await evt.reply("User not found")
+    puppet = pu.Puppet.get_by_custom_mxid(mxid)
+    if puppet:
+        puppet.sync_task.cancel()
+    await user.stop()
+    user.delete(delete_db=False)
+    user = u.User.get_by_mxid(mxid)
+    await user.ensure_started()
+    if puppet:
+        await puppet.init_custom_mxid()
+    await evt.reply(f"Reloaded and reconnected {user.mxid} (telegram: {user.human_tg_id})")

mautrix_telegram/commands/portal/bridge.py

@@ -0,0 +1,187 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Dict, Optional, Tuple, Coroutine
+import asyncio
+
+from telethon.tl.types import ChatForbidden, ChannelForbidden
+
+from ...types import MatrixRoomID, TelegramID
+from ...util import ignore_coro
+from ... import portal as po
+from .. import command_handler, CommandEvent, SECTION_CREATING_PORTALS
+from .util import user_has_power_level, get_initial_state
+
+
+@command_handler(needs_auth=False, needs_puppeting=False,
+                 help_section=SECTION_CREATING_PORTALS,
+                 help_args="[_id_]",
+                 help_text="Bridge the current Matrix room to the Telegram chat with the given "
+                           "ID. The ID must be the prefixed version that you get with the `/id` "
+                           "command of the Telegram-side bot.")
+async def bridge(evt: CommandEvent) -> Dict:
+    if len(evt.args) == 0:
+        return await evt.reply("**Usage:** "
+                               "`$cmdprefix+sp bridge <Telegram chat ID> [Matrix room ID]`")
+    force_use_bot = False
+    if evt.args[0] == "--usebot" and evt.sender.is_admin:
+        force_use_bot = True
+        evt.args = evt.args[1:]
+    room_id = MatrixRoomID(evt.args[1]) if len(evt.args) > 1 else evt.room_id
+    that_this = "This" if room_id == evt.room_id else "That"
+
+    portal = po.Portal.get_by_mxid(room_id)
+    if portal:
+        return await evt.reply(f"{that_this} room is already a portal room.")
+
+    if not await user_has_power_level(room_id, evt.az.intent, evt.sender, "bridge"):
+        return await evt.reply(f"You do not have the permissions to bridge {that_this} room.")
+
+    # The /id bot command provides the prefixed ID, so we assume
+    tgid_str = evt.args[0]
+    if tgid_str.startswith("-100"):
+        tgid = TelegramID(int(tgid_str[4:]))
+        peer_type = "channel"
+    elif tgid_str.startswith("-"):
+        tgid = TelegramID(-int(tgid_str))
+        peer_type = "chat"
+    else:
+        return await evt.reply("That doesn't seem like a prefixed Telegram chat ID.\n\n"
+                               "If you did not get the ID using the `/id` bot command, please "
+                               "prefix channel IDs with `-100` and normal group IDs with `-`.\n\n"
+                               "Bridging private chats to existing rooms is not allowed.")
+
+    portal = po.Portal.get_by_tgid(tgid, peer_type=peer_type)
+    if not portal.allow_bridging():
+        return await evt.reply("This bridge doesn't allow bridging that Telegram chat.\n"
+                               "If you're the bridge admin, try "
+                               "`$cmdprefix+sp filter whitelist <Telegram chat ID>` first.")
+    if portal.mxid:
+        has_portal_message = (
+            "That Telegram chat already has a portal at "
+            f"[{portal.alias or portal.mxid}](https://matrix.to/#/{portal.mxid}). ")
+        if not await user_has_power_level(portal.mxid, evt.az.intent, evt.sender, "unbridge"):
+            return await evt.reply(f"{has_portal_message}"
+                                   "Additionally, you do not have the permissions to unbridge "
+                                   "that room.")
+        evt.sender.command_status = {
+            "next": confirm_bridge,
+            "action": "Room bridging",
+            "mxid": portal.mxid,
+            "bridge_to_mxid": room_id,
+            "tgid": portal.tgid,
+            "peer_type": portal.peer_type,
+            "force_use_bot": force_use_bot,
+        }
+        return await evt.reply(f"{has_portal_message}"
+                               "However, you have the permissions to unbridge that room.\n\n"
+                               "To delete that portal completely and continue bridging, use "
+                               "`$cmdprefix+sp delete-and-continue`. To unbridge the portal "
+                               "without kicking Matrix users, use `$cmdprefix+sp unbridge-and-"
+                               "continue`. To cancel, use `$cmdprefix+sp cancel`")
+    evt.sender.command_status = {
+        "next": confirm_bridge,
+        "action": "Room bridging",
+        "bridge_to_mxid": room_id,
+        "tgid": portal.tgid,
+        "peer_type": portal.peer_type,
+        "force_use_bot": force_use_bot,
+    }
+    return await evt.reply("That Telegram chat has no existing portal. To confirm bridging the "
+                           "chat to this room, use `$cmdprefix+sp continue`")
+
+
+async def cleanup_old_portal_while_bridging(evt: CommandEvent, portal: "po.Portal"
+                                            ) -> Tuple[bool, Optional[Coroutine[None, None, None]]]:
+    if not portal.mxid:
+        await evt.reply("The portal seems to have lost its Matrix room between you"
+                        "calling `$cmdprefix+sp bridge` and this command.\n\n"
+                        "Continuing without touching previous Matrix room...")
+        return True, None
+    elif evt.args[0] == "delete-and-continue":
+        return True, portal.cleanup_room(portal.main_intent, portal.mxid,
+                                         message="Portal deleted (moving to another room)")
+    elif evt.args[0] == "unbridge-and-continue":
+        return True, portal.cleanup_room(portal.main_intent, portal.mxid,
+                                         message="Room unbridged (portal moving to another room)",
+                                         puppets_only=True)
+    else:
+        await evt.reply(
+            "The chat you were trying to bridge already has a Matrix portal room.\n\n"
+            "Please use `$cmdprefix+sp delete-and-continue` or `$cmdprefix+sp unbridge-and-"
+            "continue` to either delete or unbridge the existing room (respectively) and "
+            "continue with the bridging.\n\n"
+            "If you changed your mind, use `$cmdprefix+sp cancel` to cancel.")
+        return False, None
+
+
+async def confirm_bridge(evt: CommandEvent) -> Optional[Dict]:
+    status = evt.sender.command_status
+    try:
+        portal = po.Portal.get_by_tgid(status["tgid"], peer_type=status["peer_type"])
+        bridge_to_mxid = status["bridge_to_mxid"]
+    except KeyError:
+        evt.sender.command_status = None
+        return await evt.reply("Fatal error: tgid or peer_type missing from command_status. "
+                               "This shouldn't happen unless you're messing with the command "
+                               "handler code.")
+    if "mxid" in status:
+        ok, coro = await cleanup_old_portal_while_bridging(evt, portal)
+        if not ok:
+            return None
+        elif coro:
+            ignore_coro(asyncio.ensure_future(coro, loop=evt.loop))
+            await evt.reply("Cleaning up previous portal room...")
+    elif portal.mxid:
+        evt.sender.command_status = None
+        return await evt.reply("The portal seems to have created a Matrix room between you "
+                               "calling `$cmdprefix+sp bridge` and this command.\n\n"
+                               "Please start over by calling the bridge command again.")
+    elif evt.args[0] != "continue":
+        return await evt.reply("Please use `$cmdprefix+sp continue` to confirm the bridging or "
+                               "`$cmdprefix+sp cancel` to cancel.")
+
+    evt.sender.command_status = None
+    is_logged_in = await evt.sender.is_logged_in() and not status["force_use_bot"]
+    user = evt.sender if is_logged_in else evt.tgbot
+    try:
+        entity = await user.client.get_entity(portal.peer)
+    except Exception:
+        evt.log.exception("Failed to get_entity(%s) for manual bridging.", portal.peer)
+        if is_logged_in:
+            return await evt.reply("Failed to get info of telegram chat. "
+                                   "You are logged in, are you in that chat?")
+        else:
+            return await evt.reply("Failed to get info of telegram chat. "
+                                   "You're not logged in, is the relay bot in the chat?")
+    if isinstance(entity, (ChatForbidden, ChannelForbidden)):
+        if is_logged_in:
+            return await evt.reply("You don't seem to be in that chat.")
+        else:
+            return await evt.reply("The bot doesn't seem to be in that chat.")
+
+    direct = False
+
+    portal.mxid = bridge_to_mxid
+    portal.title, portal.about, levels = await get_initial_state(evt.az.intent, evt.room_id)
+    portal.photo_id = ""
+    portal.save()
+
+    ignore_coro(asyncio.ensure_future(portal.update_matrix_room(user, entity, direct,
+                                                                levels=levels),
+                                      loop=evt.loop))
+
+    return await evt.reply("Bridging complete. Portal synchronization should begin momentarily.")

mautrix_telegram/commands/portal/config.py

@@ -0,0 +1,132 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Dict, Awaitable
+from io import StringIO
+
+from ...config import yaml
+from ... import portal as po, util
+from .. import command_handler, CommandEvent, SECTION_PORTAL_MANAGEMENT
+
+
+@command_handler(help_section=SECTION_PORTAL_MANAGEMENT,
+                 help_text="View or change per-portal settings.",
+                 help_args="<`help`|_subcommand_> [...]")
+async def config(evt: CommandEvent) -> None:
+    cmd = evt.args[0].lower() if len(evt.args) > 0 else "help"
+    if cmd not in ("view", "defaults", "set", "unset", "add", "del"):
+        await config_help(evt)
+        return
+    elif cmd == "defaults":
+        await config_defaults(evt)
+        return
+
+    portal = po.Portal.get_by_mxid(evt.room_id)
+    if not portal:
+        await evt.reply("This is not a portal room.")
+        return
+    elif cmd == "view":
+        await config_view(evt, portal)
+        return
+
+    key = evt.args[1] if len(evt.args) > 1 else None
+    value = yaml.load(" ".join(evt.args[2:])) if len(evt.args) > 2 else None
+    if cmd == "set":
+        await config_set(evt, portal, key, value)
+    elif cmd == "unset":
+        await config_unset(evt, portal, key)
+    elif cmd == "add" or cmd == "del":
+        await config_add_del(evt, portal, key, value, cmd)
+    else:
+        return
+    portal.save()
+
+
+def config_help(evt: CommandEvent) -> Awaitable[Dict]:
+    return evt.reply("""**Usage:** `$cmdprefix config <subcommand> [...]`. Subcommands:
+
+* **help** - View this help text.
+* **view** - View the current config data.
+* **defaults** - View the default config values.
+* **set** <_key_> <_value_> - Set a config value.
+* **unset** <_key_> - Remove a config value.
+* **add** <_key_> <_value_> - Add a value to an array.
+* **del** <_key_> <_value_> - Remove a value from an array.
+""")
+
+
+def config_view(evt: CommandEvent, portal: po.Portal) -> Awaitable[Dict]:
+    stream = StringIO()
+    yaml.dump(portal.local_config, stream)
+    return evt.reply(f"Room-specific config:\n\n```yaml\n{stream.getvalue()}```")
+
+
+def config_defaults(evt: CommandEvent) -> Awaitable[Dict]:
+    stream = StringIO()
+    yaml.dump({
+        "bridge_notices": {
+            "default": evt.config["bridge.bridge_notices.default"],
+            "exceptions": evt.config["bridge.bridge_notices.exceptions"],
+        },
+        "bot_messages_as_notices": evt.config["bridge.bot_messages_as_notices"],
+        "inline_images": evt.config["bridge.inline_images"],
+        "message_formats": evt.config["bridge.message_formats"],
+        "state_event_formats": evt.config["bridge.state_event_formats"],
+        "telegram_link_preview": evt.config["bridge.telegram_link_preview"],
+    }, stream)
+    return evt.reply(f"Bridge instance wide config:\n\n```yaml\n{stream.getvalue()}```")
+
+
+def config_set(evt: CommandEvent, portal: po.Portal, key: str, value: str) -> Awaitable[Dict]:
+    if not key or value is None:
+        return evt.reply(f"**Usage:** `$cmdprefix+sp config set <key> <value>`")
+    elif util.recursive_set(portal.local_config, key, value):
+        return evt.reply(f"Successfully set the value of `{key}` to `{value}`.")
+    else:
+        return evt.reply(f"Failed to set value of `{key}`. "
+                         "Does the path contain non-map types?")
+
+
+def config_unset(evt: CommandEvent, portal: po.Portal, key: str) -> Awaitable[Dict]:
+    if not key:
+        return evt.reply(f"**Usage:** `$cmdprefix+sp config unset <key>`")
+    elif util.recursive_del(portal.local_config, key):
+        return evt.reply(f"Successfully deleted `{key}` from config.")
+    else:
+        return evt.reply(f"`{key}` not found in config.")
+
+
+def config_add_del(evt: CommandEvent, portal: po.Portal, key: str, value: str, cmd: str
+                   ) -> Awaitable[Dict]:
+    if not key or value is None:
+        return evt.reply(f"**Usage:** `$cmdprefix+sp config {cmd} <key> <value>`")
+
+    arr = util.recursive_get(portal.local_config, key)
+    if not arr:
+        return evt.reply(f"`{key}` not found in config. "
+                         f"Maybe do `$cmdprefix+sp config set {key} []` first?")
+    elif not isinstance(arr, list):
+        return evt.reply("`{key}` does not seem to be an array.")
+    elif cmd == "add":
+        if value in arr:
+            return evt.reply(f"The array at `{key}` already contains `{value}`.")
+        arr.append(value)
+        return evt.reply(f"Successfully added `{value}` to the array at `{key}`")
+    else:
+        if value not in arr:
+            return evt.reply(f"The array at `{key}` does not contain `{value}`.")
+        arr.remove(value)
+        return evt.reply(f"Successfully removed `{value}` from the array at `{key}`")

mautrix_telegram/commands/portal/create_chat.py

@@ -0,0 +1,61 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Dict
+
+from ... import portal as po
+from ...types import TelegramID
+from .. import command_handler, CommandEvent, SECTION_CREATING_PORTALS
+from .util import user_has_power_level, get_initial_state
+
+
+@command_handler(help_section=SECTION_CREATING_PORTALS,
+                 help_args="[_type_]",
+                 help_text="Create a Telegram chat of the given type for the current Matrix room. "
+                           "The type is either `group`, `supergroup` or `channel` (defaults to "
+                           "`group`).")
+async def create(evt: CommandEvent) -> Dict:
+    type = evt.args[0] if len(evt.args) > 0 else "group"
+    if type not in {"chat", "group", "supergroup", "channel"}:
+        return await evt.reply(
+            "**Usage:** `$cmdprefix+sp create ['group'/'supergroup'/'channel']`")
+
+    if po.Portal.get_by_mxid(evt.room_id):
+        return await evt.reply("This is already a portal room.")
+
+    if not await user_has_power_level(evt.room_id, evt.az.intent, evt.sender, "bridge"):
+        return await evt.reply("You do not have the permissions to bridge this room.")
+
+    title, about, levels = await get_initial_state(evt.az.intent, evt.room_id)
+    if not title:
+        return await evt.reply("Please set a title before creating a Telegram chat.")
+
+    supergroup = type == "supergroup"
+    type = {
+        "supergroup": "channel",
+        "channel": "channel",
+        "chat": "chat",
+        "group": "chat",
+    }[type]
+
+    portal = po.Portal(tgid=TelegramID(0), peer_type=type,
+                       mxid=evt.room_id, title=title, about=about)
+    try:
+        await portal.create_telegram_chat(evt.sender, supergroup=supergroup)
+    except ValueError as e:
+        portal.delete()
+        return await evt.reply(e.args[0])
+    return await evt.reply(f"Telegram chat created. ID: {portal.tgid}")

mautrix_telegram/commands/portal/filter.py

@@ -0,0 +1,95 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Dict, Optional
+
+from ... import portal as po
+from .. import command_handler, CommandEvent, SECTION_ADMIN
+
+
+@command_handler(needs_admin=True,
+                 help_section=SECTION_ADMIN,
+                 help_args="<`whitelist`|`blacklist`>",
+                 help_text="Change whether the bridge will allow or disallow bridging rooms by "
+                           "default.")
+async def filter_mode(evt: CommandEvent) -> Dict:
+    try:
+        mode = evt.args[0]
+        if mode not in ("whitelist", "blacklist"):
+            raise ValueError()
+    except (IndexError, ValueError):
+        return await evt.reply("**Usage:** `$cmdprefix+sp filter-mode <whitelist/blacklist>`")
+
+    evt.config["bridge.filter.mode"] = mode
+    evt.config.save()
+    po.Portal.filter_mode = mode
+    if mode == "whitelist":
+        return await evt.reply("The bridge will now disallow bridging chats by default.\n"
+                               "To allow bridging a specific chat, use"
+                               "`!filter whitelist <chat ID>`.")
+    else:
+        return await evt.reply("The bridge will now allow bridging chats by default.\n"
+                               "To disallow bridging a specific chat, use"
+                               "`!filter blacklist <chat ID>`.")
+
+
+@command_handler(name="filter", needs_admin=True,
+                 help_section=SECTION_ADMIN,
+                 help_args="<`whitelist`|`blacklist`> <_chat ID_>",
+                 help_text="Allow or disallow bridging a specific chat.")
+async def edit_filter(evt: CommandEvent) -> Optional[Dict]:
+    try:
+        action = evt.args[0]
+        if action not in ("whitelist", "blacklist", "add", "remove"):
+            raise ValueError()
+
+        id_str = evt.args[1]
+        if id_str.startswith("-100"):
+            filter_id = int(id_str[4:])
+        elif id_str.startswith("-"):
+            filter_id = int(id_str[1:])
+        else:
+            filter_id = int(id_str)
+    except (IndexError, ValueError):
+        return await evt.reply("**Usage:** `$cmdprefix+sp filter <whitelist/blacklist> <chat ID>`")
+
+    mode = evt.config["bridge.filter.mode"]
+    if mode not in ("blacklist", "whitelist"):
+        return await evt.reply(f"Unknown filter mode \"{mode}\". Please fix the bridge config.")
+
+    filter_id_list = evt.config["bridge.filter.list"]
+
+    if action in ("blacklist", "whitelist"):
+        action = "add" if mode == action else "remove"
+
+    def save() -> None:
+        evt.config["bridge.filter.list"] = filter_id_list
+        evt.config.save()
+        po.Portal.filter_list = filter_id_list
+
+    if action == "add":
+        if filter_id in filter_id_list:
+            return await evt.reply(f"That chat is already {mode}ed.")
+        filter_id_list.append(filter_id)
+        save()
+        return await evt.reply(f"Chat ID added to {mode}.")
+    elif action == "remove":
+        if filter_id not in filter_id_list:
+            return await evt.reply(f"That chat is not {mode}ed.")
+        filter_id_list.remove(filter_id)
+        save()
+        return await evt.reply(f"Chat ID removed from {mode}.")
+    return None

mautrix_telegram/commands/portal/misc.py

@@ -0,0 +1,127 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Dict
+
+from telethon.errors import (ChatAdminRequiredError, UsernameInvalidError,
+                             UsernameNotModifiedError, UsernameOccupiedError)
+
+from ... import portal as po
+from .. import command_handler, CommandEvent, SECTION_PORTAL_MANAGEMENT, SECTION_MISC
+from .util import user_has_power_level
+
+
+@command_handler(needs_admin=False, needs_puppeting=False, needs_auth=False,
+                 help_section=SECTION_MISC,
+                 help_text="Fetch Matrix room state to ensure the bridge has up-to-date info.")
+async def sync_state(evt: CommandEvent) -> Dict:
+    portal = po.Portal.get_by_mxid(evt.room_id)
+    if not portal:
+        return await evt.reply("This is not a portal room.")
+    elif not await user_has_power_level(evt.room_id, evt.az.intent, evt.sender, "bridge"):
+        return await evt.reply(f"You do not have the permissions to synchronize this room.")
+
+    await portal.sync_matrix_members()
+    await evt.reply("Synchronization complete")
+
+
+@command_handler(name="id", needs_admin=False, needs_puppeting=False, needs_auth=False,
+                 help_section=SECTION_MISC,
+                 help_text="Get the ID of the Telegram chat where this room is bridged.")
+async def get_id(evt: CommandEvent) -> Dict:
+    portal = po.Portal.get_by_mxid(evt.room_id)
+    if not portal:
+        return await evt.reply("This is not a portal room.")
+    tgid = portal.tgid
+    if portal.peer_type == "chat":
+        tgid = -tgid
+    elif portal.peer_type == "channel":
+        tgid = f"-100{tgid}"
+    await evt.reply(f"This room is bridged to Telegram chat ID `{tgid}`.")
+
+
+@command_handler(help_section=SECTION_PORTAL_MANAGEMENT,
+                 help_text="Get a Telegram invite link to the current chat.")
+async def invite_link(evt: CommandEvent) -> Dict:
+    portal = po.Portal.get_by_mxid(evt.room_id)
+    if not portal:
+        return await evt.reply("This is not a portal room.")
+
+    if portal.peer_type == "user":
+        return await evt.reply("You can't invite users to private chats.")
+
+    try:
+        link = await portal.get_invite_link(evt.sender)
+        return await evt.reply(f"Invite link to {portal.title}: {link}")
+    except ValueError as e:
+        return await evt.reply(e.args[0])
+    except ChatAdminRequiredError:
+        return await evt.reply("You don't have the permission to create an invite link.")
+
+
+@command_handler(help_section=SECTION_PORTAL_MANAGEMENT,
+                 help_text="Upgrade a normal Telegram group to a supergroup.")
+async def upgrade(evt: CommandEvent) -> Dict:
+    portal = po.Portal.get_by_mxid(evt.room_id)
+    if not portal:
+        return await evt.reply("This is not a portal room.")
+    elif portal.peer_type == "channel":
+        return await evt.reply("This is already a supergroup or a channel.")
+    elif portal.peer_type == "user":
+        return await evt.reply("You can't upgrade private chats.")
+
+    try:
+        await portal.upgrade_telegram_chat(evt.sender)
+        return await evt.reply(f"Group upgraded to supergroup. New ID: -100{portal.tgid}")
+    except ChatAdminRequiredError:
+        return await evt.reply("You don't have the permission to upgrade this group.")
+    except ValueError as e:
+        return await evt.reply(e.args[0])
+
+
+@command_handler(help_section=SECTION_PORTAL_MANAGEMENT,
+                 help_args="<_name_|`-`>",
+                 help_text="Change the username of a supergroup/channel. "
+                           "To disable, use a dash (`-`) as the name.")
+async def group_name(evt: CommandEvent) -> Dict:
+    if len(evt.args) == 0:
+        return await evt.reply("**Usage:** `$cmdprefix+sp group-name <name/->`")
+
+    portal = po.Portal.get_by_mxid(evt.room_id)
+    if not portal:
+        return await evt.reply("This is not a portal room.")
+    elif portal.peer_type != "channel":
+        return await evt.reply("Only channels and supergroups have usernames.")
+
+    try:
+        await portal.set_telegram_username(evt.sender,
+                                           evt.args[0] if evt.args[0] != "-" else "")
+        if portal.username:
+            return await evt.reply(f"Username of channel changed to {portal.username}.")
+        else:
+            return await evt.reply(f"Channel is now private.")
+    except ChatAdminRequiredError:
+        return await evt.reply(
+            "You don't have the permission to set the username of this channel.")
+    except UsernameNotModifiedError:
+        if portal.username:
+            return await evt.reply("That is already the username of this channel.")
+        else:
+            return await evt.reply("This channel is already private")
+    except UsernameOccupiedError:
+        return await evt.reply("That username is already in use.")
+    except UsernameInvalidError:
+        return await evt.reply("Invalid username")

mautrix_telegram/commands/portal/unbridge.py

@@ -0,0 +1,97 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Dict, Callable, Optional
+
+from ...types import MatrixRoomID
+from ... import portal as po
+from .. import command_handler, CommandEvent, SECTION_PORTAL_MANAGEMENT
+from .util import user_has_power_level
+
+
+async def _get_portal_and_check_permission(evt: CommandEvent, permission: str,
+                                           action: Optional[str] = None
+                                           ) -> Optional[po.Portal]:
+    room_id = MatrixRoomID(evt.args[0]) if len(evt.args) > 0 else evt.room_id
+
+    portal = po.Portal.get_by_mxid(room_id)
+    if not portal:
+        that_this = "This" if room_id == evt.room_id else "That"
+        await evt.reply(f"{that_this} is not a portal room.")
+        return None
+
+    if not await user_has_power_level(portal.mxid, evt.az.intent, evt.sender, permission):
+        action = action or f"{permission.replace('_', ' ')}s"
+        await evt.reply(f"You do not have the permissions to {action} that portal.")
+        return None
+    return portal
+
+
+def _get_portal_murder_function(action: str, room_id: str, function: Callable, command: str,
+                                completed_message: str) -> Dict:
+    async def post_confirm(confirm) -> Optional[Dict]:
+        confirm.sender.command_status = None
+        if len(confirm.args) > 0 and confirm.args[0] == f"confirm-{command}":
+            await function()
+            if confirm.room_id != room_id:
+                return await confirm.reply(completed_message)
+        else:
+            return await confirm.reply(f"{action} cancelled.")
+        return None
+
+    return {
+        "next": post_confirm,
+        "action": action,
+    }
+
+
+@command_handler(needs_auth=False, needs_puppeting=False,
+                 help_section=SECTION_PORTAL_MANAGEMENT,
+                 help_text="Remove all users from the current portal room and forget the portal. "
+                           "Only works for group chats; to delete a private chat portal, simply "
+                           "leave the room.")
+async def delete_portal(evt: CommandEvent) -> Optional[Dict]:
+    portal = await _get_portal_and_check_permission(evt, "unbridge")
+    if not portal:
+        return None
+
+    evt.sender.command_status = _get_portal_murder_function("Portal deletion", portal.mxid,
+                                                            portal.cleanup_and_delete, "delete",
+                                                            "Portal successfully deleted.")
+    return await evt.reply("Please confirm deletion of portal "
+                           f"[{portal.alias or portal.mxid}](https://matrix.to/#/{portal.mxid}) "
+                           f"to Telegram chat \"{portal.title}\" "
+                           "by typing `$cmdprefix+sp confirm-delete`"
+                           "\n\n"
+                           "**WARNING:** If the bridge bot has the power level to do so, **this "
+                           "will kick ALL users** in the room. If you just want to remove the "
+                           "bridge, use `$cmdprefix+sp unbridge` instead.")
+
+
+@command_handler(needs_auth=False, needs_puppeting=False,
+                 help_section=SECTION_PORTAL_MANAGEMENT,
+                 help_text="Remove puppets from the current portal room and forget the portal.")
+async def unbridge(evt: CommandEvent) -> Optional[Dict]:
+    portal = await _get_portal_and_check_permission(evt, "unbridge")
+    if not portal:
+        return None
+
+    evt.sender.command_status = _get_portal_murder_function("Room unbridging", portal.mxid,
+                                                            portal.unbridge, "unbridge",
+                                                            "Room successfully unbridged.")
+    return await evt.reply(f"Please confirm unbridging chat \"{portal.title}\" from room "
+                           f"[{portal.alias or portal.mxid}](https://matrix.to/#/{portal.mxid}) "
+                           "by typing `$cmdprefix+sp confirm-unbridge`")

mautrix_telegram/commands/portal/util.py

@@ -0,0 +1,56 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Dict, Tuple
+
+from mautrix_appservice import MatrixRequestError, IntentAPI
+
+from ... import user as u
+
+
+async def get_initial_state(intent: IntentAPI, room_id: str) -> Tuple[str, str, Dict]:
+    state = await intent.get_room_state(room_id)
+    title = None
+    about = None
+    levels = None
+    for event in state:
+        try:
+            if event["type"] == "m.room.name":
+                title = event["content"]["name"]
+            elif event["type"] == "m.room.topic":
+                about = event["content"]["topic"]
+            elif event["type"] == "m.room.power_levels":
+                levels = event["content"]
+            elif event["type"] == "m.room.canonical_alias":
+                title = title or event["content"]["alias"]
+        except KeyError:
+            # Some state event probably has empty content
+            pass
+    return title, about, levels
+
+
+async def user_has_power_level(room: str, intent, sender: u.User, event: str, default: int = 50
+                               ) -> bool:
+    if sender.is_admin:
+        return True
+    # Make sure the state store contains the power levels.
+    try:
+        await intent.get_power_levels(room)
+    except MatrixRequestError:
+        return False
+    return intent.state_store.has_power_level(room, sender.mxid,
+                                              event=f"net.maunium.telegram.{event}",
+                                              default=default)

mautrix_telegram/commands/telegram/__init__.py

@@ -0,0 +1 @@
+from . import account, auth, misc

mautrix_telegram/commands/telegram/account.py

@@ -0,0 +1,124 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Dict, Optional
+
+from telethon.errors import (UsernameInvalidError, UsernameNotModifiedError, UsernameOccupiedError,
+                             HashInvalidError, AuthKeyError, FirstNameInvalidError)
+from telethon.tl.types import Authorization
+from telethon.tl.functions.account import (UpdateUsernameRequest, GetAuthorizationsRequest,
+                                           ResetAuthorizationRequest, UpdateProfileRequest)
+
+from .. import command_handler, CommandEvent, SECTION_AUTH
+
+
+@command_handler(needs_auth=True,
+                 help_section=SECTION_AUTH,
+                 help_args="<_new username_>",
+                 help_text="Change your Telegram username.")
+async def username(evt: CommandEvent) -> Optional[Dict]:
+    if len(evt.args) == 0:
+        return await evt.reply("**Usage:** `$cmdprefix+sp username <new username>`")
+    if evt.sender.is_bot:
+        return await evt.reply("Bots can't set their own username.")
+    new_name = evt.args[0]
+    if new_name == "-":
+        new_name = ""
+    try:
+        await evt.sender.client(UpdateUsernameRequest(username=new_name))
+    except UsernameInvalidError:
+        return await evt.reply("Invalid username. Usernames must be between 5 and 30 alphanumeric "
+                               "characters.")
+    except UsernameNotModifiedError:
+        return await evt.reply("That is your current username.")
+    except UsernameOccupiedError:
+        return await evt.reply("That username is already in use.")
+    await evt.sender.update_info()
+    if not evt.sender.username:
+        await evt.reply("Username removed")
+    else:
+        await evt.reply(f"Username changed to {evt.sender.username}")
+
+
+@command_handler(needs_auth=True, help_section=SECTION_AUTH, help_args="<_new displayname_>",
+                 help_text="Change your Telegram displayname.")
+async def displayname(evt: CommandEvent) -> Optional[Dict]:
+    if len(evt.args) == 0:
+        return await evt.reply("**Usage:** `$cmdprefix+sp displayname <new displayname>`")
+    if evt.sender.is_bot:
+        return await evt.reply("Bots can't set their own displayname.")
+
+    first_name, last_name = ((evt.args[0], "")
+                             if len(evt.args) == 1
+                             else (" ".join(evt.args[:-1]), evt.args[-1]))
+    try:
+        await evt.sender.client(UpdateProfileRequest(first_name=first_name, last_name=last_name))
+    except FirstNameInvalidError:
+        return await evt.reply("Invalid first name")
+    await evt.sender.update_info()
+    await evt.reply("Displayname updated")
+
+
+def _format_session(sess: Authorization) -> str:
+    return (f"**{sess.app_name} {sess.app_version}**  \n"
+            f"  **Platform:** {sess.device_model} {sess.platform} {sess.system_version}  \n"
+            f"  **Active:** {sess.date_active} (created {sess.date_created})  \n"
+            f"  **From:** {sess.ip} - {sess.region}, {sess.country}")
+
+
+@command_handler(needs_auth=True,
+                 help_section=SECTION_AUTH,
+                 help_args="<`list`|`terminate`> [_hash_]",
+                 help_text="View or delete other Telegram sessions.")
+async def session(evt: CommandEvent) -> Optional[Dict]:
+    if len(evt.args) == 0:
+        return await evt.reply("**Usage:** `$cmdprefix+sp session <list|terminate> [hash]`")
+    elif evt.sender.is_bot:
+        return await evt.reply("Bots can't manage their sessions")
+    cmd = evt.args[0].lower()
+    if cmd == "list":
+        res = await evt.sender.client(GetAuthorizationsRequest())
+        session_list = res.authorizations
+        current = [s for s in session_list if s.current][0]
+        current_text = _format_session(current)
+        other_text = "\n".join(f"* {_format_session(sess)}  \n"
+                               f"  **Hash:** {sess.hash}"
+                               for sess in session_list if not sess.current)
+        return await evt.reply(f"### Current session\n"
+                               f"{current_text}\n"
+                               f"\n"
+                               f"### Other active sessions\n"
+                               f"{other_text}")
+    elif cmd == "terminate" and len(evt.args) > 1:
+        try:
+            session_hash = int(evt.args[1])
+        except ValueError:
+            return await evt.reply("Hash must be an integer")
+        try:
+            ok = await evt.sender.client(ResetAuthorizationRequest(hash=session_hash))
+        except HashInvalidError:
+            return await evt.reply("Invalid session hash.")
+        except AuthKeyError as e:
+            if e.message == "FRESH_RESET_AUTHORISATION_FORBIDDEN":
+                return await evt.reply("New sessions can't terminate other sessions. "
+                                       "Please wait a while.")
+            raise
+        if ok:
+            return await evt.reply("Session terminated successfully.")
+        else:
+            return await evt.reply("Session not found.")
+    else:
+        return await evt.reply("**Usage:** `$cmdprefix+sp session <list|terminate> [hash]`")

mautrix_telegram/commands/telegram/auth.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -23,9 +23,9 @@ from telethon.errors import (
     PhoneNumberAppSignupForbiddenError, PhoneNumberBannedError, PhoneNumberFloodError,
     PhoneNumberOccupiedError, PhoneNumberUnoccupiedError, SessionPasswordNeededError)
 
-from . import command_handler, CommandEvent, SECTION_AUTH
-from .. import puppet as pu, user as u
-from ..util import format_duration
+from ... import puppet as pu, user as u
+from ...commands import command_handler, CommandEvent, SECTION_AUTH
+from ...util import format_duration, ignore_coro
 
 
 @command_handler(needs_auth=False,
@@ -34,7 +34,8 @@ from ..util import format_duration
 async def ping(evt: CommandEvent) -> Optional[Dict]:
     me = await evt.sender.client.get_me() if await evt.sender.is_logged_in() else None
     if me:
-        return await evt.reply(f"You're logged in as @{me.username}")
+        human_tg_id = f"@{me.username}" if me.username else f"+{me.phone}"
+        return await evt.reply(f"You're logged in as {human_tg_id}")
     else:
         return await evt.reply("You're not logged in.")
 
@@ -45,79 +46,12 @@ async def ping(evt: CommandEvent) -> Optional[Dict]:
 async def ping_bot(evt: CommandEvent) -> Optional[Dict]:
     if not evt.tgbot:
         return await evt.reply("Telegram message relay bot not configured.")
-    bot_info = await evt.tgbot.client.get_me()
-    mxid = pu.Puppet.get_mxid_from_id(bot_info.id)
-    displayname = bot_info.first_name
+    info, mxid = await evt.tgbot.get_me(use_cache=False)
     return await evt.reply("Telegram message relay bot is active: "
-                           f"[{displayname}](https://matrix.to/#/{mxid}) (ID {bot_info.id})\n\n"
+                           f"[{info.first_name}](https://matrix.to/#/{mxid}) (ID {info.id})\n\n"
                            "To use the bot, simply invite it to a portal room.")
 
 
-@command_handler(needs_auth=True, needs_matrix_puppeting=True,
-                 help_section=SECTION_AUTH,
-                 help_text="Revert your Telegram account's Matrix puppet to use the default Matrix "
-                           "account.")
-async def logout_matrix(evt: CommandEvent) -> Optional[Dict]:
-    puppet = pu.Puppet.get(evt.sender.tgid)
-    if not puppet.is_real_user:
-        return await evt.reply("You are not logged in with your Matrix account.")
-    await puppet.switch_mxid(None, None)
-    return await evt.reply("Reverted your Telegram account's Matrix puppet back to the default.")
-
-
-@command_handler(needs_auth=True, management_only=True, needs_matrix_puppeting=True,
-                 help_section=SECTION_AUTH,
-                 help_text="Replace your Telegram account's Matrix puppet with your own Matrix "
-                           "account")
-async def login_matrix(evt: CommandEvent) -> Optional[Dict]:
-    puppet = pu.Puppet.get(evt.sender.tgid)
-    if puppet.is_real_user:
-        return await evt.reply("You have already logged in with your Matrix account. "
-                               "Log out with `$cmdprefix+sp logout-matrix` first.")
-    allow_matrix_login = evt.config.get("bridge.allow_matrix_login", True)
-    if allow_matrix_login:
-        evt.sender.command_status = {
-            "next": enter_matrix_token,
-            "action": "Matrix login",
-        }
-    if evt.config["appservice.public.enabled"]:
-        prefix = evt.config["appservice.public.external"]
-        url = f"{prefix}/matrix-login?token={evt.public_website.make_token(evt.sender.mxid, '/matrix-login')}"
-        if allow_matrix_login:
-            return await evt.reply(
-                "This bridge instance allows you to log in inside or outside Matrix.\n\n"
-                "If you would like to log in within Matrix, please send your Matrix access token "
-                "here.\n"
-                f"If you would like to log in outside of Matrix, [click here]({url}).\n\n"
-                "Logging in outside of Matrix is recommended, because in-Matrix login would save "
-                "your access token in the message history.")
-        return await evt.reply("This bridge instance does not allow logging in inside Matrix.\n\n"
-                               f"Please visit [the login page]({url}) to log in.")
-    elif allow_matrix_login:
-        return await evt.reply(
-            "This bridge instance does not allow you to log in outside of Matrix.\n\n"
-            "Please send your Matrix access token here to log in.")
-    return await evt.reply("This bridge instance has been configured to not allow logging in.")
-
-
-async def enter_matrix_token(evt: CommandEvent) -> Dict:
-    evt.sender.command_status = None
-
-    puppet = pu.Puppet.get(evt.sender.tgid)
-    if puppet.is_real_user:
-        return await evt.reply("You have already logged in with your Matrix account. "
-                               "Log out with `$cmdprefix+sp logout-matrix` first.")
-
-    resp = await puppet.switch_mxid(" ".join(evt.args), evt.sender.mxid)
-    if resp == pu.PuppetError.OnlyLoginSelf:
-        return await evt.reply("You can only log in as your own Matrix user.")
-    elif resp == pu.PuppetError.InvalidAccessToken:
-        return await evt.reply("Failed to verify access token.")
-    assert resp == pu.PuppetError.Success, "Encountered an unhandled PuppetError."
-    return await evt.reply(
-        f"Replaced your Telegram account's Matrix puppet with {puppet.custom_mxid}.")
-
-
 @command_handler(needs_auth=False, management_only=True,
                  help_section=SECTION_AUTH,
                  help_args="<_phone_> <_full name_>",
@@ -134,7 +68,7 @@ async def register(evt: CommandEvent) -> Optional[Dict]:
     else:
         full_name = " ".join(evt.args[1:-1]), evt.args[-1]
 
-    await request_code(evt, phone_number, {
+    await _request_code(evt, phone_number, {
         "next": enter_code_register,
         "action": "Register",
         "full_name": full_name,
@@ -149,7 +83,7 @@ async def enter_code_register(evt: CommandEvent) -> Dict:
         await evt.sender.ensure_started(even_if_no_session=True)
         first_name, last_name = evt.sender.command_status["full_name"]
         user = await evt.sender.client.sign_up(evt.args[0], first_name, last_name)
-        asyncio.ensure_future(evt.sender.post_login(user), loop=evt.loop)
+        ignore_coro(asyncio.ensure_future(evt.sender.post_login(user), loop=evt.loop))
         evt.sender.command_status = None
         return await evt.reply(f"Successfully registered to Telegram.")
     except PhoneNumberOccupiedError:
@@ -172,38 +106,43 @@ async def enter_code_register(evt: CommandEvent) -> Dict:
                  help_section=SECTION_AUTH,
                  help_text="Get instructions on how to log in.")
 async def login(evt: CommandEvent) -> Optional[Dict]:
+    override_sender = False
+    if len(evt.args) > 0 and evt.sender.is_admin:
+        evt.sender = await u.User.get_by_mxid(evt.args[0]).ensure_started()
+        override_sender = True
     if await evt.sender.is_logged_in():
         return await evt.reply(f"You are already logged in as {evt.sender.human_tg_id}.")
 
     allow_matrix_login = evt.config.get("bridge.allow_matrix_login", True)
-    if allow_matrix_login:
+    if allow_matrix_login and not override_sender:
         evt.sender.command_status = {
             "next": enter_phone_or_token,
             "action": "Login",
         }
 
+    nb = "**N.B. Logging in grants the bridge full access to your Telegram account.**"
     if evt.config["appservice.public.enabled"]:
         prefix = evt.config["appservice.public.external"]
         url = f"{prefix}/login?token={evt.public_website.make_token(evt.sender.mxid, '/login')}"
-        if allow_matrix_login:
-            return await evt.reply(
-                "This bridge instance allows you to log in inside or outside Matrix.\n\n"
-                "If you would like to log in within Matrix, please send your phone number or bot "
-                "auth token here.\n"
-                f"If you would like to log in outside of Matrix, [click here]({url}).\n\n"
-                "Logging in outside of Matrix is recommended if you have two-factor authentication "
-                "enabled, because in-Matrix login would save your password in the message history.")
-        return await evt.reply("This bridge instance does not allow logging in inside Matrix.\n\n"
-                               f"Please visit [the login page]({url}) to log in.")
+        if override_sender:
+            return await evt.reply(f"[Click here to log in]({url}) as "
+                                   f"[{evt.sender.mxid}](https://matrix.to/#/{evt.sender.mxid}).")
+        elif allow_matrix_login:
+            return await evt.reply(f"[Click here to log in]({url}). Alternatively, send your phone"
+                                   f" number (or bot auth token) here to log in.\n\n{nb}")
+        return await evt.reply(f"[Click here to log in]({url}).\n\n{nb}")
     elif allow_matrix_login:
-        return await evt.reply(
-            "This bridge instance does not allow you to log in outside of Matrix.\n\n"
-            "Please send your phone number or bot auth token here to start the login process.")
+        if override_sender:
+            return await evt.reply(
+                "This bridge instance does not allow you to log in outside of Matrix. "
+                "Logging in as another user inside Matrix is not currently possible.")
+        return await evt.reply("Please send your phone number (or bot auth token) here to start "
+                               f"the login process.\n\n{nb}")
     return await evt.reply("This bridge instance has been configured to not allow logging in.")
 
 
-async def request_code(evt: CommandEvent, phone_number: str, next_status: Dict[str, Any]
-                       ) -> Dict:
+async def _request_code(evt: CommandEvent, phone_number: str, next_status: Dict[str, Any]
+                        ) -> Dict:
     ok = False
     try:
         await evt.sender.ensure_started(even_if_no_session=True)
@@ -245,13 +184,13 @@ async def enter_phone_or_token(evt: CommandEvent) -> Optional[Dict]:
     # phone numbers don't contain colons but telegram bot auth tokens do
     if evt.args[0].find(":") > 0:
         try:
-            await sign_in(evt, bot_token=evt.args[0])
+            await _sign_in(evt, bot_token=evt.args[0])
         except Exception:
             evt.log.exception("Error sending auth token")
             return await evt.reply("Unhandled exception while sending auth token. "
                                    "Check console for more details.")
     else:
-        await request_code(evt, evt.args[0], {
+        await _request_code(evt, evt.args[0], {
             "next": enter_code,
             "action": "Login",
         })
@@ -266,7 +205,7 @@ async def enter_code(evt: CommandEvent) -> Optional[Dict]:
         return await evt.reply("This bridge instance does not allow in-Matrix login. "
                                "Please use `$cmdprefix+sp login` to get login instructions")
     try:
-        await sign_in(evt, code=evt.args[0])
+        await _sign_in(evt, code=evt.args[0])
     except Exception:
         evt.log.exception("Error sending phone code")
         return await evt.reply("Unhandled exception while sending code. "
@@ -282,7 +221,7 @@ async def enter_password(evt: CommandEvent) -> Optional[Dict]:
         return await evt.reply("This bridge instance does not allow in-Matrix login. "
                                "Please use `$cmdprefix+sp login` to get login instructions")
     try:
-        await sign_in(evt, password=" ".join(evt.args))
+        await _sign_in(evt, password=" ".join(evt.args))
     except AccessTokenInvalidError:
         return await evt.reply("That bot token is not valid.")
     except AccessTokenExpiredError:
@@ -294,7 +233,7 @@ async def enter_password(evt: CommandEvent) -> Optional[Dict]:
     return None
 
 
-async def sign_in(evt: CommandEvent, **sign_in_info) -> Dict:
+async def _sign_in(evt: CommandEvent, **sign_in_info) -> Dict:
     try:
         await evt.sender.ensure_started(even_if_no_session=True)
         user = await evt.sender.client.sign_in(**sign_in_info)
@@ -304,7 +243,7 @@ async def sign_in(evt: CommandEvent, **sign_in_info) -> Dict:
             await evt.reply(f"[{existing_user.displayname}]"
                             f"(https://matrix.to/#/{existing_user.mxid})"
                             " was logged out from the account.")
-        asyncio.ensure_future(evt.sender.post_login(user), loop=evt.loop)
+        ignore_coro(asyncio.ensure_future(evt.sender.post_login(user), loop=evt.loop))
         evt.sender.command_status = None
         name = f"@{user.username}" if user.username else f"+{user.phone}"
         return await evt.reply(f"Successfully logged in as {name}")

mautrix_telegram/commands/telegram/misc.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -14,18 +14,26 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Awaitable, Dict, List, Optional, Tuple
+from typing import Dict, List, Optional, Tuple
+import codecs
+import base64
 import re
 
-from telethon.errors import (
-    InviteHashInvalidError, InviteHashExpiredError, UserAlreadyParticipantError)
-from telethon.tl.types import User as TLUser
-from telethon.tl.types import TypeUpdates
-from telethon.tl.functions.messages import ImportChatInviteRequest, CheckChatInviteRequest
+from telethon.errors import (InviteHashInvalidError, InviteHashExpiredError, OptionsTooMuchError,
+                             UserAlreadyParticipantError)
+from telethon.tl.patched import Message
+from telethon.tl.types import (User as TLUser, TypeUpdates, MessageMediaGame, MessageMediaPoll,
+                               TypePeer)
+from telethon.tl.types.messages import BotCallbackAnswer
+from telethon.tl.functions.messages import (ImportChatInviteRequest, CheckChatInviteRequest,
+                                            GetBotCallbackAnswerRequest, SendVoteRequest)
 from telethon.tl.functions.channels import JoinChannelRequest
 
-from .. import puppet as pu, portal as po
-from . import command_handler, CommandEvent, SECTION_MISC, SECTION_CREATING_PORTALS
+from ... import puppet as pu, portal as po
+from ...abstract_user import AbstractUser
+from ...db import Message as DBMessage
+from ...types import TelegramID
+from ...commands import command_handler, CommandEvent, SECTION_MISC, SECTION_CREATING_PORTALS
 
 
 @command_handler(help_section=SECTION_MISC,
@@ -66,14 +74,13 @@ async def search(evt: CommandEvent) -> Optional[Dict]:
     return await evt.reply("\n".join(reply))
 
 
-@command_handler(name="pm",
-                 help_section=SECTION_CREATING_PORTALS,
+@command_handler(help_section=SECTION_CREATING_PORTALS,
                  help_args="<_identifier_>",
                  help_text="Open a private chat with the given Telegram user. The identifier is "
                            "either the internal user ID, the username or the phone number. "
                            "**N.B.** The phone numbers you start chats with must already be in "
                            "your contacts.")
-async def private_message(evt: CommandEvent) -> Optional[Dict]:
+async def pm(evt: CommandEvent) -> Optional[Dict]:
     if len(evt.args) == 0:
         return await evt.reply("**Usage:** `$cmdprefix+sp pm <user identifier>`")
 
@@ -158,3 +165,119 @@ async def sync(evt: CommandEvent) -> Optional[Dict]:
     if not sync_only or sync_only == "me":
         await evt.sender.update_info()
     return await evt.reply("Synchronization complete.")
+
+
+PEER_TYPE_CHAT = b"g"
+
+
+class MessageIDError(ValueError):
+    def __init__(self, message: str) -> None:
+        super().__init__(message)
+        self.message = message
+
+
+async def _parse_encoded_msgid(user: AbstractUser, enc_id: str, type_name: str
+                               ) -> Tuple[TypePeer, Message]:
+    try:
+        enc_id += (4 - len(enc_id) % 4) * "="
+        enc_id = base64.b64decode(enc_id)
+        peer_type, enc_id = bytes([enc_id[0]]), enc_id[1:]
+        tgid = TelegramID(int(codecs.encode(enc_id[0:5], "hex_codec"), 16))
+        msg_id = TelegramID(int(codecs.encode(enc_id[5:10], "hex_codec"), 16))
+        space = None
+        if peer_type == PEER_TYPE_CHAT:
+            space = TelegramID(int(codecs.encode(enc_id[10:15], "hex_codec"), 16))
+    except ValueError as e:
+        raise MessageIDError(f"Invalid {type_name} ID (format)") from e
+
+    if peer_type == PEER_TYPE_CHAT:
+        orig_msg = DBMessage.get_one_by_tgid(msg_id, space)
+        if not orig_msg:
+            raise MessageIDError(f"Invalid {type_name} ID (original message not found in db)")
+        new_msg = DBMessage.get_by_mxid(orig_msg.mxid, orig_msg.mx_room, user.tgid)
+        if not new_msg:
+            raise MessageIDError(f"Invalid {type_name} ID (your copy of message not found in db)")
+        msg_id = new_msg.tgid
+    try:
+        peer = await user.client.get_input_entity(tgid)
+    except ValueError as e:
+        raise MessageIDError(f"Invalid {type_name} ID (chat not found)") from e
+
+    msg = await user.client.get_messages(entity=peer, ids=msg_id)
+    if not msg:
+        raise MessageIDError(f"Invalid {type_name} ID (message not found)")
+    return peer, msg
+
+
+@command_handler(help_section=SECTION_MISC,
+                 help_args="<_play ID_>",
+                 help_text="Play a Telegram game.")
+async def play(evt: CommandEvent) -> Optional[Dict]:
+    if len(evt.args) < 1:
+        return await evt.reply("**Usage:** `$cmdprefix+sp play <play ID>`")
+    elif not await evt.sender.is_logged_in():
+        return await evt.reply("You must be logged in with a real account to play games.")
+    elif evt.sender.is_bot:
+        return await evt.reply("Bots can't play games :(")
+
+    try:
+        peer, msg = await _parse_encoded_msgid(evt.sender, evt.args[0], type_name="play")
+    except MessageIDError as e:
+        return await evt.reply(e.message)
+
+    if not isinstance(msg.media, MessageMediaGame):
+        return await evt.reply("Invalid play ID (message doesn't look like a game)")
+
+    game = await evt.sender.client(GetBotCallbackAnswerRequest(peer=peer, msg_id=msg.id, game=True))
+    if not isinstance(game, BotCallbackAnswer):
+        return await evt.reply("Game request response invalid")
+
+    await evt.reply(f"Click [here]({game.url}) to play {msg.media.game.title}:\n\n"
+                    f"{msg.media.game.description}")
+
+
+@command_handler(help_section=SECTION_MISC,
+                 help_args="<_poll ID_> <_choice number_>",
+                 help_text="Vote in a Telegram poll.")
+async def vote(evt: CommandEvent) -> Optional[Dict]:
+    if len(evt.args) < 1:
+        return await evt.reply("**Usage:** `$cmdprefix+sp vote <poll ID> <choice number>`")
+    elif not await evt.sender.is_logged_in():
+        return await evt.reply("You must be logged in with a real account to vote in polls.")
+    elif evt.sender.is_bot:
+        return await evt.reply("Bots can't vote in polls :(")
+
+    try:
+        peer, msg = await _parse_encoded_msgid(evt.sender, evt.args[0], type_name="poll")
+    except MessageIDError as e:
+        return await evt.reply(e.message)
+
+    if not isinstance(msg.media, MessageMediaPoll):
+        return await evt.reply("Invalid poll ID (message doesn't look like a poll)")
+
+    options = []
+    for option in evt.args[1:]:
+        try:
+            if len(option) > 10:
+                raise ValueError("option index too long")
+            option_index = int(option) - 1
+        except ValueError:
+            option_index = None
+        if option_index is None:
+            return await evt.reply(f"Invalid option number \"{option}\"",
+                                   render_markdown=False, allow_html=False)
+        elif option_index < 0:
+            return await evt.reply(f"Invalid option number {option}. "
+                                   f"Option numbers must be positive.")
+        elif option_index >= len(msg.media.poll.answers):
+            return await evt.reply(f"Invalid option number {option}. "
+                                   f"The poll only has {len(msg.media.poll.answers)} options.")
+        options.append(msg.media.poll.answers[option_index].option)
+    options = [msg.media.poll.answers[int(option) - 1].option
+               for option in evt.args[1:]]
+    try:
+        resp = await evt.sender.client(SendVoteRequest(peer=peer, msg_id=msg.id, options=options))
+    except OptionsTooMuchError:
+        return await evt.reply("You passed too many options.")
+    # TODO use response
+    return await evt.mark_read()

mautrix_telegram/config.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -103,12 +103,20 @@ class DictWithRecursion:
 
 
 class Config(DictWithRecursion):
-    def __init__(self, path: str, registration_path: str, base_path: str) -> None:
+    def __init__(self, path: str, registration_path: str, base_path: str,
+                 overrides: Dict[str, Any] = None) -> None:
         super().__init__()
         self.path = path  # type: str
         self.registration_path = registration_path  # type: str
         self.base_path = base_path  # type: str
         self._registration = None  # type: Optional[Dict]
+        self._overrides = overrides or {}  # type: Dict[str, Any]
+
+    def __getitem__(self, key: str) -> Any:
+        try:
+            return self._overrides[f"MAUTRIX_TELEGRAM_{key.replace('.', '_').upper()}"]
+        except KeyError:
+            return super().__getitem__(key)
 
     def load(self) -> None:
         with open(self.path, 'r') as stream:
@@ -151,8 +159,8 @@ class Config(DictWithRecursion):
                     base[to_path][key] = value
 
         copy("homeserver.address")
-        copy("homeserver.verify_ssl")
         copy("homeserver.domain")
+        copy("homeserver.verify_ssl")
 
         if "appservice.protocol" in self and "appservice.address" not in self:
             protocol, hostname, port = (self["appservice.protocol"], self["appservice.hostname"],
@@ -165,7 +173,6 @@ class Config(DictWithRecursion):
         copy("appservice.max_body_size")
 
         copy("appservice.database")
-        copy("appservice.sqlalchemy_core_mode")
 
         copy("appservice.public.enabled")
         copy("appservice.public.prefix")
@@ -185,14 +192,33 @@ class Config(DictWithRecursion):
         copy("appservice.as_token")
         copy("appservice.hs_token")
 
+        copy("metrics.enabled")
+        copy("metrics.listen_port")
+
         copy("bridge.username_template")
         copy("bridge.alias_template")
         copy("bridge.displayname_template")
 
         copy("bridge.displayname_preference")
 
-        copy("bridge.edits_as_replies")
-        copy("bridge.highlight_edits")
+        copy("bridge.max_initial_member_sync")
+        copy("bridge.sync_channel_members")
+        copy("bridge.skip_deleted_members")
+        copy("bridge.startup_sync")
+        copy("bridge.sync_dialog_limit")
+        copy("bridge.max_telegram_delete")
+        copy("bridge.sync_matrix_state")
+        copy("bridge.allow_matrix_login")
+        copy("bridge.plaintext_highlights")
+        copy("bridge.public_portals")
+        copy("bridge.catch_up")
+        copy("bridge.sync_with_custom_puppets")
+        copy("bridge.telegram_link_preview")
+        copy("bridge.inline_images")
+        copy("bridge.image_as_file_size")
+        copy("bridge.max_document_size")
+
+        copy("bridge.bot_messages_as_notices")
         if isinstance(self["bridge.bridge_notices"], bool):
             base["bridge.bridge_notices"] = {
                 "default": self["bridge.bridge_notices"],
@@ -200,17 +226,6 @@ class Config(DictWithRecursion):
             }
         else:
             copy("bridge.bridge_notices")
-        copy("bridge.bot_messages_as_notices")
-        copy("bridge.max_initial_member_sync")
-        copy("bridge.sync_channel_members")
-        copy("bridge.max_telegram_delete")
-        copy("bridge.allow_matrix_login")
-        copy("bridge.inline_images")
-        copy("bridge.plaintext_highlights")
-        copy("bridge.public_portals")
-        copy("bridge.native_stickers")
-        copy("bridge.catch_up")
-        copy("bridge.sync_with_custom_puppets")
 
         copy("bridge.deduplication.pre_db_check")
         copy("bridge.deduplication.cache_queue_length")
@@ -218,6 +233,7 @@ class Config(DictWithRecursion):
         if "bridge.message_formats.m_text" in self:
             del self["bridge.message_formats"]
         copy_dict("bridge.message_formats", override_existing_map=False)
+
         copy("bridge.state_event_formats.join")
         copy("bridge.state_event_formats.leave")
         copy("bridge.state_event_formats.name_change")
@@ -251,6 +267,24 @@ class Config(DictWithRecursion):
         copy("telegram.api_id")
         copy("telegram.api_hash")
         copy("telegram.bot_token")
+
+        copy("telegram.connection.timeout")
+        copy("telegram.connection.retries")
+        copy("telegram.connection.retry_delay")
+        copy("telegram.connection.flood_sleep_threshold")
+        copy("telegram.connection.request_retries")
+
+        copy("telegram.device_info.device_model")
+        copy("telegram.device_info.system_version")
+        copy("telegram.device_info.app_version")
+        copy("telegram.device_info.lang_code")
+        copy("telegram.device_info.system_lang_code")
+
+        copy("telegram.server.enabled")
+        copy("telegram.server.dc")
+        copy("telegram.server.ip")
+        copy("telegram.server.port")
+
         copy("telegram.proxy.type")
         copy("telegram.proxy.address")
         copy("telegram.proxy.port")

mautrix_telegram/context.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -19,8 +19,6 @@ from typing import Optional, Tuple, TYPE_CHECKING
 if TYPE_CHECKING:
     import asyncio
 
-    from sqlalchemy.orm import scoped_session
-
     from alchemysession import AlchemySessionContainer
     from mautrix_appservice import AppService
 
@@ -31,20 +29,17 @@ if TYPE_CHECKING:
 
 
 class Context:
-    def __init__(self, az: "AppService", db: "scoped_session", config: "Config",
-                 loop: "asyncio.AbstractEventLoop", session_container: "AlchemySessionContainer"
-                 ) -> None:
+    def __init__(self, az: 'AppService', config: 'Config', loop: 'asyncio.AbstractEventLoop',
+                 session_container: 'AlchemySessionContainer', bot: Optional['Bot']) -> None:
         self.az = az  # type: AppService
-        self.db = db  # type: scoped_session
         self.config = config  # type: Config
         self.loop = loop  # type: asyncio.AbstractEventLoop
-        self.bot = None  # type: Optional[Bot]
-        self.mx = None  # type: MatrixHandler
+        self.bot = bot  # type: Optional[Bot]
+        self.mx = None  # type: Optional[MatrixHandler]
         self.session_container = session_container  # type: AlchemySessionContainer
-        self.public_website = None  # type: PublicBridgeWebsite
-        self.provisioning_api = None  # type: ProvisioningAPI
+        self.public_website = None  # type: Optional[PublicBridgeWebsite]
+        self.provisioning_api = None  # type: Optional[ProvisioningAPI]
 
     @property
-    def core(self) -> Tuple['AppService', 'scoped_session', 'Config',
-                            'asyncio.AbstractEventLoop', Optional['Bot']]:
-        return (self.az, self.db, self.config, self.loop, self.bot)
+    def core(self) -> Tuple['AppService', 'Config', 'asyncio.AbstractEventLoop', Optional['Bot']]:
+        return self.az, self.config, self.loop, self.bot

mautrix_telegram/db.py

@@ -1,258 +0,0 @@
-# -*- coding: future_fstrings -*-
-# mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from sqlalchemy import (Column, UniqueConstraint, ForeignKey, ForeignKeyConstraint, Integer,
-                        BigInteger, String, Boolean, Text, Table,
-                        and_, func, select)
-from sqlalchemy.engine import Engine, RowProxy
-from sqlalchemy.sql import expression
-from sqlalchemy.orm import relationship, Query
-from sqlalchemy.sql.base import ImmutableColumnCollection
-from typing import Dict, Optional, List
-import json
-
-from mautrix_telegram.types import MatrixUserID, MatrixRoomID, MatrixEventID
-from .types import TelegramID
-from .base import Base
-
-
-class Portal(Base):
-    query = None  # type: Query
-    __tablename__ = "portal"
-
-    # Telegram chat information
-    tgid = Column(Integer, primary_key=True)  # type: TelegramID
-    tg_receiver = Column(Integer, primary_key=True)  # type: TelegramID
-    peer_type = Column(String, nullable=False)
-    megagroup = Column(Boolean)
-
-    # Matrix portal information
-    mxid = Column(String, unique=True, nullable=True)  # type: Optional[MatrixRoomID]
-
-    config = Column(Text, nullable=True)
-
-    # Telegram chat metadata
-    username = Column(String, nullable=True)
-    title = Column(String, nullable=True)
-    about = Column(String, nullable=True)
-    photo_id = Column(String, nullable=True)
-
-
-class Message(Base):
-    db = None  # type: Engine
-    t = None  # type: Table
-    c = None  # type: ImmutableColumnCollection
-    __tablename__ = "message"
-
-    mxid = Column(String)  # type: MatrixEventID
-    mx_room = Column(String)  # type: MatrixRoomID
-    tgid = Column(Integer, primary_key=True)  # type: TelegramID
-    tg_space = Column(Integer, primary_key=True)  # type: TelegramID
-
-    __table_args__ = (UniqueConstraint("mxid", "mx_room", "tg_space", name="_mx_id_room"),)
-
-    @staticmethod
-    def _one_or_none(rows: RowProxy) -> Optional['Message']:
-        try:
-            mxid, mx_room, tgid, tg_space = next(rows)
-            return Message(mxid=mxid, mx_room=mx_room, tgid=tgid, tg_space=tg_space)
-        except StopIteration:
-            return None
-
-    @staticmethod
-    def _all(rows: RowProxy) -> List['Message']:
-        return [Message(mxid=row[0], mx_room=row[1], tgid=row[2], tg_space=row[3])
-                for row in rows]
-
-    @classmethod
-    def get_by_tgid(cls, tgid: TelegramID, tg_space: TelegramID) -> Optional['Message']:
-        rows = cls.db.execute(cls.t.select()
-                              .where(and_(cls.c.tgid == tgid, cls.c.tg_space == tg_space)))
-        return cls._one_or_none(rows)
-
-    @classmethod
-    def count_spaces_by_mxid(cls, mxid: MatrixEventID, mx_room: MatrixRoomID) -> int:
-        rows = cls.db.execute(select([func.count(cls.c.tg_space)])
-                              .where(and_(cls.c.mxid == mxid, cls.c.mx_room == mx_room)))
-        try:
-            count, = next(rows)
-            return count
-        except StopIteration:
-            return 0
-
-    @classmethod
-    def get_by_mxid(cls, mxid: MatrixEventID, mx_room: MatrixRoomID, tg_space: TelegramID
-                    ) -> Optional['Message']:
-        rows = cls.db.execute(cls.t.select().where(
-            and_(cls.c.mxid == mxid, cls.c.mx_room == mx_room, cls.c.tg_space == tg_space)))
-        return cls._one_or_none(rows)
-
-    @classmethod
-    def update_by_tgid(cls, s_tgid: TelegramID, s_tg_space: TelegramID, **values) -> None:
-        cls.db.execute(cls.t.update()
-                       .where(and_(cls.c.tgid == s_tgid, cls.c.tg_space == s_tg_space))
-                       .values(**values))
-
-    @classmethod
-    def update_by_mxid(cls, s_mxid: MatrixEventID, s_mx_room: MatrixRoomID, **values) -> None:
-        cls.db.execute(cls.t.update()
-                       .where(and_(cls.c.mxid == s_mxid, cls.c.mx_room == s_mx_room))
-                       .values(**values))
-
-    def update(self, **values) -> None:
-        for key, value in values.items():
-            setattr(self, key, value)
-        self.update_by_tgid(self.tgid, self.tg_space, **values)
-
-    def delete(self) -> None:
-        self.db.execute(self.t.delete().where(
-            and_(self.c.tgid == self.tgid, self.c.tg_space == self.tg_space)))
-
-    def insert(self) -> None:
-        self.db.execute(self.t.insert().values(mxid=self.mxid, mx_room=self.mx_room, tgid=self.tgid,
-                                               tg_space=self.tg_space))
-
-
-class UserPortal(Base):
-    query = None  # type: Query
-    __tablename__ = "user_portal"
-
-    user = Column(Integer, ForeignKey("user.tgid", onupdate="CASCADE", ondelete="CASCADE"),
-                  primary_key=True)  # type: TelegramID
-    portal = Column(Integer, primary_key=True)  # type: TelegramID
-    portal_receiver = Column(Integer, primary_key=True)  # type: TelegramID
-
-    __table_args__ = (ForeignKeyConstraint(("portal", "portal_receiver"),
-                                           ("portal.tgid", "portal.tg_receiver"),
-                                           onupdate="CASCADE", ondelete="CASCADE"),)
-
-
-class User(Base):
-    query = None  # type: Query
-    __tablename__ = "user"
-
-    mxid = Column(String, primary_key=True)  # type: MatrixUserID
-    tgid = Column(Integer, nullable=True, unique=True)  # type: Optional[TelegramID]
-    tg_username = Column(String, nullable=True)
-    tg_phone = Column(String, nullable=True)
-    saved_contacts = Column(Integer, default=0, nullable=False)
-    contacts = relationship("Contact", uselist=True,
-                            cascade="save-update, merge, delete, delete-orphan"
-                            )  # type: List[Contact]
-    portals = relationship("Portal", secondary="user_portal")
-
-
-class RoomState(Base):
-    query = None  # type: Query
-    __tablename__ = "mx_room_state"
-
-    room_id = Column(String, primary_key=True)  # type: MatrixRoomID
-    _power_levels_text = Column("power_levels", Text, nullable=True)
-    _power_levels_json = {}  # type: Dict
-
-    @property
-    def has_power_levels(self) -> bool:
-        return bool(self._power_levels_text)
-
-    @property
-    def power_levels(self) -> Dict:
-        if not self._power_levels_json and self._power_levels_text:
-            self._power_levels_json = json.loads(self._power_levels_text)
-        return self._power_levels_json
-
-    @power_levels.setter
-    def power_levels(self, val: Dict) -> None:
-        self._power_levels_json = val
-        self._power_levels_text = json.dumps(val)
-
-
-class UserProfile(Base):
-    query = None  # type: Query
-    __tablename__ = "mx_user_profile"
-
-    room_id = Column(String, primary_key=True)  # type: MatrixRoomID
-    user_id = Column(String, primary_key=True)  # type: MatrixUserID
-    membership = Column(String, nullable=False, default="leave")
-    displayname = Column(String, nullable=True)
-    avatar_url = Column(String, nullable=True)
-
-    def dict(self) -> Dict[str, str]:
-        return {
-            "membership": self.membership,
-            "displayname": self.displayname,
-            "avatar_url": self.avatar_url,
-        }
-
-
-class Contact(Base):
-    query = None  # type: Query
-    __tablename__ = "contact"
-
-    user = Column(Integer, ForeignKey("user.tgid"), primary_key=True)  # type: TelegramID
-    contact = Column(Integer, ForeignKey("puppet.id"), primary_key=True)  # type: TelegramID
-
-
-class Puppet(Base):
-    query = None  # type: Query
-    __tablename__ = "puppet"
-
-    id = Column(Integer, primary_key=True)  # type: TelegramID
-    custom_mxid = Column(String, nullable=True)  # type: Optional[MatrixUserID]
-    access_token = Column(String, nullable=True)
-    displayname = Column(String, nullable=True)
-    displayname_source = Column(Integer, nullable=True)  # type: Optional[TelegramID]
-    username = Column(String, nullable=True)
-    photo_id = Column(String, nullable=True)
-    is_bot = Column(Boolean, nullable=True)
-    matrix_registered = Column(Boolean, nullable=False, server_default=expression.false())
-
-
-# Fucking Telegram not telling bots what chats they are in 3:<
-class BotChat(Base):
-    query = None  # type: Query
-    __tablename__ = "bot_chat"
-    id = Column(Integer, primary_key=True)  # type: TelegramID
-    type = Column(String, nullable=False)
-
-
-class TelegramFile(Base):
-    query = None  # type: Query
-    __tablename__ = "telegram_file"
-
-    id = Column(String, primary_key=True)
-    mxc = Column(String)
-    mime_type = Column(String)
-    was_converted = Column(Boolean)
-    timestamp = Column(BigInteger)
-    size = Column(Integer, nullable=True)
-    width = Column(Integer, nullable=True)
-    height = Column(Integer, nullable=True)
-    thumbnail_id = Column("thumbnail", String, ForeignKey("telegram_file.id"), nullable=True)
-    thumbnail = relationship("TelegramFile", uselist=False)
-
-
-def init(db_session, db_engine) -> None:
-    Portal.query = db_session.query_property()
-    Message.db = db_engine
-    Message.t = Message.__table__
-    Message.c = Message.t.c
-    UserPortal.query = db_session.query_property()
-    User.query = db_session.query_property()
-    Puppet.query = db_session.query_property()
-    BotChat.query = db_session.query_property()
-    TelegramFile.query = db_session.query_property()
-    UserProfile.query = db_session.query_property()
-    RoomState.query = db_session.query_property()

mautrix_telegram/db/__init__.py

@@ -0,0 +1,33 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from .base import Base
+from .bot_chat import BotChat
+from .message import Message
+from .portal import Portal
+from .puppet import Puppet
+from .room_state import RoomState
+from .telegram_file import TelegramFile
+from .user import User, UserPortal, Contact
+from .user_profile import UserProfile
+
+
+def init(db_engine) -> None:
+    for table in (Portal, Message, User, Contact, UserPortal, Puppet, TelegramFile, UserProfile,
+                  RoomState, BotChat):
+        table.db = db_engine
+        table.t = table.__table__
+        table.c = table.t.c

mautrix_telegram/db/base.py

@@ -0,0 +1,58 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from abc import abstractmethod
+
+from sqlalchemy import Table
+from sqlalchemy.engine.base import Engine
+from sqlalchemy.engine.result import RowProxy
+from sqlalchemy.sql.base import ImmutableColumnCollection
+from sqlalchemy.ext.declarative import declarative_base
+
+
+class BaseBase:
+    db = None  # type: Engine
+    t = None  # type: Table
+    __table__ = None  # type: Table
+    c = None  # type: ImmutableColumnCollection
+
+    @classmethod
+    @abstractmethod
+    def _one_or_none(cls, rows: RowProxy):
+        pass
+
+    @classmethod
+    def _select_one_or_none(cls, *args):
+        return cls._one_or_none(cls.db.execute(cls.t.select().where(*args)))
+
+    @property
+    @abstractmethod
+    def _edit_identity(self):
+        pass
+
+    def update(self, **values) -> None:
+        with self.db.begin() as conn:
+            conn.execute(self.t.update()
+                         .where(self._edit_identity)
+                         .values(**values))
+        for key, value in values.items():
+            setattr(self, key, value)
+
+    def delete(self) -> None:
+        with self.db.begin() as conn:
+            conn.execute(self.t.delete().where(self._edit_identity))
+
+Base = declarative_base(cls=BaseBase)

mautrix_telegram/db/base.pyi

@@ -0,0 +1,26 @@
+from abc import abstractmethod
+
+from sqlalchemy import Table
+from sqlalchemy.engine.base import Engine
+from sqlalchemy.engine.result import RowProxy
+from sqlalchemy.sql.base import ImmutableColumnCollection
+from sqlalchemy.ext.declarative import declarative_base
+
+class Base(declarative_base):
+	db: Engine
+	t: Table
+	__table__: Table
+	c: ImmutableColumnCollection
+
+    @classmethod
+    @abstractmethod
+    def _one_or_none(cls, rows: RowProxy): ...
+
+	@classmethod
+	def _select_one_or_none(cls, *args): ...
+
+    def _edit_identity(self): ...
+
+    def update(self, **values) -> None: ...
+
+    def delete(self) -> None: ...

mautrix_telegram/db/bot_chat.py

@@ -0,0 +1,45 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Iterable
+
+from sqlalchemy import Column, Integer, String
+
+from ..types import TelegramID
+from .base import Base
+
+
+# Fucking Telegram not telling bots what chats they are in 3:<
+class BotChat(Base):
+    __tablename__ = "bot_chat"
+    id = Column(Integer, primary_key=True)  # type: TelegramID
+    type = Column(String, nullable=False)
+
+    @classmethod
+    def delete(cls, chat_id: TelegramID) -> None:
+        with cls.db.begin() as conn:
+            conn.execute(cls.t.delete().where(cls.c.id == chat_id))
+
+    @classmethod
+    def all(cls) -> Iterable['BotChat']:
+        rows = cls.db.execute(cls.t.select())
+        for row in rows:
+            chat_id, chat_type = row
+            yield cls(id=chat_id, type=chat_type)
+
+    def insert(self) -> None:
+        with self.db.begin() as conn:
+            conn.execute(self.t.insert().values(id=self.id, type=self.type))

mautrix_telegram/db/message.py

@@ -0,0 +1,113 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from sqlalchemy import Column, UniqueConstraint, Integer, String, and_, func, desc, select
+from sqlalchemy.engine.result import RowProxy
+from typing import Optional, List
+
+from ..types import MatrixRoomID, MatrixEventID, TelegramID
+from .base import Base
+
+
+class Message(Base):
+    __tablename__ = "message"
+
+    mxid = Column(String)  # type: MatrixEventID
+    mx_room = Column(String)  # type: MatrixRoomID
+    tgid = Column(Integer, primary_key=True)  # type: TelegramID
+    tg_space = Column(Integer, primary_key=True)  # type: TelegramID
+    edit_index = Column(Integer, primary_key=True)  # type: int
+
+    __table_args__ = (UniqueConstraint("mxid", "mx_room", "tg_space", name="_mx_id_room"),)
+
+    @classmethod
+    def _one_or_none(cls, rows: RowProxy) -> Optional['Message']:
+        try:
+            mxid, mx_room, tgid, tg_space, edit_index = next(rows)
+            return cls(mxid=mxid, mx_room=mx_room, tgid=tgid, tg_space=tg_space,
+                       edit_index=edit_index)
+        except StopIteration:
+            return None
+
+    @staticmethod
+    def _all(rows: RowProxy) -> List['Message']:
+        return [Message(mxid=row[0], mx_room=row[1], tgid=row[2], tg_space=row[3],
+                        edit_index=row[4])
+                for row in rows]
+
+    @classmethod
+    def get_all_by_tgid(cls, tgid: TelegramID, tg_space: TelegramID) -> List['Message']:
+        return cls._all(cls.db.execute(cls.t.select().where(and_(cls.c.tgid == tgid,
+                                                                 cls.c.tg_space == tg_space))))
+
+    @classmethod
+    def get_one_by_tgid(cls, tgid: TelegramID, tg_space: TelegramID, edit_index: int = 0
+                        ) -> Optional['Message']:
+        query = cls.t.select()
+        if edit_index < 0:
+            query = (query
+                     .where(and_(cls.c.tgid == tgid, cls.c.tg_space == tg_space))
+                     .order_by(desc(cls.c.edit_index))
+                     .limit(1)
+                     .offset(-edit_index - 1))
+        else:
+            query = query.where(and_(cls.c.tgid == tgid, cls.c.tg_space == tg_space,
+                                     cls.c.edit_index == edit_index))
+        return cls._one_or_none(cls.db.execute(query))
+
+    @classmethod
+    def count_spaces_by_mxid(cls, mxid: MatrixEventID, mx_room: MatrixRoomID) -> int:
+        rows = cls.db.execute(select([func.count(cls.c.tg_space)])
+                              .where(and_(cls.c.mxid == mxid, cls.c.mx_room == mx_room)))
+        try:
+            count, = next(rows)
+            return count
+        except StopIteration:
+            return 0
+
+    @classmethod
+    def get_by_mxid(cls, mxid: MatrixEventID, mx_room: MatrixRoomID, tg_space: TelegramID
+                    ) -> Optional['Message']:
+        return cls._select_one_or_none(and_(cls.c.mxid == mxid,
+                                            cls.c.mx_room == mx_room,
+                                            cls.c.tg_space == tg_space))
+
+    @classmethod
+    def update_by_tgid(cls, s_tgid: TelegramID, s_tg_space: TelegramID, s_edit_index: int,
+                       **values) -> None:
+        with cls.db.begin() as conn:
+            conn.execute(cls.t.update()
+                         .where(and_(cls.c.tgid == s_tgid, cls.c.tg_space == s_tg_space,
+                                     cls.c.edit_index == s_edit_index))
+                         .values(**values))
+
+    @classmethod
+    def update_by_mxid(cls, s_mxid: MatrixEventID, s_mx_room: MatrixRoomID, **values) -> None:
+        with cls.db.begin() as conn:
+            conn.execute(cls.t.update()
+                         .where(and_(cls.c.mxid == s_mxid, cls.c.mx_room == s_mx_room))
+                         .values(**values))
+
+    @property
+    def _edit_identity(self):
+        return and_(self.c.tgid == self.tgid, self.c.tg_space == self.tg_space,
+                    self.c.edit_index == self.edit_index)
+
+    def insert(self) -> None:
+        with self.db.begin() as conn:
+            conn.execute(self.t.insert().values(mxid=self.mxid, mx_room=self.mx_room,
+                                                tgid=self.tgid, tg_space=self.tg_space,
+                                                edit_index=self.edit_index))

mautrix_telegram/db/portal.py

@@ -0,0 +1,81 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from sqlalchemy import Column, Integer, String, Boolean, Text, and_
+from sqlalchemy.engine.result import RowProxy
+from typing import Optional
+
+from ..types import MatrixRoomID, TelegramID
+from .base import Base
+
+
+class Portal(Base):
+    __tablename__ = "portal"
+
+    # Telegram chat information
+    tgid = Column(Integer, primary_key=True)  # type: TelegramID
+    tg_receiver = Column(Integer, primary_key=True)  # type: TelegramID
+    peer_type = Column(String, nullable=False)
+    megagroup = Column(Boolean)
+
+    # Matrix portal information
+    mxid = Column(String, unique=True, nullable=True)  # type: Optional[MatrixRoomID]
+
+    config = Column(Text, nullable=True)
+
+    # Telegram chat metadata
+    username = Column(String, nullable=True)
+    title = Column(String, nullable=True)
+    about = Column(String, nullable=True)
+    photo_id = Column(String, nullable=True)
+
+    @classmethod
+    def scan(cls, row) -> Optional['Portal']:
+        (tgid, tg_receiver, peer_type, megagroup, mxid, config, username, title, about,
+         photo_id) = row
+        return cls(tgid=tgid, tg_receiver=tg_receiver, peer_type=peer_type, megagroup=megagroup,
+                   mxid=mxid, config=config, username=username, title=title, about=about,
+                   photo_id=photo_id)
+
+    @classmethod
+    def _one_or_none(cls, rows: RowProxy) -> Optional['Portal']:
+        try:
+            return cls.scan(next(rows))
+        except StopIteration:
+            return None
+
+    @classmethod
+    def get_by_tgid(cls, tgid: TelegramID, tg_receiver: TelegramID) -> Optional['Portal']:
+        return cls._select_one_or_none(and_(cls.c.tgid == tgid, cls.c.tg_receiver == tg_receiver))
+
+    @classmethod
+    def get_by_mxid(cls, mxid: MatrixRoomID) -> Optional['Portal']:
+        return cls._select_one_or_none(cls.c.mxid == mxid)
+
+    @classmethod
+    def get_by_username(cls, username: str) -> Optional['Portal']:
+        return cls._select_one_or_none(cls.c.username == username)
+
+    @property
+    def _edit_identity(self):
+        return and_(self.c.tgid == self.tgid, self.c.tg_receiver == self.tg_receiver)
+
+    def insert(self) -> None:
+        with self.db.begin() as conn:
+            conn.execute(self.t.insert().values(
+                tgid=self.tgid, tg_receiver=self.tg_receiver, peer_type=self.peer_type,
+                megagroup=self.megagroup, mxid=self.mxid, config=self.config,
+                username=self.username, title=self.title, about=self.about, photo_id=self.photo_id))

mautrix_telegram/db/puppet.py

@@ -0,0 +1,88 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from sqlalchemy import Column, Integer, String, Boolean
+from sqlalchemy.engine.result import RowProxy
+from sqlalchemy.sql import expression
+from typing import Optional, Iterable
+
+from ..types import MatrixUserID, MatrixRoomID, TelegramID
+from .base import Base
+
+
+class Puppet(Base):
+    __tablename__ = "puppet"
+
+    id = Column(Integer, primary_key=True)  # type: TelegramID
+    custom_mxid = Column(String, nullable=True)  # type: Optional[MatrixUserID]
+    access_token = Column(String, nullable=True)
+    displayname = Column(String, nullable=True)
+    displayname_source = Column(Integer, nullable=True)  # type: Optional[TelegramID]
+    username = Column(String, nullable=True)
+    photo_id = Column(String, nullable=True)
+    is_bot = Column(Boolean, nullable=True)
+    matrix_registered = Column(Boolean, nullable=False, server_default=expression.false())
+    disable_updates = Column(Boolean, nullable=False, server_default=expression.false())
+
+    @classmethod
+    def scan(cls, row) -> Optional['Puppet']:
+        (id, custom_mxid, access_token, displayname, displayname_source, username, photo_id,
+         is_bot, matrix_registered, disable_updates) = row
+        return cls(id=id, custom_mxid=custom_mxid, access_token=access_token,
+                   displayname=displayname, displayname_source=displayname_source,
+                   username=username, photo_id=photo_id, is_bot=is_bot,
+                   matrix_registered=matrix_registered, disable_updates=disable_updates)
+
+    @classmethod
+    def _one_or_none(cls, rows: RowProxy) -> Optional['Puppet']:
+        try:
+            return cls.scan(next(rows))
+        except StopIteration:
+            return None
+
+    @classmethod
+    def all_with_custom_mxid(cls) -> Iterable['Puppet']:
+        rows = cls.db.execute(cls.t.select().where(cls.c.custom_mxid != None))
+        for row in rows:
+            yield cls.scan(row)
+
+    @classmethod
+    def get_by_tgid(cls, tgid: TelegramID) -> Optional['Puppet']:
+        return cls._select_one_or_none(cls.c.id == tgid)
+
+    @classmethod
+    def get_by_custom_mxid(cls, mxid: MatrixUserID) -> Optional['Puppet']:
+        return cls._select_one_or_none(cls.c.custom_mxid == mxid)
+
+    @classmethod
+    def get_by_username(cls, username: str) -> Optional['Puppet']:
+        return cls._select_one_or_none(cls.c.username == username)
+
+    @classmethod
+    def get_by_displayname(cls, displayname: str) -> Optional['Puppet']:
+        return cls._select_one_or_none(cls.c.displayname == displayname)
+
+    @property
+    def _edit_identity(self):
+        return self.c.id == self.id
+
+    def insert(self) -> None:
+        with self.db.begin() as conn:
+            conn.execute(self.t.insert().values(
+                id=self.id, custom_mxid=self.custom_mxid, access_token=self.access_token,
+                displayname=self.displayname, displayname_source=self.displayname_source,
+                username=self.username, photo_id=self.photo_id, is_bot=self.is_bot,
+                matrix_registered=self.matrix_registered, disable_updates=self.disable_updates))

mautrix_telegram/db/room_state.py

@@ -0,0 +1,62 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from sqlalchemy import Column, String, Text
+from typing import Dict, Optional
+import json
+
+from ..types import MatrixRoomID
+from .base import Base
+
+
+class RoomState(Base):
+    __tablename__ = "mx_room_state"
+
+    room_id = Column(String, primary_key=True)  # type: MatrixRoomID
+    power_levels = Column("power_levels", Text, nullable=True)  # type: Optional[Dict]
+
+    @property
+    def _power_levels_text(self) -> Optional[str]:
+        return json.dumps(self.power_levels) if self.power_levels else None
+
+    @property
+    def has_power_levels(self) -> bool:
+        return bool(self.power_levels)
+
+    @classmethod
+    def get(cls, room_id: MatrixRoomID) -> Optional['RoomState']:
+        rows = cls.db.execute(cls.t.select().where(cls.c.room_id == room_id))
+        try:
+            room_id, power_levels_text = next(rows)
+            return cls(room_id=room_id, power_levels=(json.loads(power_levels_text)
+                                                      if power_levels_text else None))
+        except StopIteration:
+            return None
+
+    def update(self) -> None:
+        with self.db.begin() as conn:
+            conn.execute(self.t.update()
+                         .where(self.c.room_id == self.room_id)
+                         .values(power_levels=self._power_levels_text))
+
+    @property
+    def _edit_identity(self):
+        return self.c.room_id == self.room_id
+
+    def insert(self) -> None:
+        with self.db.begin() as conn:
+            conn.execute(self.t.insert().values(room_id=self.room_id,
+                                                power_levels=self._power_levels_text))

mautrix_telegram/db/telegram_file.py

@@ -0,0 +1,56 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from sqlalchemy import Column, ForeignKey, Integer, BigInteger, String, Boolean
+from typing import Optional
+
+from .base import Base
+
+
+class TelegramFile(Base):
+    __tablename__ = "telegram_file"
+
+    id = Column(String, primary_key=True)
+    mxc = Column(String)
+    mime_type = Column(String)
+    was_converted = Column(Boolean)
+    timestamp = Column(BigInteger)
+    size = Column(Integer, nullable=True)
+    width = Column(Integer, nullable=True)
+    height = Column(Integer, nullable=True)
+    thumbnail_id = Column("thumbnail", String, ForeignKey("telegram_file.id"), nullable=True)
+    thumbnail = None  # type: Optional[TelegramFile]
+
+    @classmethod
+    def get(cls, loc_id: str) -> Optional['TelegramFile']:
+        rows = cls.db.execute(cls.t.select().where(cls.c.id == loc_id))
+        try:
+            loc_id, mxc, mime, conv, ts, s, w, h, thumb_id = next(rows)
+            thumb = None
+            if thumb_id:
+                thumb = cls.get(thumb_id)
+            return cls(id=loc_id, mxc=mxc, mime_type=mime, was_converted=conv, timestamp=ts,
+                       size=s, width=w, height=h, thumbnail_id=thumb_id, thumbnail=thumb)
+        except StopIteration:
+            return None
+
+    def insert(self) -> None:
+        with self.db.begin() as conn:
+            conn.execute(self.t.insert().values(
+                id=self.id, mxc=self.mxc, mime_type=self.mime_type,
+                was_converted=self.was_converted, timestamp=self.timestamp, size=self.size,
+                width=self.width, height=self.height,
+                thumbnail=self.thumbnail.id if self.thumbnail else self.thumbnail_id))

mautrix_telegram/db/user.py

@@ -0,0 +1,130 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from sqlalchemy import Column, ForeignKey, ForeignKeyConstraint, Integer, String
+from sqlalchemy.engine.result import RowProxy
+from typing import Optional, Iterable, Tuple
+
+from ..types import MatrixUserID, TelegramID
+from .base import Base
+
+
+class User(Base):
+    __tablename__ = "user"
+
+    mxid = Column(String, primary_key=True)  # type: MatrixUserID
+    tgid = Column(Integer, nullable=True, unique=True)  # type: Optional[TelegramID]
+    tg_username = Column(String, nullable=True)
+    tg_phone = Column(String, nullable=True)
+    saved_contacts = Column(Integer, default=0, nullable=False)
+
+    @classmethod
+    def _one_or_none(cls, rows: RowProxy) -> Optional['User']:
+        try:
+            mxid, tgid, tg_username, tg_phone, saved_contacts = next(rows)
+            return cls(mxid=mxid, tgid=tgid, tg_username=tg_username, tg_phone=tg_phone,
+                       saved_contacts=saved_contacts)
+        except StopIteration:
+            return None
+
+    @classmethod
+    def all(cls) -> Iterable['User']:
+        rows = cls.db.execute(cls.t.select())
+        for row in rows:
+            mxid, tgid, tg_username, tg_phone, saved_contacts = row
+            yield cls(mxid=mxid, tgid=tgid, tg_username=tg_username, tg_phone=tg_phone,
+                      saved_contacts=saved_contacts)
+
+    @classmethod
+    def get_by_tgid(cls, tgid: TelegramID) -> Optional['User']:
+        return cls._select_one_or_none(cls.c.tgid == tgid)
+
+    @classmethod
+    def get_by_mxid(cls, mxid: MatrixUserID) -> Optional['User']:
+        return cls._select_one_or_none(cls.c.mxid == mxid)
+
+    @classmethod
+    def get_by_username(cls, username: str) -> Optional['User']:
+        return cls._select_one_or_none(cls.c.tg_username == username)
+
+    @property
+    def _edit_identity(self):
+        return self.c.mxid == self.mxid
+
+    def insert(self) -> None:
+        with self.db.begin() as conn:
+            conn.execute(self.t.insert().values(
+                mxid=self.mxid, tgid=self.tgid, tg_username=self.tg_username,
+                tg_phone=self.tg_phone, saved_contacts=self.saved_contacts))
+
+    @property
+    def contacts(self) -> Iterable[TelegramID]:
+        rows = self.db.execute(Contact.t.select().where(Contact.c.user == self.tgid))
+        for row in rows:
+            user, contact = row
+            yield contact
+
+    @contacts.setter
+    def contacts(self, puppets: Iterable[TelegramID]) -> None:
+        with self.db.begin() as conn:
+            conn.execute(Contact.t.delete().where(Contact.c.user == self.tgid))
+            insert_puppets = [{"user": self.tgid, "contact": tgid} for tgid in puppets]
+            if insert_puppets:
+                conn.execute(Contact.t.insert(), insert_puppets)
+
+    @property
+    def portals(self) -> Iterable[Tuple[TelegramID, TelegramID]]:
+        rows = self.db.execute(UserPortal.t.select().where(UserPortal.c.user == self.tgid))
+        for row in rows:
+            user, portal, portal_receiver = row
+            yield (portal, portal_receiver)
+
+    @portals.setter
+    def portals(self, portals: Iterable[Tuple[TelegramID, TelegramID]]) -> None:
+        with self.db.begin() as conn:
+            conn.execute(UserPortal.t.delete().where(UserPortal.c.user == self.tgid))
+            insert_portals = [{
+                "user": self.tgid,
+                "portal": tgid,
+                "portal_receiver": tg_receiver
+            } for tgid, tg_receiver in portals]
+            if insert_portals:
+                conn.execute(UserPortal.t.insert(), insert_portals)
+
+    def delete(self) -> None:
+        super().delete()
+        self.portals = []
+        self.contacts = []
+
+
+class UserPortal(Base):
+    __tablename__ = "user_portal"
+
+    user = Column(Integer, ForeignKey("user.tgid", onupdate="CASCADE", ondelete="CASCADE"),
+                  primary_key=True)  # type: TelegramID
+    portal = Column(Integer, primary_key=True)  # type: TelegramID
+    portal_receiver = Column(Integer, primary_key=True)  # type: TelegramID
+
+    __table_args__ = (ForeignKeyConstraint(("portal", "portal_receiver"),
+                                           ("portal.tgid", "portal.tg_receiver"),
+                                           onupdate="CASCADE", ondelete="CASCADE"),)
+
+
+class Contact(Base):
+    __tablename__ = "contact"
+
+    user = Column(Integer, ForeignKey("user.tgid"), primary_key=True)  # type: TelegramID
+    contact = Column(Integer, ForeignKey("puppet.id"), primary_key=True)  # type: TelegramID

mautrix_telegram/db/user_profile.py

@@ -0,0 +1,69 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from sqlalchemy import Column, String, and_
+from typing import Dict, Optional
+
+from ..types import MatrixUserID, MatrixRoomID
+from .base import Base
+
+
+class UserProfile(Base):
+    __tablename__ = "mx_user_profile"
+
+    room_id = Column(String, primary_key=True)  # type: MatrixRoomID
+    user_id = Column(String, primary_key=True)  # type: MatrixUserID
+    membership = Column(String, nullable=False, default="leave")
+    displayname = Column(String, nullable=True)
+    avatar_url = Column(String, nullable=True)
+
+    def dict(self) -> Dict[str, str]:
+        return {
+            "membership": self.membership,
+            "displayname": self.displayname,
+            "avatar_url": self.avatar_url,
+        }
+
+    @classmethod
+    def get(cls, room_id: MatrixRoomID, user_id: MatrixUserID) -> Optional['UserProfile']:
+        rows = cls.db.execute(
+            cls.t.select().where(and_(cls.c.room_id == room_id, cls.c.user_id == user_id)))
+        try:
+            room_id, user_id, membership, displayname, avatar_url = next(rows)
+            return cls(room_id=room_id, user_id=user_id, membership=membership,
+                       displayname=displayname, avatar_url=avatar_url)
+        except StopIteration:
+            return None
+
+    @classmethod
+    def delete_all(cls, room_id: MatrixRoomID) -> None:
+        with cls.db.begin() as conn:
+            conn.execute(cls.t.delete().where(cls.c.room_id == room_id))
+
+    def update(self) -> None:
+        super().update(membership=self.membership, displayname=self.displayname,
+                       avatar_url=self.avatar_url)
+
+    @property
+    def _edit_identity(self):
+        return and_(self.c.room_id == self.room_id, self.c.user_id == self.user_id)
+
+    def insert(self) -> None:
+        with self.db.begin() as conn:
+            conn.execute(self.t.insert().values(room_id=self.room_id, user_id=self.user_id,
+                                                membership=self.membership,
+                                                displayname=self.displayname,
+                                                avatar_url=self.avatar_url))

mautrix_telegram/formatter/__init__.py

@@ -1,9 +1,8 @@
 from .from_matrix import (matrix_reply_to_telegram, matrix_to_telegram, matrix_text_to_telegram,
                           init_mx)
-from .from_telegram import (telegram_reply_to_matrix, telegram_to_matrix, init_tg)
+from .from_telegram import telegram_reply_to_matrix, telegram_to_matrix
 from .. import context as c
 
 
 def init(context: c.Context) -> None:
     init_mx(context)
-    init_tg(context)

mautrix_telegram/formatter/from_matrix/__init__.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -26,12 +26,7 @@ from ...types import TelegramID, MatrixRoomID
 from ...db import Message as DBMessage
 from ..util import (add_surrogates, remove_surrogates, trim_reply_fallback_html,
                     trim_reply_fallback_text)
-from .parser_common import ParsedMessage
-
-try:
-    from mautrix_telegram.formatter.from_matrix.parser_lxml import parse_html
-except ImportError:
-    from mautrix_telegram.formatter.from_matrix.parser_htmlparser import parse_html
+from .parser import ParsedMessage, parse_html
 
 if TYPE_CHECKING:
     from ...context import Context
@@ -41,7 +36,7 @@ should_bridge_plaintext_highlights = False  # type: bool
 
 command_regex = re.compile(r"^!([A-Za-z0-9@]+)")  # type: Pattern
 not_command_regex = re.compile(r"^\\(![A-Za-z0-9@]+)")  # type: Pattern
-plain_mention_regex = None  # type: Pattern
+plain_mention_regex = None  # type: Optional[Pattern]
 
 
 def plain_mention_to_html(match: Match) -> str:
@@ -81,7 +76,6 @@ def matrix_to_telegram(html: str) -> ParsedMessage:
         if should_bridge_plaintext_highlights:
             html = plain_mention_regex.sub(plain_mention_to_html, html)
 
-        html = add_surrogates(html)
         text, entities = parse_html(add_surrogates(html))
         text = remove_surrogates(text.strip())
         text, entities = cut_long_message(text, entities)
@@ -93,23 +87,28 @@ def matrix_to_telegram(html: str) -> ParsedMessage:
 
 def matrix_reply_to_telegram(content: Dict[str, Any], tg_space: TelegramID,
                              room_id: Optional[MatrixRoomID] = None) -> Optional[TelegramID]:
+    relates_to = content.get("m.relates_to", None) or {}
+    if not relates_to:
+        return None
+    reply = (relates_to if relates_to.get("rel_type", None) == "m.reference"
+             else relates_to.get("m.in_reply_to", None) or {})
+    if not reply:
+        return None
+    room_id = room_id or reply.get("room_id", None)
+    event_id = reply.get("event_id", None)
+    if not event_id:
+        return
+
     try:
-        reply = content["m.relates_to"]["m.in_reply_to"]
-        room_id = room_id or reply["room_id"]
-        event_id = reply["event_id"]
-
-        try:
-            if content["format"] == "org.matrix.custom.html":
-                content["formatted_body"] = trim_reply_fallback_html(content["formatted_body"])
-        except KeyError:
-            pass
-        content["body"] = trim_reply_fallback_text(content["body"])
-
-        message = DBMessage.get_by_mxid(event_id, room_id, tg_space)
-        if message:
-            return message.tgid
+        if content["format"] == "org.matrix.custom.html":
+            content["formatted_body"] = trim_reply_fallback_html(content["formatted_body"])
     except KeyError:
         pass
+    content["body"] = trim_reply_fallback_text(content["body"])
+
+    message = DBMessage.get_by_mxid(event_id, room_id, tg_space)
+    if message:
+        return message.tgid
     return None
 
 
@@ -151,5 +150,5 @@ def init_mx(context: "Context") -> None:
     config = context.config
     dn_template = config.get("bridge.displayname_template", "{displayname} (Telegram)")
     dn_template = re.escape(dn_template).replace(re.escape("{displayname}"), "[^>]+")
-    plain_mention_regex = re.compile(f"(\s|^)({dn_template})")
+    plain_mention_regex = re.compile(f"^({dn_template})")
     should_bridge_plaintext_highlights = config["bridge.plaintext_highlights"] or False

mautrix_telegram/formatter/from_matrix/html_reader.py

@@ -0,0 +1,66 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Dict, List, Tuple
+
+from html.parser import HTMLParser
+
+
+class HTMLNode(list):
+    def __init__(self, tag: str, attrs: List[Tuple[str, str]]):
+        super().__init__()
+        self.tag = tag  # type: str
+        self.text = ""  # type: str
+        self.tail = ""  # type: str
+        self.attrib = dict(attrs)  # type: Dict[str, str]
+
+
+class NodeifyingParser(HTMLParser):
+    # From https://www.w3.org/TR/html5/syntax.html#writing-html-documents-elements
+    void_tags = ("area", "base", "br", "col", "command", "embed", "hr", "img", "input", "link",
+                 "meta", "param", "source", "track", "wbr")
+
+    def __init__(self):
+        super().__init__()
+        self.stack = [HTMLNode("html", [])]  # type: List[HTMLNode]
+
+    def handle_starttag(self, tag, attrs):
+        node = HTMLNode(tag, attrs)
+        self.stack[-1].append(node)
+        if tag not in self.void_tags:
+            self.stack.append(node)
+
+    def handle_startendtag(self, tag, attrs):
+        self.stack[-1].append(HTMLNode(tag, attrs))
+
+    def handle_endtag(self, tag):
+        if tag == self.stack[-1].tag:
+            self.stack.pop()
+
+    def handle_data(self, data):
+        if len(self.stack[-1]) > 0:
+            self.stack[-1][-1].tail += data
+        else:
+            self.stack[-1].text += data
+
+    def error(self, message):
+        pass
+
+
+def read_html(data: str) -> HTMLNode:
+    parser = NodeifyingParser()
+    parser.feed(data)
+    return parser.stack[0]

mautrix_telegram/formatter/from_matrix/html_reader.pyi

@@ -0,0 +1,11 @@
+from typing import Dict, List
+
+
+class HTMLNode(List['HTMLNode']):
+    tag: str
+    text: str
+    tail: str
+    attrib: Dict[str, str]
+
+
+def read_html(data: str) -> HTMLNode: ...

mautrix_telegram/formatter/from_matrix/parser.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -14,21 +14,25 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import List, Tuple
-from lxml import html
+from typing import List, Tuple, Pattern
+import re
 
 from telethon.tl.types import (MessageEntityMention as Mention, MessageEntityBotCommand as Command,
                                MessageEntityMentionName as MentionName, MessageEntityEmail as Email,
                                MessageEntityUrl as URL, MessageEntityTextUrl as TextURL,
                                MessageEntityBold as Bold, MessageEntityItalic as Italic,
-                               MessageEntityCode as Code, MessageEntityPre as Pre)
+                               MessageEntityCode as Code, MessageEntityPre as Pre,
+                               TypeMessageEntity)
 
 from ... import user as u, puppet as pu, portal as po
 from ...types import MatrixUserID
 from ..util import html_to_unicode
-from .parser_common import MatrixParserCommon, ParsedMessage
 from .telegram_message import TelegramMessage, Entity, offset_length_multiply
 
+from .html_reader import HTMLNode, read_html
+
+ParsedMessage = Tuple[str, List[TypeMessageEntity]]
+
 
 def parse_html(input_html: str) -> ParsedMessage:
     return MatrixParser.parse(input_html)
@@ -52,9 +56,21 @@ class RecursionContext:
         return RecursionContext(strip_linebreaks=False, ul_depth=self.ul_depth)
 
 
-class MatrixParser(MatrixParserCommon):
+class MatrixParser:
+    mention_regex = re.compile("https://matrix.to/#/(@.+:.+)")  # type: Pattern
+    room_regex = re.compile("https://matrix.to/#/(#.+:.+)")  # type: Pattern
+    block_tags = ("p", "pre", "blockquote",
+                  "ol", "ul", "li",
+                  "h1", "h2", "h3", "h4", "h5", "h6",
+                  "div", "hr", "table")  # type: Tuple[str, ...]
+    list_bullets = ("●", "○", "■", "‣")  # type: Tuple[str, ...]
+
     @classmethod
-    def list_to_tmessage(cls, node: html.HtmlElement, ctx: RecursionContext) -> TelegramMessage:
+    def list_bullet(cls, depth: int) -> str:
+        return cls.list_bullets[(depth - 1) % len(cls.list_bullets)] + " "
+
+    @classmethod
+    def list_to_tmessage(cls, node: HTMLNode, ctx: RecursionContext) -> TelegramMessage:
         ordered = node.tag == "ol"
         tagged_children = cls.node_to_tagged_tmessages(node, ctx)
         counter = 1
@@ -86,23 +102,21 @@ class MatrixParser(MatrixParserCommon):
         return TelegramMessage.join(children, "\n")
 
     @classmethod
-    def blockquote_to_tmessage(cls, node: html.HtmlElement, ctx: RecursionContext
-                               ) -> TelegramMessage:
+    def blockquote_to_tmessage(cls, node: HTMLNode, ctx: RecursionContext) -> TelegramMessage:
         msg = cls.tag_aware_parse_node(node, ctx)
         children = msg.trim().split("\n")
         children = [child.prepend("> ") for child in children]
         return TelegramMessage.join(children, "\n")
 
     @classmethod
-    def header_to_tmessage(cls, node: html.HtmlElement, ctx: RecursionContext) -> TelegramMessage:
+    def header_to_tmessage(cls, node: HTMLNode, ctx: RecursionContext) -> TelegramMessage:
         children = cls.node_to_tmessages(node, ctx)
         length = int(node.tag[1])
         prefix = "#" * length + " "
         return TelegramMessage.join(children, "").prepend(prefix).format(Bold)
 
     @classmethod
-    def basic_format_to_tmessage(cls, node: html.HtmlElement, ctx: RecursionContext
-                                 ) -> TelegramMessage:
+    def basic_format_to_tmessage(cls, node: HTMLNode, ctx: RecursionContext) -> TelegramMessage:
         msg = cls.tag_aware_parse_node(node, ctx)
         if node.tag in ("b", "strong"):
             msg.format(Bold)
@@ -121,7 +135,7 @@ class MatrixParser(MatrixParserCommon):
         return msg
 
     @classmethod
-    def link_to_tstring(cls, node: html.HtmlElement, ctx: RecursionContext) -> TelegramMessage:
+    def link_to_tstring(cls, node: HTMLNode, ctx: RecursionContext) -> TelegramMessage:
         msg = cls.tag_aware_parse_node(node, ctx)
         href = node.attrib.get("href", "")
         if not href:
@@ -140,8 +154,8 @@ class MatrixParser(MatrixParserCommon):
             if user.username:
                 return TelegramMessage(f"@{user.username}").format(Mention)
             elif user.tgid:
-                return TelegramMessage(user.displayname or msg.text).format(MentionName,
-                                                                            user_id=user.tgid)
+                displayname = user.plain_displayname or msg.text
+                return TelegramMessage(displayname).format(MentionName, user_id=user.tgid)
             return msg
 
         room = cls.room_regex.match(href)
@@ -156,7 +170,7 @@ class MatrixParser(MatrixParserCommon):
                 else msg.format(TextURL, url=href))
 
     @classmethod
-    def node_to_tmessage(cls, node: html.HtmlElement, ctx: RecursionContext) -> TelegramMessage:
+    def node_to_tmessage(cls, node: HTMLNode, ctx: RecursionContext) -> TelegramMessage:
         if node.tag == "blockquote":
             return cls.blockquote_to_tmessage(node, ctx)
         elif node.tag == "ol":
@@ -193,7 +207,7 @@ class MatrixParser(MatrixParserCommon):
         return TelegramMessage(text)
 
     @classmethod
-    def node_to_tagged_tmessages(cls, node: html.HtmlElement, ctx: RecursionContext
+    def node_to_tagged_tmessages(cls, node: HTMLNode, ctx: RecursionContext
                                  ) -> List[Tuple[TelegramMessage, str]]:
         output = []
 
@@ -206,12 +220,12 @@ class MatrixParser(MatrixParserCommon):
         return output
 
     @classmethod
-    def node_to_tmessages(cls, node: html.HtmlElement, ctx: RecursionContext
+    def node_to_tmessages(cls, node: HTMLNode, ctx: RecursionContext
                           ) -> List[TelegramMessage]:
         return [msg for (msg, tag) in cls.node_to_tagged_tmessages(node, ctx)]
 
     @classmethod
-    def tag_aware_parse_node(cls, node: html.HtmlElement, ctx: RecursionContext
+    def tag_aware_parse_node(cls, node: HTMLNode, ctx: RecursionContext
                              ) -> TelegramMessage:
         msgs = cls.node_to_tagged_tmessages(node, ctx)
         output = TelegramMessage()
@@ -226,11 +240,10 @@ class MatrixParser(MatrixParserCommon):
         return output.trim()
 
     @classmethod
-    def parse_node(cls, node: html.HtmlElement, ctx: RecursionContext) -> TelegramMessage:
+    def parse_node(cls, node: HTMLNode, ctx: RecursionContext) -> TelegramMessage:
         return TelegramMessage.join(cls.node_to_tmessages(node, ctx))
 
     @classmethod
     def parse(cls, data: str) -> ParsedMessage:
-        document = html.fromstring(f"<html>{data}</html>")
-        msg = cls.parse_node(document, RecursionContext())
+        msg = cls.node_to_tmessage(read_html(f"<body>{data}</body>"), RecursionContext())
         return msg.text, msg.entities

mautrix_telegram/formatter/from_matrix/parser_common.py

@@ -1,36 +0,0 @@
-# -*- coding: future_fstrings -*-
-# mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
-import re
-from typing import List, Tuple, Pattern
-from telethon.tl.types import TypeMessageEntity
-
-
-class MatrixParserCommon:
-    mention_regex = re.compile("https://matrix.to/#/(@.+:.+)")  # type: Pattern
-    room_regex = re.compile("https://matrix.to/#/(#.+:.+)")  # type: Pattern
-    block_tags = ("p", "pre", "blockquote",
-                  "ol", "ul", "li",
-                  "h1", "h2", "h3", "h4", "h5", "h6",
-                  "div", "hr", "table")  # type: Tuple[str, ...]
-    list_bullets = ("●", "○", "■", "‣")  # type: Tuple[str, ...]
-
-    @classmethod
-    def list_bullet(cls, depth: int) -> str:
-        return cls.list_bullets[(depth - 1) % len(cls.list_bullets)] + " "
-
-
-ParsedMessage = Tuple[str, List[TypeMessageEntity]]

mautrix_telegram/formatter/from_matrix/parser_htmlparser.py

@@ -1,241 +0,0 @@
-# -*- coding: future_fstrings -*-
-# mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import (Optional, List, Tuple, Type, Dict, Any, TYPE_CHECKING, Match)
-from html import unescape
-from html.parser import HTMLParser
-from collections import deque
-import math
-
-from telethon.tl.types import (MessageEntityMention, MessageEntityMentionName, MessageEntityEmail,
-                               MessageEntityUrl, MessageEntityTextUrl, MessageEntityBold,
-                               MessageEntityItalic, MessageEntityCode, MessageEntityPre,
-                               MessageEntityBotCommand, TypeMessageEntity)
-
-from ... import user as u, puppet as pu, portal as po
-from ...types import MatrixUserID
-from ..util import html_to_unicode
-from .parser_common import MatrixParserCommon, ParsedMessage
-
-if TYPE_CHECKING:
-    from typing import Deque
-
-
-def parse_html(html: str) -> ParsedMessage:
-    parser = MatrixParser()
-    parser.feed(html)
-    return parser.text, parser.entities
-
-
-class MatrixParser(HTMLParser, MatrixParserCommon):
-    def __init__(self):
-        super(MatrixParser, self).__init__()
-        self.text = ""  # type: str
-        self.entities = []  # type: List[TypeMessageEntity]
-        self._building_entities = {}  # type: Dict[str, TypeMessageEntity]
-        self._list_counter = 0  # type: int
-        self._open_tags = deque()  # type: Deque[str]
-        self._open_tags_meta = deque()  # type: Deque[Any]
-        self._line_is_new = True  # type: bool
-        self._list_entry_is_new = False  # type: bool
-
-    def _parse_url(self, url: str, args: Dict[str, Any]
-                   ) -> Tuple[Optional[Type[TypeMessageEntity]], Optional[str]]:
-        mention = self.mention_regex.match(url)  # type: Match
-        if mention:
-            mxid = MatrixUserID(mention.group(1))
-            user = (pu.Puppet.get_by_mxid(mxid)
-                    or u.User.get_by_mxid(mxid, create=False))
-            if not user:
-                return None, None
-            if user.username:
-                return MessageEntityMention, f"@{user.username}"
-            elif user.tgid:
-                args["user_id"] = user.tgid
-                return MessageEntityMentionName, user.displayname or None
-            else:
-                return None, None
-
-        room = self.room_regex.match(url)  # type: Match
-        if room:
-            username = po.Portal.get_username_from_mx_alias(room.group(1))
-            portal = po.Portal.find_by_username(username)
-            if portal and portal.username:
-                return MessageEntityMention, f"@{portal.username}"
-
-        if url.startswith("mailto:"):
-            return MessageEntityEmail, url[len("mailto:"):]
-        elif self.get_starttag_text() == url:
-            return MessageEntityUrl, url
-        else:
-            args["url"] = url
-            return MessageEntityTextUrl, None
-
-    def handle_starttag(self, tag: str, attrs_list: List[Tuple[str, str]]):
-        self._open_tags.appendleft(tag)
-        self._open_tags_meta.appendleft(0)
-
-        attrs = dict(attrs_list)
-        entity_type = None  # type: Optional[Type[TypeMessageEntity]]
-        args = {}  # type: Dict[str, Any]
-        if tag in ("strong", "b"):
-            entity_type = MessageEntityBold
-        elif tag in ("em", "i"):
-            entity_type = MessageEntityItalic
-        elif tag == "code":
-            try:
-                pre = self._building_entities["pre"]
-                try:
-                    # Pre tag and language found, add language to MessageEntityPre
-                    pre.language = attrs["class"][len("language-"):]
-                except KeyError:
-                    # Pre tag found, but language not found, keep pre as-is
-                    pass
-            except KeyError:
-                # No pre tag found, this is inline code
-                entity_type = MessageEntityCode
-        elif tag == "pre":
-            entity_type = MessageEntityPre
-            args["language"] = ""
-        elif tag == "command":
-            entity_type = MessageEntityBotCommand
-        elif tag == "li":
-            self._list_entry_is_new = True
-        elif tag == "a":
-            try:
-                url = attrs["href"]
-            except KeyError:
-                return
-            entity_type, url = self._parse_url(url, args)
-            self._open_tags_meta.popleft()
-            self._open_tags_meta.appendleft(url)
-
-        if (tag in self.block_tags and ("blockquote" not in self._open_tags)) or tag == "br":
-            self._newline()
-
-        if entity_type and tag not in self._building_entities:
-            offset = len(self.text)
-            self._building_entities[tag] = entity_type(offset=offset, length=0, **args)
-
-    @property
-    def _list_indent(self) -> int:
-        indent = 0
-        first_skipped = False
-        for index, tag in enumerate(self._open_tags):
-            if not first_skipped and tag in ("ol", "ul"):
-                # The first list level isn't indented, so skip it.
-                first_skipped = True
-                continue
-            if tag == "ol":
-                n = self._open_tags_meta[index]
-                extra_length_for_long_index = (int(math.log(n, 10)) - 1) * 3
-                indent += 4 + extra_length_for_long_index
-            elif tag == "ul":
-                indent += 3
-        return indent
-
-    def _newline(self, allow_multi: bool = False):
-        if self._line_is_new and not allow_multi:
-            return
-        self.text += "\n"
-        self._line_is_new = True
-        for entity in self._building_entities.values():
-            entity.length += 1
-
-    def _handle_special_previous_tags(self, text: str) -> str:
-        if "pre" not in self._open_tags and "code" not in self._open_tags:
-            text = text.replace("\n", "")
-        else:
-            text = text.strip()
-
-        previous_tag = self._open_tags[0] if len(self._open_tags) > 0 else ""
-        if previous_tag == "a":
-            url = self._open_tags_meta[0]
-            if url:
-                text = url
-        elif previous_tag == "command":
-            text = f"/{text}"
-        return text
-
-    def _html_to_unicode(self, text: str) -> str:
-        strikethrough, underline = "del" in self._open_tags, "u" in self._open_tags
-        if strikethrough and underline:
-            text = html_to_unicode(text, "\u0336\u0332")
-        elif strikethrough:
-            text = html_to_unicode(text, "\u0336")
-        elif underline:
-            text = html_to_unicode(text, "\u0332")
-        return text
-
-    def _handle_tags_for_data(self, text: str) -> Tuple[str, int]:
-        extra_offset = 0
-        list_entry_handled_once = False
-        # In order to maintain order of things like blockquotes in lists or lists in blockquotes,
-        # we can't just have ifs/elses and we need to actually loop through the open tags in order.
-        for index, tag in enumerate(self._open_tags):
-            if tag == "blockquote" and self._line_is_new:
-                text = f"> {text}"
-                extra_offset += 2
-            elif tag == "li" and not list_entry_handled_once:
-                list_type_index = index + 1
-                list_type = self._open_tags[list_type_index]
-                indent = self._list_indent * " " if self._line_is_new else ""
-                if list_type == "ol":
-                    n = self._open_tags_meta[list_type_index]
-                    if self._list_entry_is_new:
-                        n += 1
-                        self._open_tags_meta[list_type_index] = n
-                        prefix = f"{n}. "
-                    else:
-                        prefix = int(math.log(n, 10)) * 3 * " " + 4 * " "
-                else:
-                    prefix = (self.list_bullet(self._open_tags.count('ul'))
-                              if self._list_entry_is_new else 3 * " ")
-                if not self._list_entry_is_new and not self._line_is_new:
-                    prefix = ""
-                extra_offset += len(indent) + len(prefix)
-                text = indent + prefix + text
-                self._list_entry_is_new = False
-                list_entry_handled_once = True
-        return text, extra_offset
-
-    def _extend_entities_in_construction(self, text: str, extra_offset: int):
-        for tag, entity in self._building_entities.items():
-            entity.length += len(text) - extra_offset
-            entity.offset += extra_offset
-
-    def handle_data(self, text: str):
-        text = unescape(text)
-        text = self._handle_special_previous_tags(text)
-        text = self._html_to_unicode(text)
-        text, extra_offset = self._handle_tags_for_data(text)
-        self._extend_entities_in_construction(text, extra_offset)
-        self._line_is_new = False
-        self.text += text
-
-    def handle_endtag(self, tag: str):
-        try:
-            self._open_tags.popleft()
-            self._open_tags_meta.popleft()
-        except IndexError:
-            pass
-
-        entity = self._building_entities.pop(tag, None)
-        if entity:
-            self.entities.append(entity)
-
-        if tag in self.block_tags and tag != "br" and "blockquote" not in self._open_tags:
-            self._newline(allow_multi=tag == "br")

mautrix_telegram/formatter/from_matrix/telegram_message.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -153,5 +153,6 @@ class TelegramMessage:
                 msg = TelegramMessage(text=msg)
             main.entities += Entity.adjust(msg.entities, offset_diff(len(main.text)))
             main.text += msg.text + separator
-        main.text = main.text[:-len(separator)]
+        if len(separator) > 0:
+            main.text = main.text[:-len(separator)]
         return main

mautrix_telegram/formatter/from_telegram.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -37,16 +37,8 @@ from .util import (add_surrogates, remove_surrogates, trim_reply_fallback_html,
 
 if TYPE_CHECKING:
     from ..abstract_user import AbstractUser
-    from ..context import Context
-
-try:
-    from lxml.html.diff import htmldiff
-except ImportError:
-    htmldiff = None  # type: ignore
-
 
 log = logging.getLogger("mau.fmt.tg")  # type: logging.Logger
-should_highlight_edits = False  # type: bool
 
 
 def telegram_reply_to_matrix(evt: Message, source: 'AbstractUser') -> Dict:
@@ -54,13 +46,16 @@ def telegram_reply_to_matrix(evt: Message, source: 'AbstractUser') -> Dict:
         space = (evt.to_id.channel_id
                  if isinstance(evt, Message) and isinstance(evt.to_id, PeerChannel)
                  else source.tgid)
-        msg = DBMessage.get_by_tgid(evt.reply_to_msg_id, space)
+        msg = DBMessage.get_one_by_tgid(evt.reply_to_msg_id, space)
         if msg:
             return {
                 "m.in_reply_to": {
                     "event_id": msg.mxid,
                     "room_id": msg.mx_room,
-                }
+                },
+                "rel_type": "m.reference",
+                "event_id": msg.mxid,
+                "room_id": msg.mx_room,
             }
     return {}
 
@@ -71,21 +66,37 @@ async def _add_forward_header(source, text: str, html: Optional[str],
         html = escape(text)
     fwd_from_html, fwd_from_text = None, None
     if fwd_from.from_id:
-        user = u.User.get_by_tgid(fwd_from.from_id)
+        user = u.User.get_by_tgid(TelegramID(fwd_from.from_id))
         if user:
             fwd_from_text = user.displayname or user.mxid
-            fwd_from_html = f"<a href='https://matrix.to/#/{user.mxid}'>{fwd_from_text}</a>"
+            fwd_from_html = (f"<a href='https://matrix.to/#/{user.mxid}'>"
+                             f"{escape(fwd_from_text)}</a>")
 
         if not fwd_from_text:
             puppet = pu.Puppet.get(TelegramID(fwd_from.from_id), create=False)
             if puppet and puppet.displayname:
                 fwd_from_text = puppet.displayname or puppet.mxid
-                fwd_from_html = f"<a href='https://matrix.to/#/{puppet.mxid}'>{fwd_from_text}</a>"
+                fwd_from_html = (f"<a href='https://matrix.to/#/{puppet.mxid}'>"
+                                 f"{escape(fwd_from_text)}</a>")
 
         if not fwd_from_text:
             user = await source.client.get_entity(PeerUser(fwd_from.from_id))
             if user:
                 fwd_from_text = pu.Puppet.get_displayname(user, False)
+                fwd_from_html = f"<b>{escape(fwd_from_text)}</b>"
+    else:
+        portal = po.Portal.get_by_tgid(TelegramID(fwd_from.channel_id))
+        if portal:
+            fwd_from_text = portal.title
+            if portal.alias:
+                fwd_from_html = (f"<a href='https://matrix.to/#/{portal.alias}'>"
+                                 f"{escape(fwd_from_text)}</a>")
+            else:
+                fwd_from_html = f"<b>{escape(fwd_from_text)}</b>"
+        else:
+            channel = await source.client.get_entity(PeerChannel(fwd_from.channel_id))
+            if channel:
+                fwd_from_text = channel.title
                 fwd_from_html = f"<b>{fwd_from_text}</b>"
 
     if not fwd_from_text:
@@ -102,32 +113,19 @@ async def _add_forward_header(source, text: str, html: Optional[str],
     return text, html
 
 
-def highlight_edits(new_html: str, old_html: str) -> str:
-    # Don't include `Edit:` text in diff.
-    if old_html.startswith("<u>Edit:</u> "):
-        old_html = old_html[len("<u>Edit:</u> "):]
-
-    # Generate diff with lxml
-    new_html = htmldiff(old_html, new_html)
-
-    # Replace <ins> with <u> since Riot doesn't allow <ins>
-    new_html = new_html.replace("<ins>", "<u>").replace("</ins>", "</u>")
-    # Remove <del>s since we just want to hide deletions.
-    new_html = re.sub("<del>.+?</del>", "", new_html)
-    return new_html
-
-
 async def _add_reply_header(source: "AbstractUser", text: str, html: str, evt: Message,
-                            relates_to: Dict, main_intent: IntentAPI, is_edit: bool
-                            ) -> Tuple[str, str]:
+                            relates_to: Dict, main_intent: IntentAPI) -> Tuple[str, str]:
     space = (evt.to_id.channel_id
              if isinstance(evt, Message) and isinstance(evt.to_id, PeerChannel)
              else source.tgid)
 
-    msg = DBMessage.get_by_tgid(evt.reply_to_msg_id, space)
+    msg = DBMessage.get_one_by_tgid(evt.reply_to_msg_id, space)
     if not msg:
         return text, html
 
+    relates_to["rel_type"] = "m.reference"
+    relates_to["event_id"] = msg.mxid
+    relates_to["room_id"] = msg.mx_room
     relates_to["m.in_reply_to"] = {
         "event_id": msg.mxid,
         "room_id": msg.mx_room,
@@ -146,22 +144,14 @@ async def _add_reply_header(source: "AbstractUser", text: str, html: str, evt: M
 
         puppet = pu.Puppet.get_by_mxid(r_sender, create=False)
         r_displayname = puppet.displayname if puppet else r_sender
-        r_sender_link = f"<a href='https://matrix.to/#/{r_sender}'>{r_displayname}</a>"
-
-        if is_edit and should_highlight_edits:
-            html = highlight_edits(html or escape(text), r_html_body)
+        r_sender_link = f"<a href='https://matrix.to/#/{r_sender}'>{escape(r_displayname)}</a>"
     except (ValueError, KeyError, MatrixRequestError):
         r_sender_link = "unknown user"
         r_displayname = "unknown user"
         r_text_body = "Failed to fetch message"
         r_html_body = "<em>Failed to fetch message</em>"
 
-    if is_edit:
-        html = f"<u>Edit:</u> {html or escape(text)}"
-        text = f"Edit: {text}"
-
-    r_keyword = "In reply to" if not is_edit else "Edit to"
-    r_msg_link = f"<a href='https://matrix.to/#/{msg.mx_room}/{msg.mxid}'>{r_keyword}</a>"
+    r_msg_link = f"<a href='https://matrix.to/#/{msg.mx_room}/{msg.mxid}'>In reply to</a>"
     html = (
         f"<mx-reply><blockquote>{r_msg_link} {r_sender_link}\n{r_html_body}</blockquote></mx-reply>"
         + (html or escape(text)))
@@ -178,10 +168,13 @@ async def _add_reply_header(source: "AbstractUser", text: str, html: str, evt: M
 
 async def telegram_to_matrix(evt: Message, source: "AbstractUser",
                              main_intent: Optional[IntentAPI] = None,
-                             is_edit: bool = False, prefix_text: Optional[str] = None,
-                             prefix_html: Optional[str] = None) -> Tuple[str, str, Dict]:
-    text = add_surrogates(evt.message)
-    html = _telegram_entities_to_matrix_catch(text, evt.entities) if evt.entities else None
+                             prefix_text: Optional[str] = None, prefix_html: Optional[str] = None,
+                             override_text: str = None,
+                             override_entities: List[TypeMessageEntity] = None,
+                             no_reply_fallback: bool = False) -> Tuple[str, str, Dict]:
+    text = add_surrogates(override_text or evt.message)
+    entities = override_entities or evt.entities
+    html = _telegram_entities_to_matrix_catch(text, entities) if entities else None
     relates_to = {}  # type: Dict
 
     if prefix_html:
@@ -192,9 +185,8 @@ async def telegram_to_matrix(evt: Message, source: "AbstractUser",
     if evt.fwd_from:
         text, html = await _add_forward_header(source, text, html, evt.fwd_from)
 
-    if evt.reply_to_msg_id:
-        text, html = await _add_reply_header(source, text, html, evt, relates_to, main_intent,
-                                             is_edit)
+    if evt.reply_to_msg_id and not no_reply_fallback:
+        text, html = await _add_reply_header(source, text, html, evt, relates_to, main_intent)
 
     if isinstance(evt, Message) and evt.post and evt.post_author:
         if not html:
@@ -242,9 +234,9 @@ def _telegram_entities_to_matrix(text: str, entities: List[TypeMessageEntity]) -
         elif entity_type == MessageEntityItalic:
             html.append(f"<em>{entity_text}</em>")
         elif entity_type == MessageEntityCode:
-            html.append(f"<pre><code>{entity_text}</code></pre>"
-                        if "\n" in entity_text
-                        else f"<code>{entity_text}</code>")
+            html.append(("<pre><code>{entity_text}</code></pre>"
+                         if "\n" in entity_text
+                         else "<code>{entity_text}</code>").format(entity_text=entity_text))
         elif entity_type == MessageEntityPre:
             skip_entity = _parse_pre(html, entity_text, entity.language)
         elif entity_type == MessageEntityMention:
@@ -325,14 +317,9 @@ def _parse_url(html: List[str], entity_text: str, url: str) -> bool:
 
         portal = po.Portal.find_by_username(group)
         if portal:
-            message = DBMessage.get_by_tgid(TelegramID(msgid), portal.tgid)
+            message = DBMessage.get_one_by_tgid(TelegramID(msgid), portal.tgid)
             if message:
                 url = f"https://matrix.to/#/{portal.mxid}/{message.mxid}"
 
     html.append(f"<a href='{url}'>{entity_text}</a>")
     return False
-
-
-def init_tg(context: "Context") -> None:
-    global should_highlight_edits
-    should_highlight_edits = htmldiff and context.config["bridge.highlight_edits"]

mautrix_telegram/formatter/util.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by

mautrix_telegram/matrix.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -17,6 +17,7 @@
 from typing import Dict, List, Match, Optional, Set, Tuple, TYPE_CHECKING
 import logging
 import asyncio
+import time
 import re
 
 from mautrix_appservice import MatrixRequestError, IntentError
@@ -27,12 +28,21 @@ from . import user as u, portal as po, puppet as pu, commands as com
 if TYPE_CHECKING:
     from .context import Context
 
+try:
+    from prometheus_client import Histogram
+
+    EVENT_TIME = Histogram("matrix_event", "Time spent processing Matrix events",
+                           ["event_type"])
+except ImportError:
+    Histogram = None
+    EVENT_TIME = None
+
 
 class MatrixHandler:
     log = logging.getLogger("mau.mx")  # type: logging.Logger
 
     def __init__(self, context: 'Context') -> None:
-        self.az, self.db, self.config, _, self.tgbot = context.core
+        self.az, self.config, _, self.tgbot = context.core
         self.commands = com.CommandProcessor(context)  # type: com.CommandProcessor
         self.previously_typing = []  # type: List[MatrixUserID]
 
@@ -126,12 +136,31 @@ class MatrixHandler:
 
         if not inviter.whitelisted:
             await self.az.intent.send_notice(
-                room_id, text="",
-                html="You are not whitelisted to use this bridge.<br/><br/>"
+                room_id,
+                text="You are not whitelisted to use this bridge.\n\n"
                      "If you are the owner of this bridge, see the "
-                     "<code>bridge.permissions</code> section in your config file.")
+                     "`bridge.permissions` section in your config file.",
+                html="<p>You are not whitelisted to use this bridge.</p>"
+                     "<p>If you are the owner of this bridge, see the "
+                     "<code>bridge.permissions</code> section in your config file.</p>")
             await self.az.intent.leave_room(room_id)
 
+        try:
+            is_management = len(await self.az.intent.get_room_members(room_id)) == 2
+        except MatrixRequestError:
+            is_management = False
+        cmd_prefix = self.commands.command_prefix
+        text = html = "Hello, I'm a Telegram bridge bot. "
+        if is_management and inviter.puppet_whitelisted and not await inviter.is_logged_in():
+            text += f"Use `{cmd_prefix} help` for help or `{cmd_prefix} login` to log in."
+            html += (f"Use <code>{cmd_prefix} help</code> for help"
+                     f" or <code>{cmd_prefix} login</code> to log in.")
+            pass
+        else:
+            text += f"Use `{cmd_prefix} help` for help."
+            html += f"Use <code>{cmd_prefix} help</code> for help."
+        await self.az.intent.send_notice(room_id, text=text, html=html)
+
     async def handle_invite(self, room_id: MatrixRoomID, user_id: MatrixUserID,
                             inviter_mxid: MatrixUserID) -> None:
         self.log.debug(f"{inviter_mxid} invited {user_id} to {room_id}")
@@ -213,7 +242,7 @@ class MatrixHandler:
         prefix = self.config["bridge.command_prefix"]
         is_command = text.startswith(prefix)
         if is_command:
-            text = text[len(prefix) + 1:]
+            text = text[len(prefix) + 1:].lstrip()
         return is_command, text
 
     async def handle_message(self, room: MatrixRoomID, sender_id: MatrixUserID, message: Dict,
@@ -222,7 +251,7 @@ class MatrixHandler:
         sender = await u.User.get_by_mxid(sender_id).ensure_started()
         if not sender.relaybot_whitelisted:
             self.log.debug(f"Ignoring message \"{message}\" from {sender} to {room}:"
-                           " u.User is not whitelisted.")
+                           " User is not whitelisted.")
             return
         self.log.debug(f"Received Matrix event \"{message}\" from {sender} in {room}")
 
@@ -248,7 +277,7 @@ class MatrixHandler:
                 # Not enough values to unpack, i.e. no arguments
                 command = text
                 args = []
-            await self.commands.handle(room, sender, command, args, is_management,
+            await self.commands.handle(room, event_id, sender, command, args, is_management,
                                        is_portal=portal is not None)
 
     @staticmethod
@@ -296,11 +325,17 @@ class MatrixHandler:
             events = new_events - old_events
             if len(events) > 0:
                 # New event pinned, set that as pinned in Telegram.
-                await portal.handle_matrix_pin(sender, events.pop())
+                await portal.handle_matrix_pin(sender, MatrixEventID(events.pop()))
             elif len(new_events) == 0:
                 # All pinned events removed, remove pinned event in Telegram.
                 await portal.handle_matrix_pin(sender, None)
 
+    @staticmethod
+    async def handle_room_upgrade(room_id: MatrixRoomID, new_room_id: MatrixRoomID) -> None:
+        portal = po.Portal.get_by_mxid(room_id)
+        if portal:
+            await portal.handle_matrix_upgrade(new_room_id)
+
     @staticmethod
     async def handle_name_change(room_id: MatrixRoomID, user_id: MatrixUserID, displayname: str,
                                  prev_displayname: str, event_id: MatrixEventID) -> None:
@@ -373,6 +408,7 @@ class MatrixHandler:
     async def handle_event(self, evt: MatrixEvent) -> None:
         if self.filter_matrix_event(evt):
             return
+        start_time = time.time()
         self.log.debug("Received event: %s", evt)
         evt_type = evt.get("type", "m.unknown")  # type: str
         room_id = evt.get("room_id", None)  # type: Optional[MatrixRoomID]
@@ -416,9 +452,15 @@ class MatrixHandler:
             except KeyError:
                 old_events = set()
             await self.handle_room_pin(room_id, sender, new_events, old_events)
+        elif evt_type == "m.room.tombstone":
+            await self.handle_room_upgrade(room_id, evt["content"]["replacement_room"])
         elif evt_type == "m.receipt":
             await self.handle_read_receipts(room_id, self.parse_read_receipts(content))
         elif evt_type == "m.presence":
             await self.handle_presence(sender, content.get("presence", "offline"))
         elif evt_type == "m.typing":
             await self.handle_typing(room_id, content.get("user_ids", []))
+        else:
+            return
+        if EVENT_TIME:
+            EVENT_TIME.labels(event_type=evt_type).observe(time.time() - start_time)

mautrix_telegram/puppet.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -14,7 +14,7 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Awaitable, Coroutine, Dict, List, Optional, Pattern, TYPE_CHECKING
+from typing import Awaitable, Any, Dict, List, Iterable, Optional, Pattern, Union, TYPE_CHECKING
 from difflib import SequenceMatcher
 from enum import Enum
 from aiohttp import ServerDisconnectedError
@@ -22,9 +22,8 @@ import asyncio
 import logging
 import re
 
-from sqlalchemy import orm
-
-from telethon.tl.types import UserProfilePhoto, User, FileLocation
+from telethon.tl.types import (UserProfilePhoto, User, UpdateUserName, PeerUser, TypeInputPeer,
+                               InputPeerPhotoFileLocation, UserProfilePhotoEmpty)
 from mautrix_appservice import AppService, IntentAPI, IntentError, MatrixRequestError
 
 from .types import MatrixUserID, TelegramID
@@ -35,7 +34,6 @@ if TYPE_CHECKING:
     from .matrix import MatrixHandler
     from .config import Config
     from .context import Context
-    from . import user as u
     from .abstract_user import AbstractUser
 
 PuppetError = Enum('PuppetError', 'Success OnlyLoginSelf InvalidAccessToken')
@@ -45,7 +43,6 @@ config = None  # type: Config
 
 class Puppet:
     log = logging.getLogger("mau.puppet")  # type: logging.Logger
-    db = None  # type: orm.Session
     az = None  # type: AppService
     mx = None  # type: MatrixHandler
     loop = None  # type: asyncio.AbstractEventLoop
@@ -65,6 +62,7 @@ class Puppet:
                  photo_id: Optional[str] = None,
                  is_bot: bool = False,
                  is_registered: bool = False,
+                 disable_updates: bool = False,
                  db_instance: Optional[DBPuppet] = None) -> None:
         self.id = id  # type: TelegramID
         self.access_token = access_token  # type: Optional[str]
@@ -77,10 +75,12 @@ class Puppet:
         self.photo_id = photo_id  # type: Optional[str]
         self.is_bot = is_bot  # type: bool
         self.is_registered = is_registered  # type: bool
+        self.disable_updates = disable_updates  # type: bool
         self._db_instance = db_instance  # type: Optional[DBPuppet]
 
         self.default_mxid_intent = self.az.intent.user(self.default_mxid)
         self.intent = self._fresh_intent()  # type: IntentAPI
+        self.sync_task = None  # type: Optional[asyncio.Future]
 
         self.cache[id] = self
         if self.custom_mxid:
@@ -104,6 +104,19 @@ class Puppet:
         """ Is True if the puppet is logged in. """
         return True
 
+    @property
+    def plain_displayname(self) -> str:
+        tpl = config["bridge.displayname_template"]
+        if tpl == "{displayname}":
+            # Template has no extra stuff, no need to parse.
+            return self.displayname
+        regex = re.compile("^" + re.escape(tpl).replace(re.escape("{displayname}"), "(.+?)") + "$")
+        match = regex.match(self.displayname)
+        return match.group(1) or self.displayname
+
+    def get_input_entity(self, user: 'AbstractUser') -> Awaitable[TypeInputPeer]:
+        return user.client.get_input_entity(PeerUser(user_id=self.tgid))
+
     # region Custom puppet management
     def _fresh_intent(self) -> IntentAPI:
         return (self.az.intent.user(self.custom_mxid, self.access_token)
@@ -143,7 +156,7 @@ class Puppet:
                 return PuppetError.OnlyLoginSelf
             return PuppetError.InvalidAccessToken
         if config["bridge.sync_with_custom_puppets"]:
-            asyncio.ensure_future(self.sync(), loop=self.loop)
+            self.sync_task = asyncio.ensure_future(self.sync(), loop=self.loop)
         return PuppetError.Success
 
     async def leave_rooms_with_default_user(self) -> None:
@@ -225,6 +238,8 @@ class Puppet:
     async def sync(self) -> None:
         try:
             await self._sync()
+        except asyncio.CancelledError:
+            self.log.info("Syncing cancelled")
         except Exception:
             self.log.exception("Fatal error syncing")
 
@@ -272,28 +287,26 @@ class Puppet:
         return DBPuppet(id=self.id, access_token=self.access_token, custom_mxid=self.custom_mxid,
                         username=self.username, displayname=self.displayname,
                         displayname_source=self.displayname_source, photo_id=self.photo_id,
-                        is_bot=self.is_bot, matrix_registered=self.is_registered)
+                        is_bot=self.is_bot, matrix_registered=self.is_registered,
+                        disable_updates=self.disable_updates)
 
     @classmethod
     def from_db(cls, db_puppet: DBPuppet) -> 'Puppet':
         return Puppet(db_puppet.id, db_puppet.access_token, db_puppet.custom_mxid,
                       db_puppet.username, db_puppet.displayname, db_puppet.displayname_source,
                       db_puppet.photo_id, db_puppet.is_bot, db_puppet.matrix_registered,
-                      db_instance=db_puppet)
+                      db_puppet.disable_updates, db_instance=db_puppet)
 
     def save(self) -> None:
-        self.db_instance.access_token = self.access_token
-        self.db_instance.custom_mxid = self.custom_mxid
-        self.db_instance.username = self.username
-        self.db_instance.displayname = self.displayname
-        self.db_instance.displayname_source = self.displayname_source
-        self.db_instance.photo_id = self.photo_id
-        self.db_instance.is_bot = self.is_bot
-        self.db_instance.matrix_registered = self.is_registered
-        self.db.commit()
+        self.db_instance.update(access_token=self.access_token, custom_mxid=self.custom_mxid,
+                                username=self.username, displayname=self.displayname,
+                                displayname_source=self.displayname_source, photo_id=self.photo_id,
+                                is_bot=self.is_bot, matrix_registered=self.is_registered,
+                                disable_updates=self.disable_updates)
 
     # endregion
     # region Info updating
+
     def similarity(self, query: str) -> int:
         username_similarity = (SequenceMatcher(None, self.username, query).ratio()
                                if self.username else 0)
@@ -312,25 +325,26 @@ class Puppet:
             "first name": info.first_name,
             "last name": info.last_name,
         }
-        preferences = config.get("bridge.displayname_preference",
-                                 ["full name", "username", "phone"])
+        preferences = config["bridge.displayname_preference"]
         name = None
         for preference in preferences:
             name = data[preference]
             if name:
                 break
 
-        if info.deleted:
+        if isinstance(info, User) and info.deleted:
             name = f"Deleted account {info.id}"
         elif not name:
             name = info.id
 
         if not enable_format:
             return name
-        return config.get("bridge.displayname_template", "{displayname} (Telegram)").format(
+        return config["bridge.displayname_template"].format(
             displayname=name)
 
     async def update_info(self, source: 'AbstractUser', info: User) -> None:
+        if self.disable_updates:
+            return
         changed = False
         if self.username != info.username:
             self.username = info.username
@@ -338,39 +352,75 @@ class Puppet:
 
         changed = await self.update_displayname(source, info) or changed
         if isinstance(info.photo, UserProfilePhoto):
-            changed = await self.update_avatar(source, info.photo.photo_big) or changed
+            changed = await self.update_avatar(source, info.photo) or changed
 
         self.is_bot = info.bot
 
         if changed:
             self.save()
 
-    async def update_displayname(self, source: 'AbstractUser', info: User) -> bool:
-        ignore_source = (not source.is_relaybot
-                         and self.displayname_source is not None
-                         and self.displayname_source != source.tgid)
-        if ignore_source:
+    async def update_displayname(self, source: 'AbstractUser', info: Union[User, UpdateUserName]
+                                 ) -> bool:
+        if self.disable_updates:
             return False
+        is_main_source = (source.is_relaybot or (self.displayname_source is not None
+                                                 and self.displayname_source == source.tgid))
+        # No phone -> not in contact list -> can't set custom name -> name is trustworthy
+        is_trustworthy_source = isinstance(info, User) and info.phone is None
+        if not is_main_source and not is_trustworthy_source:
+            return False
+        elif isinstance(info, UpdateUserName):
+            info = await source.client.get_entity(PeerUser(self.tgid))
 
         displayname = self.get_displayname(info)
         if displayname != self.displayname:
-            await self.default_mxid_intent.set_display_name(displayname)
             self.displayname = displayname
             self.displayname_source = source.tgid
+            try:
+                await self.default_mxid_intent.set_display_name(displayname)
+            except MatrixRequestError:
+                self.log.exception("Failed to set displayname")
+                self.displayname = ""
+                self.displayname_source = None
             return True
         elif source.is_relaybot or self.displayname_source is None:
             self.displayname_source = source.tgid
             return True
         return False
 
-    async def update_avatar(self, source: 'AbstractUser', photo: FileLocation) -> bool:
-        photo_id = f"{photo.volume_id}-{photo.local_id}"
+    async def update_avatar(self, source: 'AbstractUser',
+                            photo: Union[UserProfilePhoto, UserProfilePhotoEmpty]) -> bool:
+        if self.disable_updates:
+            return False
+
+        if isinstance(photo, UserProfilePhotoEmpty):
+            photo_id = ""
+        else:
+            photo_id = str(photo.photo_id)
         if self.photo_id != photo_id:
-            file = await util.transfer_file_to_matrix(self.db, source.client,
-                                                      self.default_mxid_intent, photo)
+            if not photo_id:
+                self.photo_id = ""
+                try:
+                    await self.default_mxid_intent.set_avatar("")
+                except MatrixRequestError:
+                    self.log.exception("Failed to set avatar")
+                    self.photo_id = ""
+                return True
+
+            loc = InputPeerPhotoFileLocation(
+                peer=await self.get_input_entity(source),
+                local_id=photo.photo_big.local_id,
+                volume_id=photo.photo_big.volume_id,
+                big=True
+            )
+            file = await util.transfer_file_to_matrix(source.client, self.default_mxid_intent, loc)
             if file:
-                await self.default_mxid_intent.set_avatar(file.mxc)
                 self.photo_id = photo_id
+                try:
+                    await self.default_mxid_intent.set_avatar(file.mxc)
+                except MatrixRequestError:
+                    self.log.exception("Failed to set avatar")
+                    self.photo_id = ""
                 return True
         return False
 
@@ -384,14 +434,13 @@ class Puppet:
         except KeyError:
             pass
 
-        puppet = DBPuppet.query.get(tgid)
+        puppet = DBPuppet.get_by_tgid(tgid)
         if puppet:
             return cls.from_db(puppet)
 
         if create:
             puppet = cls(tgid)
-            cls.db.add(puppet.db_instance)
-            cls.db.commit()
+            puppet.db_instance.insert()
             return puppet
 
         return None
@@ -414,7 +463,7 @@ class Puppet:
         except KeyError:
             pass
 
-        puppet = DBPuppet.query.filter(DBPuppet.custom_mxid == mxid).one_or_none()
+        puppet = DBPuppet.get_by_custom_mxid(mxid)
         if puppet:
             puppet = cls.from_db(puppet)
             return puppet
@@ -422,11 +471,11 @@ class Puppet:
         return None
 
     @classmethod
-    def get_all_with_custom_mxid(cls) -> List['Puppet']:
-        return [cls.by_custom_mxid[puppet.mxid]
+    def all_with_custom_mxid(cls) -> Iterable['Puppet']:
+        return (cls.by_custom_mxid[puppet.mxid]
                 if puppet.custom_mxid in cls.by_custom_mxid
                 else cls.from_db(puppet)
-                for puppet in DBPuppet.query.filter(DBPuppet.custom_mxid is not None).all()]
+                for puppet in DBPuppet.all_with_custom_mxid())
 
     @classmethod
     def get_id_from_mxid(cls, mxid: MatrixUserID) -> Optional[TelegramID]:
@@ -448,7 +497,7 @@ class Puppet:
             if puppet.username and puppet.username.lower() == username.lower():
                 return puppet
 
-        dbpuppet = DBPuppet.query.filter(DBPuppet.username == username).one_or_none()
+        dbpuppet = DBPuppet.get_by_username(username)
         if dbpuppet:
             return cls.from_db(dbpuppet)
 
@@ -463,7 +512,7 @@ class Puppet:
             if puppet.displayname and puppet.displayname == displayname:
                 return puppet
 
-        dbpuppet = DBPuppet.query.filter(DBPuppet.displayname == displayname).one_or_none()
+        dbpuppet = DBPuppet.get_by_displayname(displayname)
         if dbpuppet:
             return cls.from_db(dbpuppet)
 
@@ -471,12 +520,12 @@ class Puppet:
     # endregion
 
 
-def init(context: 'Context') -> List[Coroutine]:  # [None, None, PuppetError]
+def init(context: 'Context') -> List[Awaitable[Any]]:  # [None, None, PuppetError]
     global config
-    Puppet.az, Puppet.db, config, Puppet.loop, _ = context.core
+    Puppet.az, config, Puppet.loop, _ = context.core
     Puppet.mx = context.mx
     Puppet.username_template = config.get("bridge.username_template", "telegram_{userid}")
     Puppet.hs_domain = config["homeserver"]["domain"]
     Puppet.mxid_regex = re.compile(
         f"@{Puppet.username_template.format(userid='([0-9]+)')}:{Puppet.hs_domain}")
-    return [puppet.init_custom_mxid() for puppet in Puppet.get_all_with_custom_mxid()]
+    return [puppet.init_custom_mxid() for puppet in Puppet.all_with_custom_mxid()]

mautrix_telegram/scripts/dbms_migrate/__main__.py

@@ -11,12 +11,19 @@ parser.add_argument("-f", "--from-url", type=str, required=True, metavar="<url>"
                     help="the old database path")
 parser.add_argument("-t", "--to-url", type=str, required=True, metavar="<url>",
                     help="the new database path")
+parser.add_argument("-v", "--verbose", action="store_true", help="Verbose logs while migrating")
 args = parser.parse_args()
+verbose = args.verbose or False
+
+
+def log(message, end="\n"):
+    if verbose:
+        print(message, end=end, flush=True)
 
 
 def connect(to):
-    import mautrix_telegram.base as base
-    base.Base = declarative_base()
+    import mautrix_telegram.db.base as base
+    base.Base = declarative_base(cls=base.BaseBase)
     from mautrix_telegram.db import (Portal, Message, UserPortal, User, RoomState, UserProfile,
                                      Contact, Puppet, BotChat, TelegramFile)
     db_engine = sql.create_engine(to)
@@ -45,15 +52,30 @@ def connect(to):
         "TelegramFile": TelegramFile,
     }
 
-
+log("Connecting to old database")
 session, tables = connect(args.from_url)
 
 data = {}
 for name, table in tables.items():
+    log("Reading table {name}...".format(name=name), end=" ")
     data[name] = session.query(table).all()
+    log("Done!")
 
+log("Connecting to new database")
 session, tables = connect(args.to_url)
+
 for name, table in tables.items():
+    log("Writing table {name}".format(name=name), end="")
+    length = len(data[name])
+    n = 0
     for row in data[name]:
         session.merge(row)
+        n += 5
+        if n >= length:
+            log(".", end="")
+            n = 0
+    log(" Done!")
+
+log("Committing changes to database...", end=" ")
 session.commit()
+log("Done!")

mautrix_telegram/scripts/telematrix_import/__main__.py

@@ -1,10 +1,27 @@
-import argparse
-import sqlalchemy as sql
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Dict
 from sqlalchemy import orm
+import sqlalchemy as sql
+import argparse
 
-from mautrix_telegram.base import Base
+from mautrix_telegram.db import Base, Portal, Message, Puppet, BotChat
 from mautrix_telegram.config import Config
-from mautrix_telegram.db import Portal, Message, Puppet, BotChat
+
 from .models import ChatLink, TgUser, MatrixUser, Message as TMMessage, Base as TelematrixBase
 
 parser = argparse.ArgumentParser(
@@ -21,7 +38,8 @@ args = parser.parse_args()
 config = Config(args.config, None, None)
 config.load()
 
-mxtg_db_engine = sql.create_engine(config.get("appservice.database", "sqlite:///mautrix-telegram.db"))
+mxtg_db_engine = sql.create_engine(
+    config.get("appservice.database", "sqlite:///mautrix-telegram.db"))
 mxtg = orm.sessionmaker(bind=mxtg_db_engine)()
 Base.metadata.bind = mxtg_db_engine
 
@@ -37,10 +55,11 @@ tm_messages = telematrix.query(TMMessage).all()
 telematrix.close()
 telematrix_db_engine.dispose()
 
-portals = {}  # Dict[int, Portal]
-chats = {}  # Dict[int, BotChat]
-messages = {}  # Dict[str, Message]
-puppets = {}  # Dict[int, Puppet]
+portals_by_tgid = {}  # type: Dict[int, Portal]
+portals_by_mxid = {}  # type: Dict[str, Portal]
+chats = {}  # type: Dict[int, BotChat]
+messages = {}  # type: Dict[str, Message]
+puppets = {}  # type: Dict[int, Puppet]
 
 for chat_link in chat_links:
     if type(chat_link.tg_room) is str:
@@ -61,16 +80,28 @@ for chat_link in chat_links:
 
     portal = Portal(tgid=tgid, tg_receiver=tgid, peer_type=peer_type, megagroup=megagroup,
                     mxid=chat_link.matrix_room)
-    bot_chat = BotChat(id=tgid, type=peer_type)
-    portals[chat_link.tg_room] = portal
-    chats[tgid] = bot_chat
+    chats[tgid] = BotChat(id=tgid, type=peer_type)
+    if chat_link.tg_room in portals_by_tgid:
+        print(f"Warning: Ignoring bridge from {portal.tgid} to {portal.mxid} "
+              f"in favor of {portals_by_tgid[portal.tgid].mxid}")
+        continue
+    elif chat_link.matrix_room in portals_by_mxid:
+        print(f"Warning: Ignoring bridge from {portal.mxid} to {portal.tgid} "
+              f"in favor of {portals_by_mxid[portal.mxid].tgid}")
+        continue
+    portals_by_tgid[portal.tgid] = portal
+    portals_by_mxid[portal.mxid] = portal
 
 for tm_msg in tm_messages:
     try:
-        portal = portals[tm_msg.tg_group_id]
+        portal = portals_by_tgid[tm_msg.tg_group_id]
     except KeyError:
-        print("Found message entry %d in unlinked chat %d, ignoring..." % (tm_msg.tg_message_id,
-                                                                           tm_msg.tg_group_id))
+        print(f"Found message entry {tm_msg.tg_message_id} in unlinked chat {tm_msg.tg_group_id},"
+              " ignoring...")
+        continue
+    if tm_msg.matrix_room_id != portal.mxid:
+        print(f"Found message entry {tm_msg.tg_message_id} with "
+              f"mismatching matrix room ID {tm_msg.matrix_room_id} (expected {portal.mxid})")
         continue
     tg_space = portal.tgid if portal.peer_type == "channel" else args.bot_id
     message = Message(mxid=tm_msg.matrix_event_id, mx_room=tm_msg.matrix_room_id,
@@ -81,7 +112,7 @@ for user in tg_users:
     puppets[user.tg_id] = Puppet(id=user.tg_id, displayname=user.name,
                                  displayname_source=args.bot_id)
 
-for k, v in portals.items():
+for k, v in portals_by_tgid.items():
     mxtg.add(v)
 for k, v in chats.items():
     mxtg.add(v)

mautrix_telegram/sqlstatestore.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -16,8 +16,6 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 from typing import Dict, Tuple
 
-from sqlalchemy import orm
-
 from mautrix_appservice import StateStore
 
 from .types import MatrixUserID, MatrixRoomID
@@ -26,9 +24,8 @@ from .db import RoomState, UserProfile
 
 
 class SQLStateStore(StateStore):
-    def __init__(self, db: orm.Session) -> None:
+    def __init__(self) -> None:
         super().__init__()
-        self.db = db  # type: orm.Session
         self.profile_cache = {}  # type: Dict[Tuple[str, str], UserProfile]
         self.room_state_cache = {}  # type: Dict[str, RoomState]
 
@@ -59,13 +56,12 @@ class SQLStateStore(StateStore):
         except KeyError:
             pass
 
-        profile = UserProfile.query.get(key)
+        profile = UserProfile.get(*key)
         if profile:
             self.profile_cache[key] = profile
         elif create:
-            profile = UserProfile(room_id=room_id, user_id=user_id)
-            self.db.add(profile)
-            self.db.commit()
+            profile = UserProfile(room_id=room_id, user_id=user_id, membership="leave")
+            profile.insert()
             self.profile_cache[key] = profile
         return profile
 
@@ -77,7 +73,7 @@ class SQLStateStore(StateStore):
         profile.membership = member.get("membership", profile.membership or "leave")
         profile.displayname = member.get("displayname", profile.displayname)
         profile.avatar_url = member.get("avatar_url", profile.avatar_url)
-        self.db.commit()
+        profile.update()
 
     def set_membership(self, room: MatrixRoomID, user: MatrixUserID, membership: str) -> None:
         self.set_member(room, user, {
@@ -90,16 +86,17 @@ class SQLStateStore(StateStore):
         except KeyError:
             pass
 
-        room = RoomState.query.get(room_id)
+        room = RoomState.get(room_id)
         if room:
             self.room_state_cache[room_id] = room
         elif create:
             room = RoomState(room_id=room_id)
+            room.insert()
             self.room_state_cache[room_id] = room
         return room
 
     def has_power_levels(self, room: MatrixRoomID) -> bool:
-        return self._get_room_state(room).has_power_levels
+        return bool(self._get_room_state(room).power_levels)
 
     def get_power_levels(self, room: MatrixRoomID) -> Dict:
         return self._get_room_state(room).power_levels
@@ -114,9 +111,9 @@ class SQLStateStore(StateStore):
             }
         power_levels[room]["users"][user] = level
         room_state.power_levels = power_levels
-        self.db.commit()
+        room_state.update()
 
     def set_power_levels(self, room: MatrixRoomID, content: Dict) -> None:
         state = self._get_room_state(room)
         state.power_levels = content
-        self.db.commit()
+        state.update()

mautrix_telegram/tgclient.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -21,17 +21,17 @@ from telethon.tl.functions.messages import SendMediaRequest
 from telethon.tl.types import (
     InputMediaUploadedDocument, InputMediaUploadedPhoto, TypeDocumentAttribute, TypeInputMedia,
     TypeInputPeer, TypeMessageEntity, TypeMessageMedia, TypePeer)
-from telethon.tl import custom
+from telethon.tl.patched import Message
 
 
 class MautrixTelegramClient(TelegramClient):
     async def upload_file_direct(self, file: bytes, mime_type: str = None,
                                  attributes: List[TypeDocumentAttribute] = None,
-                                 file_name: str = None
+                                 file_name: str = None, max_image_size: float = 10 * 1000 ** 2,
                                  ) -> Union[InputMediaUploadedDocument, InputMediaUploadedPhoto]:
         file_handle = await super().upload_file(file, file_name=file_name, use_cache=False)
 
-        if mime_type == "image/png" or mime_type == "image/jpeg":
+        if (mime_type == "image/png" or mime_type == "image/jpeg") and len(file) < max_image_size:
             return InputMediaUploadedPhoto(file_handle)
         else:
             attributes = attributes or []
@@ -45,7 +45,7 @@ class MautrixTelegramClient(TelegramClient):
     async def send_media(self, entity: Union[TypeInputPeer, TypePeer],
                          media: Union[TypeInputMedia, TypeMessageMedia],
                          caption: str = None, entities: List[TypeMessageEntity] = None,
-                         reply_to: int = None) -> Optional[custom.Message]:
+                         reply_to: int = None) -> Optional[Message]:
         entity = await self.get_input_entity(entity)
         reply_to = utils.get_message_id(reply_to)
         request = SendMediaRequest(entity, media, message=caption or "", entities=entities or [],

mautrix_telegram/user.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -14,22 +14,21 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Coroutine, Dict, List, Match, NewType, Optional, Tuple, cast, TYPE_CHECKING
+from typing import Awaitable, Dict, List, Iterable, Match, NewType, Optional, Tuple, TYPE_CHECKING
 import logging
 import asyncio
 import re
 
 from telethon.tl.types import (
     TypeUpdate, UpdateNewMessage, UpdateNewChannelMessage, PeerUser,
-    UpdateShortChatMessage, UpdateShortMessage)
-from telethon.tl.types import User as TLUser
+    UpdateShortChatMessage, UpdateShortMessage, User as TLUser)
 from telethon.tl.types.contacts import ContactsNotModified
 from telethon.tl.functions.contacts import GetContactsRequest, SearchRequest
 from telethon.tl.functions.account import UpdateStatusRequest
 from mautrix_appservice import MatrixRequestError
 
 from .types import MatrixUserID, TelegramID
-from .db import User as DBUser, Contact as DBContact, Portal as DBPortal
+from .db import User as DBUser
 from .abstract_user import AbstractUser
 from . import portal as po, puppet as pu
 
@@ -49,9 +48,9 @@ class User(AbstractUser):
 
     def __init__(self, mxid: MatrixUserID, tgid: Optional[TelegramID] = None,
                  username: Optional[str] = None, phone: Optional[str] = None,
-                 db_contacts: Optional[List[DBContact]] = None,
+                 db_contacts: Optional[Iterable[TelegramID]] = None,
                  saved_contacts: int = 0, is_bot: bool = False,
-                 db_portals: Optional[List[DBPortal]] = None,
+                 db_portals: Optional[Iterable[Tuple[TelegramID, TelegramID]]] = None,
                  db_instance: Optional[DBUser] = None) -> None:
         super().__init__()
         self.mxid = mxid  # type: MatrixUserID
@@ -61,12 +60,12 @@ class User(AbstractUser):
         self.phone = phone  # type: str
         self.contacts = []  # type: List[pu.Puppet]
         self.saved_contacts = saved_contacts  # type: int
-        self.db_contacts = db_contacts  # type: List[DBContact]
-        self.portals = {}  # type: Dict[Tuple[int, int], po.Portal]
-        self.db_portals = db_portals or []  # type: List[DBPortal]
+        self.db_contacts = db_contacts
+        self.portals = {}  # type: Dict[Tuple[TelegramID, TelegramID], po.Portal]
+        self.db_portals = db_portals or []
         self._db_instance = db_instance  # type: Optional[DBUser]
 
-        self.command_status = None  # type: Dict
+        self.command_status = None  # type: Optional[Dict]
 
         (self.relaybot_whitelisted,
          self.whitelisted,
@@ -98,24 +97,30 @@ class User(AbstractUser):
         return self.mxid_localpart
 
     @property
-    def db_contacts(self) -> List[DBContact]:
-        return [self.db.merge(DBContact(user=self.tgid, contact=puppet.id))
-                for puppet in self.contacts]
-
-    @db_contacts.setter
-    def db_contacts(self, contacts: List[DBContact]) -> None:
-        self.contacts = [pu.Puppet.get(entry.contact) for entry in contacts] if contacts else []
+    def plain_displayname(self) -> str:
+        return self.displayname
 
     @property
-    def db_portals(self) -> List[DBPortal]:
-        return [portal.db_instance for portal in self.portals.values() if not portal.deleted]
+    def db_contacts(self) -> Iterable[TelegramID]:
+        return (puppet.id
+                for puppet in self.contacts
+                if puppet)
+
+    @db_contacts.setter
+    def db_contacts(self, contacts: Iterable[TelegramID]) -> None:
+        self.contacts = [pu.Puppet.get(entry) for entry in contacts] if contacts else []
+
+    @property
+    def db_portals(self) -> Iterable[Tuple[TelegramID, TelegramID]]:
+        return (portal.tgid_full
+                for portal in self.portals.values()
+                if portal and not portal.deleted)
 
     @db_portals.setter
-    def db_portals(self, portals: List[DBPortal]) -> None:
+    def db_portals(self, portals: Iterable[Tuple[TelegramID, TelegramID]]) -> None:
         self.portals = {
-            (portal.tgid, portal.tg_receiver): po.Portal.get_by_tgid(portal.tgid,
-                                                                     portal.tg_receiver)
-            for portal in portals
+            tgid_full: po.Portal.get_by_tgid(*tgid_full)
+            for tgid_full in portals
         } if portals else {}
 
     # region Database conversion
@@ -128,27 +133,24 @@ class User(AbstractUser):
 
     def new_db_instance(self) -> DBUser:
         return DBUser(mxid=self.mxid, tgid=self.tgid, tg_username=self.username,
-                      contacts=self.db_contacts, saved_contacts=self.saved_contacts,
-                      portals=self.db_portals)
+                      saved_contacts=self.saved_contacts, portals=self.db_portals)
 
-    def save(self) -> None:
-        self.db_instance.tgid = self.tgid
-        self.db_instance.tg_username = self.username
-        self.db_instance.tg_phone = self.phone
-        self.db_instance.contacts = self.db_contacts
-        self.db_instance.saved_contacts = self.saved_contacts
-        self.db_instance.portals = self.db_portals
-        self.db.commit()
+    def save(self, contacts: bool = False, portals: bool = False) -> None:
+        self.db_instance.update(tgid=self.tgid, tg_username=self.username, tg_phone=self.phone,
+                                saved_contacts=self.saved_contacts)
+        if contacts:
+            self.db_instance.contacts = self.db_contacts
+        if portals:
+            self.db_instance.portals = self.db_portals
 
-    def delete(self) -> None:
+    def delete(self, delete_db: bool = True) -> None:
         try:
             del self.by_mxid[self.mxid]
             del self.by_tgid[self.tgid]
         except KeyError:
             pass
-        if self._db_instance:
-            self.db.delete(self._db_instance)
-            self.db.commit()
+        if delete_db and self._db_instance:
+            self._db_instance.delete()
 
     @classmethod
     def from_db(cls, db_user: DBUser) -> 'User':
@@ -159,6 +161,9 @@ class User(AbstractUser):
     # endregion
     # region Telegram connection management
 
+    def ensure_started(self, even_if_no_session=False) -> Awaitable['User']:
+        return super().ensure_started(even_if_no_session)
+
     async def start(self, delete_unless_authenticated: bool = False) -> 'User':
         await super().start()
         if await self.is_logged_in():
@@ -173,7 +178,7 @@ class User(AbstractUser):
     async def post_login(self, info: TLUser = None) -> None:
         try:
             await self.update_info(info)
-            if not self.is_bot:
+            if not self.is_bot and config["bridge.startup_sync"]:
                 await self.sync_dialogs()
                 await self.sync_contacts()
             if config["bridge.catch_up"]:
@@ -207,9 +212,6 @@ class User(AbstractUser):
     # endregion
     # region Telegram actions that need custom methods
 
-    def ensure_started(self, even_if_no_session: bool = False) -> Coroutine[None, None, 'User']:
-        return cast(Coroutine[None, None, 'User'], super().ensure_started(even_if_no_session))
-
     async def set_presence(self, online: bool = True) -> None:
         if not self.is_bot:
             await self.client(UpdateStatusRequest(offline=not online))
@@ -237,7 +239,7 @@ class User(AbstractUser):
         if puppet.is_real_user:
             await puppet.switch_mxid(None, None)
         for _, portal in self.portals.items():
-            if not portal.mxid or portal.has_bot:
+            if not portal or portal.deleted or not portal.mxid or portal.has_bot:
                 continue
             try:
                 await portal.main_intent.kick(portal.mxid, self.mxid, "Logged out of Telegram.")
@@ -245,7 +247,7 @@ class User(AbstractUser):
                 pass
         self.portals = {}
         self.contacts = []
-        self.save()
+        self.save(portals=True, contacts=True)
         if self.tgid:
             try:
                 del self.by_tgid[self.tgid]
@@ -294,13 +296,13 @@ class User(AbstractUser):
 
     async def sync_dialogs(self, synchronous_create: bool = False) -> None:
         creators = []
-        for entity in await self.get_dialogs(limit=30):
+        for entity in await self.get_dialogs(limit=config["bridge.sync_dialog_limit"] or None):
             portal = po.Portal.get_by_entity(entity)
             self.portals[portal.tgid_full] = portal
             creators.append(
                 portal.create_matrix_room(self, entity, invites=[self.mxid],
                                           synchronous=synchronous_create))
-        self.save()
+        self.save(portals=True)
         await asyncio.gather(*creators, loop=self.loop)
 
     def register_portal(self, portal: po.Portal) -> None:
@@ -310,37 +312,37 @@ class User(AbstractUser):
         except KeyError:
             pass
         self.portals[portal.tgid_full] = portal
-        self.save()
+        self.save(portals=True)
 
     def unregister_portal(self, portal: po.Portal) -> None:
         try:
             del self.portals[portal.tgid_full]
-            self.save()
+            self.save(portals=True)
         except KeyError:
             pass
 
     async def needs_relaybot(self, portal: po.Portal) -> bool:
         return not await self.is_logged_in() or (
-            self.is_bot and portal.tgid_full not in self.portals)
+            (portal.has_bot or self.bot) and portal.tgid_full not in self.portals)
 
     def _hash_contacts(self) -> int:
         acc = 0
-        for id in sorted([self.saved_contacts] + [contact.id for contact in self.contacts]):
-            acc = (acc * 20261 + id) & 0xffffffff
+        for contact in sorted([self.saved_contacts] + [contact.id for contact in self.contacts]):
+            acc = (acc * 20261 + contact) & 0xffffffff
         return acc & 0x7fffffff
 
     async def sync_contacts(self) -> None:
         response = await self.client(GetContactsRequest(hash=self._hash_contacts()))
         if isinstance(response, ContactsNotModified):
             return
-        self.log.debug("Updating contacts...")
+        self.log.debug(f"Updating contacts of {self.name}...")
         self.contacts = []
         self.saved_contacts = response.saved_count
         for user in response.users:
             puppet = pu.Puppet.get(user.id)
             await puppet.update_info(self, user)
             self.contacts.append(puppet)
-        self.save()
+        self.save(contacts=True)
 
     # endregion
     # region Class instance lookup
@@ -355,27 +357,26 @@ class User(AbstractUser):
         except KeyError:
             pass
 
-        user = DBUser.query.get(mxid)
+        user = DBUser.get_by_mxid(mxid)
         if user:
             user = cls.from_db(user)
             return user
 
         if create:
             user = cls(mxid)
-            cls.db.add(user.db_instance)
-            cls.db.commit()
+            user.db_instance.insert()
             return user
 
         return None
 
     @classmethod
-    def get_by_tgid(cls, tgid: int) -> Optional['User']:
+    def get_by_tgid(cls, tgid: TelegramID) -> Optional['User']:
         try:
             return cls.by_tgid[tgid]
         except KeyError:
             pass
 
-        user = DBUser.query.filter(DBUser.tgid == tgid).one_or_none()
+        user = DBUser.get_by_tgid(tgid)
         if user:
             user = cls.from_db(user)
             return user
@@ -391,7 +392,7 @@ class User(AbstractUser):
             if user.username and user.username.lower() == username.lower():
                 return user
 
-        puppet = DBUser.query.filter(DBUser.tg_username == username).one_or_none()
+        puppet = DBUser.get_by_username(username)
         if puppet:
             return cls.from_db(puppet)
 
@@ -399,9 +400,9 @@ class User(AbstractUser):
     # endregion
 
 
-def init(context: 'Context') -> List[Coroutine]:  # [None, None, AbstractUser]
+def init(context: 'Context') -> List[Awaitable['User']]:
     global config
     config = context.config
 
-    users = [User.from_db(user) for user in DBUser.query.all()]
-    return [user.ensure_started() for user in users]
+    users = [User.from_db(user) for user in DBUser.all()]
+    return [user.ensure_started() for user in users if user.tgid]

mautrix_telegram/util/__init__.py

@@ -2,3 +2,6 @@ from .file_transfer import transfer_file_to_matrix, convert_image
 from .format_duration import format_duration
 from .signed_token import sign_token, verify_token
 from .recursive_dict import recursive_del, recursive_set, recursive_get
+
+def ignore_coro(coro):
+    pass

mautrix_telegram/util/file_transfer.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -21,18 +21,18 @@ import logging
 import asyncio
 
 import magic
-from sqlalchemy import orm
 from sqlalchemy.exc import IntegrityError, InvalidRequestError
-from sqlalchemy.orm.exc import FlushError
 
-from telethon.tl.types import (Document, FileLocation, InputFileLocation,
-                               InputDocumentFileLocation, PhotoSize, PhotoCachedSize)
+from telethon.tl.types import (Document, InputFileLocation, InputDocumentFileLocation,
+                               TypePhotoSize, PhotoSize, PhotoCachedSize, InputPhotoFileLocation,
+                               InputPeerPhotoFileLocation)
 from telethon.errors import (AuthBytesInvalidError, AuthKeyInvalidError, LocationInvalidError,
                              SecurityError)
 from mautrix_appservice import IntentAPI
 
 from ..tgclient import MautrixTelegramClient
 from ..db import TelegramFile as DBTelegramFile
+from ..util import sane_mimetypes
 
 try:
     from PIL import Image
@@ -49,7 +49,8 @@ except ImportError:
 
 log = logging.getLogger("mau.util")  # type: logging.Logger
 
-TypeLocation = Union[Document, InputDocumentFileLocation, FileLocation, InputFileLocation]
+TypeLocation = Union[Document, InputDocumentFileLocation, InputPeerPhotoFileLocation,
+                     InputFileLocation, InputPhotoFileLocation]
 
 
 def convert_image(file: bytes, source_mime: str = "image/webp", target_type: str = "png",
@@ -101,9 +102,9 @@ def _read_video_thumbnail(data: bytes, video_ext: str = "mp4", frame_ext: str =
 
 
 def _location_to_id(location: TypeLocation) -> str:
-    if isinstance(location, (Document, InputDocumentFileLocation)):
-        return f"{location.id}-{location.version}"
-    elif isinstance(location, (FileLocation, InputFileLocation)):
+    if isinstance(location, (Document, InputDocumentFileLocation, InputPhotoFileLocation)):
+        return f"{location.id}-{location.access_hash}"
+    elif isinstance(location, (InputFileLocation, InputPeerPhotoFileLocation)):
         return f"{location.volume_id}-{location.local_id}"
 
 
@@ -117,7 +118,11 @@ async def transfer_thumbnail_to_matrix(client: MautrixTelegramClient, intent: In
     if not loc_id:
         return None
 
-    video_ext = mimetypes.guess_extension(mime)
+    db_file = DBTelegramFile.get(loc_id)
+    if db_file:
+        return db_file
+
+    video_ext = sane_mimetypes.guess_extension(mime)
     if VideoFileClip and video_ext:
         try:
             file, width, height = _read_video_thumbnail(video, video_ext, frame_ext="png")
@@ -131,22 +136,31 @@ async def transfer_thumbnail_to_matrix(client: MautrixTelegramClient, intent: In
 
     content_uri = await intent.upload_file(file, mime_type)
 
-    return DBTelegramFile(id=loc_id, mxc=content_uri, mime_type=mime_type,
-                          was_converted=False, timestamp=int(time.time()), size=len(file),
-                          width=width, height=height)
+    db_file = DBTelegramFile(id=loc_id, mxc=content_uri, mime_type=mime_type,
+                             was_converted=False, timestamp=int(time.time()), size=len(file),
+                             width=width, height=height)
+    try:
+        db_file.insert()
+    except (IntegrityError, InvalidRequestError) as e:
+        log.exception(f"{e.__class__.__name__} while saving transferred file thumbnail data. "
+                      "This was probably caused by two simultaneous transfers of the same file, "
+                      "and might (but probably won't) cause problems with thumbnails or something.")
+    return db_file
 
 
 transfer_locks = {}  # type: Dict[str, asyncio.Lock]
 
+TypeThumbnail = Optional[Union[TypeLocation, TypePhotoSize]]
 
-async def transfer_file_to_matrix(db: orm.Session, client: MautrixTelegramClient, intent: IntentAPI,
-                                  location: TypeLocation, thumbnail: Optional[TypeLocation] = None,
+
+async def transfer_file_to_matrix(client: MautrixTelegramClient, intent: IntentAPI,
+                                  location: TypeLocation, thumbnail: TypeThumbnail = None,
                                   is_sticker: bool = False) -> Optional[DBTelegramFile]:
     location_id = _location_to_id(location)
     if not location_id:
         return None
 
-    db_file = DBTelegramFile.query.get(location_id)
+    db_file = DBTelegramFile.get(location_id)
     if db_file:
         return db_file
 
@@ -156,15 +170,15 @@ async def transfer_file_to_matrix(db: orm.Session, client: MautrixTelegramClient
         lock = asyncio.Lock()
         transfer_locks[location_id] = lock
     async with lock:
-        return await _unlocked_transfer_file_to_matrix(db, client, intent, location_id, location,
+        return await _unlocked_transfer_file_to_matrix(client, intent, location_id, location,
                                                        thumbnail, is_sticker)
 
 
-async def _unlocked_transfer_file_to_matrix(db: orm.Session, client: MautrixTelegramClient,
-                                            intent: IntentAPI, loc_id: str, location: TypeLocation,
-                                            thumbnail: Optional[TypeLocation],
-                                            is_sticker: bool) -> Optional[DBTelegramFile]:
-    db_file = DBTelegramFile.query.get(loc_id)
+async def _unlocked_transfer_file_to_matrix(client: MautrixTelegramClient, intent: IntentAPI,
+                                            loc_id: str, location: TypeLocation,
+                                            thumbnail: TypeThumbnail, is_sticker: bool
+                                            ) -> Optional[DBTelegramFile]:
+    db_file = DBTelegramFile.get(loc_id)
     if db_file:
         return db_file
 
@@ -201,16 +215,9 @@ async def _unlocked_transfer_file_to_matrix(db: orm.Session, client: MautrixTele
                                                                mime_type)
 
     try:
-        db.add(db_file)
-        db.commit()
-    except FlushError as e:
-        log.exception(f"{e.__class__.__name__} while saving transferred file data. "
-                      "This was probably caused by two simultaneous transfers of the same file, "
-                      "and should not cause any problems.")
+        db_file.insert()
     except (IntegrityError, InvalidRequestError) as e:
-        db.rollback()
         log.exception(f"{e.__class__.__name__} while saving transferred file data. "
                       "This was probably caused by two simultaneous transfers of the same file, "
                       "and should not cause any problems.")
-
     return db_file

mautrix_telegram/util/format_duration.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by

mautrix_telegram/util/recursive_dict.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by

mautrix_telegram/util/sane_mimetypes.py

@@ -0,0 +1,38 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+import mimetypes
+
+mimetypes.init()
+
+sanity_overrides = {
+    "image/jpeg": ".jpeg",
+    "image/tiff": ".tiff",
+    "text/plain": ".txt",
+    "text/html": ".html",
+    "audio/mpeg": ".mp3",
+    "audio/ogg": ".ogg",
+    "application/xml": ".xml",
+    "application/octet-stream": "",
+    "application/x-msdos-program": ".exe",
+}
+
+
+def guess_extension(mime: str) -> str:
+    try:
+        return sanity_overrides[mime]
+    except KeyError:
+        return mimetypes.guess_extension(mime)

mautrix_telegram/util/signed_token.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by

mautrix_telegram/web/common/auth_api.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -24,8 +24,8 @@ import logging
 
 from telethon.errors import *
 
-from ...commands.auth import enter_password
-from ...util import format_duration
+from ...commands.telegram.auth import enter_password
+from ...util import format_duration, ignore_coro
 from ...puppet import Puppet, PuppetError
 from ...user import User
 
@@ -56,7 +56,7 @@ class AuthAPI(abc.ABC):
                                               error="You have already logged in with your Matrix "
                                                     "account.", errcode="already-logged-in")
 
-        resp = await puppet.switch_mxid(token, user.mxid)
+        resp = await puppet.switch_mxid(token.strip(), user.mxid)
         if resp == PuppetError.OnlyLoginSelf:
             return self.get_mx_login_response(status=403, errcode="only-login-self",
                                               error="You can only log in as your own Matrix user.")
@@ -72,10 +72,15 @@ class AuthAPI(abc.ABC):
                                           errcode="not-yet-implemented")
 
     async def post_login_phone(self, user: User, phone: str) -> web.Response:
+        if not phone or not phone.strip():
+            return self.get_login_response(mxid=user.mxid, state="request", status=400,
+                                           errcode="phone_number_invalid",
+                                           error="Phone number not given.")
         try:
-            await user.client.sign_in(phone or "+123")
+            await user.client.sign_in(phone.strip())
             return self.get_login_response(mxid=user.mxid, state="code", status=200,
-                                           message="Code requested successfully.")
+                                           message="Code requested successfully. Check your SMS "
+                                                   "or Telegram client and enter the code below.")
         except PhoneNumberInvalidError:
             return self.get_login_response(mxid=user.mxid, state="request", status=400,
                                            errcode="phone_number_invalid",
@@ -87,7 +92,8 @@ class AuthAPI(abc.ABC):
         except PhoneNumberAppSignupForbiddenError:
             return self.get_login_response(mxid=user.mxid, state="request", status=403,
                                            errcode="phone_number_app_signup_forbidden",
-                                           error="You have disabled 3rd party apps on your account.")
+                                           error="You have disabled 3rd party apps on your "
+                                                 "account.")
         except PhoneNumberUnoccupiedError:
             return self.get_login_response(mxid=user.mxid, state="request", status=404,
                                            errcode="phone_number_unoccupied",
@@ -112,14 +118,13 @@ class AuthAPI(abc.ABC):
         existing_user = User.get_by_tgid(user_info.id)
         if existing_user and existing_user != user:
             await existing_user.log_out()
-        asyncio.ensure_future(user.post_login(user_info), loop=self.loop)
+        ignore_coro(asyncio.ensure_future(user.post_login(user_info), loop=self.loop))
         if user.command_status and user.command_status["action"] == "Login":
             user.command_status = None
 
-
     async def post_login_token(self, user: User, token: str) -> web.Response:
         try:
-            user_info = await user.client.sign_in(bot_token=token)
+            user_info = await user.client.sign_in(bot_token=token.strip())
             await self.postprocess_login(user, user_info)
             return self.get_login_response(mxid=user.mxid, state="logged-in", status=200,
                                            username=user_info.username, phone=None,
@@ -173,7 +178,7 @@ class AuthAPI(abc.ABC):
 
     async def post_login_password(self, user: User, password: str) -> web.Response:
         try:
-            user_info = await user.client.sign_in(password=password)
+            user_info = await user.client.sign_in(password=password.strip())
             await self.postprocess_login(user, user_info)
             human_tg_id = f"@{user_info.username}" if user_info.username else f"+{user_info.phone}"
             return self.get_login_response(mxid=user.mxid, state="logged-in", status=200,

mautrix_telegram/web/provisioning/__init__.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -27,7 +27,8 @@ from mautrix_appservice import AppService, MatrixRequestError, IntentError
 from ...types import MatrixUserID, TelegramID
 from ...user import User
 from ...portal import Portal
-from ...commands.portal import user_has_power_level, get_initial_state
+from ...util import ignore_coro
+from ...commands.portal.util import user_has_power_level, get_initial_state
 from ..common import AuthAPI
 
 if TYPE_CHECKING:
@@ -190,8 +191,9 @@ class ProvisioningAPI(AuthAPI):
         portal.photo_id = ""
         portal.save()
 
-        asyncio.ensure_future(portal.update_matrix_room(user, entity, direct, levels=levels),
-                              loop=self.loop)
+        ignore_coro(asyncio.ensure_future(portal.update_matrix_room(user, entity, direct,
+                                                                    levels=levels),
+                                          loop=self.loop))
 
         return web.Response(status=202, body="{}")
 
@@ -245,7 +247,7 @@ class ProvisioningAPI(AuthAPI):
             "group": "chat",
         }[type]
 
-        portal = Portal(tgid=None, mxid=room_id, title=title, about=about, peer_type=type)
+        portal = Portal(tgid=TelegramID(0), mxid=room_id, title=title, about=about, peer_type=type)
         try:
             await portal.create_telegram_chat(user, supergroup=supergroup)
         except ValueError as e:
@@ -285,7 +287,7 @@ class ProvisioningAPI(AuthAPI):
                 self.log.exception("Failed to disconnect chat")
                 return self.get_error_response(500, "exception", "Failed to disconnect chat")
         else:
-            asyncio.ensure_future(coro, loop=self.loop)
+            ignore_coro(asyncio.ensure_future(coro, loop=self.loop))
         return web.json_response({}, status=200 if sync else 202)
 
     async def get_user_info(self, request: web.Request) -> web.Response:
@@ -363,7 +365,7 @@ class ProvisioningAPI(AuthAPI):
 
     async def bridge_info(self, request: web.Request) -> web.Response:
         return web.json_response({
-            "relaybot_username": self.context.bot.username,
+            "relaybot_username": self.context.bot.username if self.context.bot is not None else None,
         }, status=200)
 
     @staticmethod

mautrix_telegram/web/public/__init__.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -87,7 +87,8 @@ class PublicBridgeWebsite(AuthAPI):
         return self.get_login_response(mxid=user.mxid, human_tg_id=user.human_tg_id)
 
     async def get_matrix_login(self, request: web.Request) -> web.Response:
-        mxid = self.verify_token(request.rel_url.query.get("token", None), endpoint="/matrix-login")
+        mxid = self.verify_token(request.rel_url.query.get("token", None),
+                                 endpoint="/matrix-login")
         if not mxid:
             return self.get_mx_login_response(status=401, state="invalid-token")
         user = User.get_by_mxid(mxid, create=False) if mxid else None
@@ -124,7 +125,8 @@ class PublicBridgeWebsite(AuthAPI):
                                                       error=error, message=message, mxid=mxid))
 
     async def post_matrix_login(self, request: web.Request) -> web.Response:
-        mxid = self.verify_token(request.rel_url.query.get("token", None), endpoint="/matrix-login")
+        mxid = self.verify_token(request.rel_url.query.get("token", None),
+                                 endpoint="/matrix-login")
         if not mxid:
             return self.get_mx_login_response(status=401, state="invalid-token")
 
@@ -167,7 +169,13 @@ class PublicBridgeWebsite(AuthAPI):
         elif "bot_token" in data:
             return await self.post_login_token(user, data["bot_token"])
         elif "code" in data:
-            resp = await self.post_login_code(user, data["code"],
+            try:
+                code = int(data["code"].strip())
+            except ValueError:
+                return self.get_login_response(mxid=user.mxid, state="code", status=400,
+                                               errcode="phone_code_invalid",
+                                               error="Phone code must be a number.")
+            resp = await self.post_login_code(user, code,
                                               password_in_data="password" in data)
             if resp or "password" not in data:
                 return resp

mautrix_telegram/web/public/login.css

@@ -1,6 +1,6 @@
 /*
  * mautrix-telegram - A Matrix-Telegram puppeting bridge
- * Copyright (C) 2018 Tulir Asokan
+ * Copyright (C) 2019 Tulir Asokan
  *
  * This program is free software: you can redistribute it and/or modify
  * it under the terms of the GNU Affero General Public License as published by

mautrix_telegram/web/public/login.html.mako

@@ -1,6 +1,6 @@
 <!--
 mautrix-telegram - A Matrix-Telegram puppeting bridge
-Copyright (C) 2018 Tulir Asokan
+Copyright (C) 2019 Tulir Asokan
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU Affero General Public License as published by
@@ -16,7 +16,7 @@ You should have received a copy of the GNU Affero General Public License
 along with this program.  If not, see <https://www.gnu.org/licenses/>.
 -->
 <!DOCTYPE html>
-<html>
+<html lang="en">
 <head>
 	<title>Login - Mautrix-Telegram bridge</title>
 	<link rel="icon" type="image/png" href="favicon.png"/>
@@ -25,9 +25,10 @@ along with this program.  If not, see <https://www.gnu.org/licenses/>.
 	<meta property="og:image" content="favicon.png">
 	<meta charset="utf-8">
 	<link rel="stylesheet" href="//fonts.googleapis.com/css?family=Roboto:300,700">
-	<link rel="stylesheet" href="//cdn.rawgit.com/necolas/normalize.css/master/normalize.css">
 	<link rel="stylesheet"
-		  href="//cdn.rawgit.com/milligram/milligram/master/dist/milligram.min.css">
+		  href="https://cdnjs.cloudflare.com/ajax/libs/normalize/8.0.1/normalize.min.css">
+	<link rel="stylesheet"
+		  href="https://cdnjs.cloudflare.com/ajax/libs/milligram/1.3.0/milligram.min.css">
 	<link rel="stylesheet" href="login.css"/>
 
 	<script>
@@ -94,13 +95,14 @@ along with this program.  If not, see <https://www.gnu.org/licenses/>.
 				% if state == "request":
 					<label for="value">Phone number</label>
 					<input type="tel" id="value" name="phone" placeholder="Enter phone number"/>
-					<button type="submit">Request code</button>
-					<button class="button-clear" type="button" onclick="switchToBotLogin()">
+					<button type="submit">Start</button>
+					<button class="button-clear float-right" type="button" onclick="switchToBotLogin()">
 						Use bot token
 					</button>
 				% elif state == "bot_token":
 					<label for="value">Bot token</label>
-					<input type="text" id="value" name="bot_token" placeholder="Enter bot API token"/>
+					<input type="text" id="value" name="bot_token"
+						   placeholder="Enter bot API token"/>
 					<button type="submit">Sign in</button>
 				% elif state == "code":
 					<label for="value">Phone code</label>

mautrix_telegram/web/public/matrix-login.html.mako

@@ -1,6 +1,6 @@
 <!--
 mautrix-telegram - A Matrix-Telegram puppeting bridge
-Copyright (C) 2018 Tulir Asokan
+Copyright (C) 2019 Tulir Asokan
 
 This program is free software: you can redistribute it and/or modify
 it under the terms of the GNU Affero General Public License as published by
@@ -16,7 +16,7 @@ You should have received a copy of the GNU Affero General Public License
 along with this program.  If not, see <https://www.gnu.org/licenses/>.
 -->
 <!DOCTYPE html>
-<html>
+<html lang="en">
 <head>
 	<title>Matrix login - Mautrix-Telegram bridge</title>
 	<link rel="icon" type="image/png" href="favicon.png"/>
@@ -25,9 +25,10 @@ along with this program.  If not, see <https://www.gnu.org/licenses/>.
 	<meta property="og:image" content="favicon.png">
 	<meta charset="utf-8">
 	<link rel="stylesheet" href="//fonts.googleapis.com/css?family=Roboto:300,700">
-	<link rel="stylesheet" href="//cdn.rawgit.com/necolas/normalize.css/master/normalize.css">
 	<link rel="stylesheet"
-		  href="//cdn.rawgit.com/milligram/milligram/master/dist/milligram.min.css">
+		  href="https://cdnjs.cloudflare.com/ajax/libs/normalize/8.0.1/normalize.min.css">
+	<link rel="stylesheet"
+		  href="https://cdnjs.cloudflare.com/ajax/libs/milligram/1.3.0/milligram.min.css">
 	<link rel="stylesheet" href="login.css"/>
 </head>
 <body>

optional-requirements.txt

@@ -1,4 +1,4 @@
-lxml
 cryptg
 Pillow
 moviepy
+prometheus-client

setup.cfg

@@ -0,0 +1,2 @@
+[aliases]
+test=pytest

setup.py

@@ -3,13 +3,17 @@ import glob
 import mautrix_telegram
 
 extras = {
-    "highlight_edits": ["lxml>=4.1.1,<5"],
-    "better_formatter": ["lxml>=4.1.1,<5"],
-    "fast_crypto": ["cryptg>=0.1,<0.2"],
-    "webp_convert": ["Pillow>=5.0.0,<6"],
-    "hq_thumbnails": ["moviepy>=0.2,<0.3"],
+    "fast_crypto": ["cryptg>=0.1,<0.3"],
+    "webp_convert": ["Pillow>=4.3.0,<7"],
+    "hq_thumbnails": ["moviepy>=1.0,<2.0"],
+    "metrics": ["prometheus-client>=0.6.0,<0.7.0"],
 }
-extras["all"] = list(set(deps[0] for deps in extras.values()))
+extras["all"] = list({dep for deps in extras.values() for dep in deps})
+
+try:
+    long_desc = open("README.md").read()
+except IOError:
+    long_desc = "Failed to read README.md"
 
 setuptools.setup(
     name="mautrix-telegram",
@@ -20,25 +24,28 @@ setuptools.setup(
     author_email="tulir@maunium.net",
 
     description="A Matrix-Telegram hybrid puppeting/relaybot bridge.",
-    long_description=open("README.md").read(),
+    long_description=long_desc,
     long_description_content_type="text/markdown",
 
     packages=setuptools.find_packages(),
 
     install_requires=[
         "aiohttp>=3.0.1,<4",
-        "mautrix-appservice>=0.3.7,<0.4.0",
+        "mautrix-appservice>=0.3.11,<0.4.0",
         "SQLAlchemy>=1.2.3,<2",
         "alembic>=1.0.0,<2",
         "commonmark>=0.8.1,<1",
         "ruamel.yaml>=0.15.35,<0.16",
         "future-fstrings>=0.4.2",
         "python-magic>=0.4.15,<0.5",
-        "telethon>=1.0,<1.3",
-        "telethon-session-sqlalchemy>=0.2.3,<0.3",
+        "telethon>=1.7,<1.9",
+        "telethon-session-sqlalchemy>=0.2.14,<0.3",
     ],
     extras_require=extras,
 
+    setup_requires=["pytest-runner"],
+    tests_require=["pytest", "pytest-asyncio", "pytest-mock"],
+
     classifiers=[
         "Development Status :: 4 - Beta",
         "License :: OSI Approved :: GNU Affero General Public License v3 or later (AGPLv3+)",
@@ -62,4 +69,3 @@ setuptools.setup(
         ("alembic/versions", glob.glob("alembic/versions/*.py"))
     ],
 )
-

tests/commands/test_handler.py

@@ -0,0 +1,370 @@
+from typing import Tuple
+from unittest.mock import Mock
+
+import pytest
+from _pytest.fixtures import FixtureRequest
+from pytest_mock import MockFixture
+
+import mautrix_telegram.commands.handler
+from mautrix_telegram.commands.handler import (CommandEvent, CommandHandler, CommandProcessor,
+                                               HelpSection)
+from mautrix_telegram.config import Config
+from mautrix_telegram.context import Context
+from mautrix_telegram.types import MatrixEventID, MatrixRoomID, MatrixUserID
+import mautrix_telegram.user as u
+
+from tests.utils.helpers import AsyncMock, list_true_once_each
+
+
+@pytest.fixture
+def context(request: FixtureRequest) -> Context:
+    """Returns a Context with mocked Attributes.
+
+    Uses the attribute cls.config as Config.
+    """
+    # Config(path, registration_path, base_path)
+    config = getattr(request.cls, 'config', Config("", "", ""))
+    return Context(az=Mock(), config=config, loop=Mock(), session_container=Mock(), bot=Mock())
+
+
+@pytest.fixture
+def command_processor(context: Context) -> CommandProcessor:
+    """Returns a mocked CommandProcessor."""
+    return CommandProcessor(context)
+
+
+class TestCommandEvent:
+    config = Config("", "", "")
+    config["bridge.command_prefix"] = "tg"
+    config["bridge.permissions"] = {"*": "noperm"}
+
+    def test_reply(
+        self, command_processor: CommandProcessor, mocker: MockFixture
+    ) -> None:
+        mocker.patch("mautrix_telegram.user.config", self.config)
+
+        evt = CommandEvent(
+            processor=command_processor,
+            room=MatrixRoomID("#mock_room:example.org"),
+            event=MatrixEventID("$H45H:example.org"),
+            sender=u.User(MatrixUserID("@sender:example.org")),
+            command="help",
+            args=[],
+            is_management=True,
+            is_portal=False,
+        )
+
+        mock_az = command_processor.az
+
+        message = "**This** <i>was</i><br/><strong>all</strong>fun*!"
+
+        # html, no markdown
+        evt.reply(message, allow_html=True, render_markdown=False)
+        mock_az.intent.send_notice.assert_called_with(
+            MatrixRoomID("#mock_room:example.org"),
+            "**This** <i>was</i><br/><strong>all</strong>fun*!",
+            html="**This** <i>was</i><br/><strong>all</strong>fun*!\n",
+        )
+
+        # html, markdown (default)
+        evt.reply(message, allow_html=True, render_markdown=True)
+        mock_az.intent.send_notice.assert_called_with(
+            MatrixRoomID("#mock_room:example.org"),
+            "**This** <i>was</i><br/><strong>all</strong>fun*!",
+            html=(
+                "<p><strong>This</strong> <i>was</i><br/>"
+                "<strong>all</strong>fun*!</p>\n"
+            ),
+        )
+
+        # no html, no markdown
+        evt.reply(message, allow_html=False, render_markdown=False)
+        mock_az.intent.send_notice.assert_called_with(
+            MatrixRoomID("#mock_room:example.org"),
+            "**This** <i>was</i><br/><strong>all</strong>fun*!",
+            html=None,
+        )
+
+        # no html, markdown
+        evt.reply(message, allow_html=False, render_markdown=True)
+        mock_az.intent.send_notice.assert_called_with(
+            MatrixRoomID("#mock_room:example.org"),
+            "**This** <i>was</i><br/><strong>all</strong>fun*!",
+            html="<p><strong>This</strong> &lt;i&gt;was&lt;/i&gt;&lt;br/&gt;"
+                 "&lt;strong&gt;all&lt;/strong&gt;fun*!</p>\n"
+        )
+
+    def test_reply_with_cmdprefix(self, command_processor: CommandProcessor, mocker: MockFixture
+                                  ) -> None:
+        mocker.patch("mautrix_telegram.user.config", self.config)
+
+        evt = CommandEvent(
+            processor=command_processor,
+            room=MatrixRoomID("#mock_room:example.org"),
+            event=MatrixEventID("$H45H:example.org"),
+            sender=u.User(MatrixUserID("@sender:example.org")),
+            command="help",
+            args=[],
+            is_management=False,
+            is_portal=False,
+        )
+
+        mock_az = command_processor.az
+
+        evt.reply("$cmdprefix+sp ....$cmdprefix+sp...$cmdprefix $cmdprefix", allow_html=False,
+                  render_markdown=False)
+
+        mock_az.intent.send_notice.assert_called_with(
+            MatrixRoomID("#mock_room:example.org"),
+            "tg ....tg+sp...tg tg",
+            html=None,
+        )
+
+    def test_reply_with_cmdprefix_in_management_room(self, command_processor: CommandProcessor,
+                                                     mocker: MockFixture) -> None:
+        mocker.patch("mautrix_telegram.user.config", self.config)
+
+        evt = CommandEvent(
+            processor=command_processor,
+            room=MatrixRoomID("#mock_room:example.org"),
+            event=MatrixEventID("$H45H:example.org"),
+            sender=u.User(MatrixUserID("@sender:example.org")),
+            command="help",
+            args=[],
+            is_management=True,
+            is_portal=False,
+        )
+
+        mock_az = command_processor.az
+
+        evt.reply(
+            "$cmdprefix+sp ....$cmdprefix+sp...$cmdprefix $cmdprefix",
+            allow_html=True,
+            render_markdown=True,
+        )
+
+        mock_az.intent.send_notice.assert_called_with(
+            MatrixRoomID("#mock_room:example.org"),
+            "....tg+sp...tg tg",
+            html="<p>....tg+sp...tg tg</p>\n",
+        )
+
+
+class TestCommandHandler:
+    config = Config("", "", "")
+    config["bridge.permissions"] = {"*": "noperm"}
+
+    @pytest.mark.parametrize(
+        (
+            "needs_auth,"
+            "needs_puppeting,"
+            "needs_matrix_puppeting,"
+            "needs_admin,"
+            "management_only,"
+        ),
+        [l for l in list_true_once_each(length=5)]
+    )
+    @pytest.mark.asyncio
+    async def test_permissions_denied(
+        self,
+        needs_auth: bool,
+        needs_puppeting: bool,
+        needs_matrix_puppeting: bool,
+        needs_admin: bool,
+        management_only: bool,
+        command_processor: CommandProcessor,
+        boolean: bool,
+        mocker: MockFixture,
+    ) -> None:
+        mocker.patch("mautrix_telegram.user.config", self.config)
+
+        command = "testcmd"
+
+        mock_handler = Mock()
+
+        command_handler = CommandHandler(
+            handler=mock_handler,
+            needs_auth=needs_auth,
+            needs_puppeting=needs_puppeting,
+            needs_matrix_puppeting=needs_matrix_puppeting,
+            needs_admin=needs_admin,
+            management_only=management_only,
+            name=command,
+            help_text="No real command",
+            help_args="mock mockmock",
+            help_section=HelpSection("Mock Section", 42, ""),
+        )
+
+        sender = u.User(MatrixUserID("@sender:example.org"))
+        sender.puppet_whitelisted = False
+        sender.matrix_puppet_whitelisted = False
+        sender.is_admin = False
+
+        event = CommandEvent(
+            processor=command_processor,
+            room=MatrixRoomID("#mock_room:example.org"),
+            event=MatrixEventID("$H45H:example.org"),
+            sender=sender,
+            command=command,
+            args=[],
+            is_management=False,
+            is_portal=boolean,
+        )
+
+        assert await command_handler.get_permission_error(event)
+        assert not command_handler.has_permission(False, False, False, False, False)
+
+    @pytest.mark.parametrize(
+        (
+            "is_management,"
+            "puppet_whitelisted,"
+            "matrix_puppet_whitelisted,"
+            "is_admin,"
+            "is_logged_in,"
+        ),
+        [l for l in list_true_once_each(length=5)]
+    )
+    @pytest.mark.asyncio
+    async def test_permission_granted(
+        self,
+        is_management: bool,
+        puppet_whitelisted: bool,
+        matrix_puppet_whitelisted: bool,
+        is_admin: bool,
+        is_logged_in: bool,
+        command_processor: CommandProcessor,
+        boolean: bool,
+        mocker: MockFixture,
+    ) -> None:
+        mocker.patch("mautrix_telegram.user.config", self.config)
+
+        command = "testcmd"
+
+        mock_handler = Mock()
+
+        command_handler = CommandHandler(
+            handler=mock_handler,
+            needs_auth=False,
+            needs_puppeting=False,
+            needs_matrix_puppeting=False,
+            needs_admin=False,
+            management_only=False,
+            name=command,
+            help_text="No real command",
+            help_args="mock mockmock",
+            help_section=HelpSection("Mock Section", 42, ""),
+        )
+
+        sender = u.User(MatrixUserID("@sender:example.org"))
+        sender.puppet_whitelisted = puppet_whitelisted
+        sender.matrix_puppet_whitelisted = matrix_puppet_whitelisted
+        sender.is_admin = is_admin
+        mocker.patch.object(u.User, 'is_logged_in', return_value=is_logged_in)
+
+        event = CommandEvent(
+            processor=command_processor,
+            room=MatrixRoomID("#mock_room:example.org"),
+            event=MatrixEventID("$H45H:example.org"),
+            sender=sender,
+            command=command,
+            args=[],
+            is_management=is_management,
+            is_portal=boolean,
+        )
+
+        assert not await command_handler.get_permission_error(event)
+        assert command_handler.has_permission(
+            is_management=is_management,
+            puppet_whitelisted=puppet_whitelisted,
+            matrix_puppet_whitelisted=matrix_puppet_whitelisted,
+            is_admin=is_admin,
+            is_logged_in=is_logged_in,
+        )
+
+
+class TestCommandProcessor:
+    config = Config("", "", "")
+    config["bridge.command_prefix"] = "tg"
+    config["bridge.permissions"] = {"*": "relaybot"}
+
+    @pytest.mark.asyncio
+    async def test_handle(self, command_processor: CommandProcessor, boolean2: Tuple[bool, bool],
+                          mocker: MockFixture) -> None:
+        mocker.patch('mautrix_telegram.user.config', self.config)
+        mocker.patch(
+            'mautrix_telegram.commands.handler.command_handlers',
+            {"help": AsyncMock(), "unknown-command": AsyncMock()}
+        )
+
+        sender = u.User(MatrixUserID("@sender:example.org"))
+
+        result = await command_processor.handle(
+            room=MatrixRoomID("#mock_room:example.org"),
+            event_id=MatrixEventID("$H45H:example.org"),
+            sender=sender,
+            command="hElp",
+            args=[],
+            is_management=boolean2[0],
+            is_portal=boolean2[1],
+        )
+
+        assert result is None
+        command_handlers = mautrix_telegram.commands.handler.command_handlers
+        command_handlers["help"].mock.assert_called_once()  # type: ignore
+
+    @pytest.mark.asyncio
+    async def test_handle_unknown_command(self, command_processor: CommandProcessor,
+                                          boolean2: Tuple[bool, bool], mocker: MockFixture) -> None:
+        mocker.patch('mautrix_telegram.user.config', self.config)
+        mocker.patch(
+            'mautrix_telegram.commands.handler.command_handlers',
+            {"help": AsyncMock(), "unknown-command": AsyncMock()}
+        )
+
+        sender = u.User(MatrixUserID("@sender:example.org"))
+        sender.command_status = {}
+
+        result = await command_processor.handle(
+            room=MatrixRoomID("#mock_room:example.org"),
+            event_id=MatrixEventID("$H45H:example.org"),
+            sender=sender,
+            command="foo",
+            args=[],
+            is_management=boolean2[0],
+            is_portal=boolean2[1],
+        )
+
+        assert result is None
+        command_handlers = mautrix_telegram.commands.handler.command_handlers
+        command_handlers["help"].mock.assert_not_called()  # type: ignore
+        command_handlers["unknown-command"].mock.assert_called_once()  # type: ignore
+
+    @pytest.mark.asyncio
+    async def test_handle_delegated_handler(self, command_processor: CommandProcessor,
+                                            boolean2: Tuple[bool, bool],
+                                            mocker: MockFixture) -> None:
+        mocker.patch('mautrix_telegram.user.config', self.config)
+        mocker.patch(
+            'mautrix_telegram.commands.handler.command_handlers',
+            {"help": AsyncMock(), "unknown-command": AsyncMock()}
+        )
+
+        sender = u.User(MatrixUserID("@sender:example.org"))
+        sender.command_status = {"foo": AsyncMock(), "next": AsyncMock()}
+
+        result = await command_processor.handle(
+            room=MatrixRoomID("#mock_room:example.org"),
+            event_id=MatrixEventID("$H45H:example.org"),
+            sender=sender,  # u.User
+            command="foo",
+            args=[],
+            is_management=boolean2[0],
+            is_portal=boolean2[1]
+        )
+
+        assert result is None
+        command_handlers = mautrix_telegram.commands.handler.command_handlers
+        command_handlers["help"].mock.assert_not_called()  # type: ignore
+        command_handlers["unknown-command"].mock.assert_not_called()  # type: ignore
+        sender.command_status["foo"].mock.assert_not_called()  # type: ignore
+        sender.command_status["next"].mock.assert_called_once()  # type: ignore

tests/conftest.py

@@ -0,0 +1,3 @@
+pytest_plugins = [
+    "tests.utils.fixtures",
+]

tests/utils/fixtures.py

@@ -0,0 +1,27 @@
+"""This module provides utility fixtures for testing."""
+from typing import Tuple
+
+from _pytest.fixtures import FixtureRequest
+import pytest
+
+
+@pytest.fixture(params=[True, False])
+def boolean(request: FixtureRequest) -> bool:
+    return request.param
+
+
+@pytest.fixture
+def boolean1(boolean: bool) -> Tuple[bool]:
+    return boolean,
+
+
+@pytest.fixture(params=[True, False])
+def boolean2(request: FixtureRequest, boolean: bool) -> Tuple[bool, bool]:
+    return boolean, request.param
+
+
+@pytest.fixture(params=[True, False])
+def boolean3(request: FixtureRequest, boolean2: Tuple[bool, bool]) -> Tuple[bool, bool, bool]:
+    return boolean2[0], boolean2[1], request.param
+
+# …

tests/utils/helpers.py

@@ -0,0 +1,24 @@
+"""This module provides utility functions for testing."""
+from typing import Generator, Tuple
+from unittest.mock import Mock
+
+
+def AsyncMock(*args, **kwargs):
+    """Mocks a asyncronous coroutine which can be called with 'await'."""
+    m = Mock(*args, **kwargs)
+
+    async def mock_coro(*args, **kwargs):
+        return m(*args, **kwargs)
+
+    mock_coro.mock = m
+    return mock_coro
+
+
+def list_true_once_each(length: int) -> Generator[Tuple[bool, ...], None, None]:
+    """Yields tuples of bools with exactly one entry being True, starting left.
+
+    Args:
+        length: Length of the resulting tuples
+    """
+    for i in range(length):
+        yield tuple(i == j for j in range(length))