Bump version to 0.3.0rc3

* Add pre-send message database check for deduplication
* Make dedup cache queue length configurable

.codeclimate.yml

@@ -0,0 +1,8 @@
+engines:
+  sonar-python:
+    enabled: true
+    checks:
+      python:S107:
+        enabled: false
+exclude_patterns:
+- "alembic/"

.dockerignore

@@ -0,0 +1,4 @@
+.editorconfig
+.codeclimate.yml
+*.png
+*.md

.gitignore

@@ -7,6 +7,5 @@ __pycache__
 
 config.yaml
 registration.yaml
+*.log
 *.db
-*.session
-*.json

Dockerfile

@@ -1,11 +1,14 @@
-FROM docker.io/alpine:3.7
+FROM docker.io/alpine:3.8
 
 ENV UID=1337 \
-    GID=1337
+    GID=1337 \
+    FFMPEG_BINARY=/usr/bin/ffmpeg
 
-COPY . /opt/mautrixtelegram
+COPY . /opt/mautrix-telegram
+WORKDIR /opt/mautrix-telegram
 RUN apk add --no-cache \
       python3-dev \
+      build-base \
       py3-virtualenv \
       py3-pillow \
       py3-aiohttp \
@@ -14,17 +17,13 @@ RUN apk add --no-cache \
       py3-numpy \
       py3-asn1crypto \
       py3-sqlalchemy \
-      build-base \
+      py3-markdown \
+      py3-psycopg2 \
       ffmpeg \
-      bash \
       ca-certificates \
       su-exec \
-      s6 \
- && cd /opt/mautrixtelegram \
- && cp -r docker/root/* / \
- && rm docker -rf \
  && pip3 install -r requirements.txt -r optional-requirements.txt
 
 VOLUME /data
 
-CMD ["/bin/s6-svscan", "/etc/s6.d"]
+CMD ["/opt/mautrix-telegram/docker-run.sh"]

alembic/env.py

@@ -4,10 +4,12 @@ from logging.config import fileConfig
 
 import sys
 from os.path import abspath, dirname
+
 sys.path.insert(0, dirname(dirname(abspath(__file__))))
 
 from mautrix_telegram.base import Base
 from mautrix_telegram.config import Config
+from alchemysession import AlchemySessionContainer
 import mautrix_telegram.db
 
 # this is the Alembic Config object, which provides
@@ -20,6 +22,15 @@ mxtg_config.load()
 config.set_main_option("sqlalchemy.url",
                        mxtg_config.get("appservice.database", "sqlite:///mautrix-telegram.db"))
 
+
+class FakeDB:
+    @staticmethod
+    def query_property():
+        return None
+
+
+AlchemySessionContainer.create_table_classes(FakeDB(), "telethon_", Base)
+
 # Interpret the config file for Python logging.
 # This line sets up loggers basically.
 fileConfig(config.config_file_name)
@@ -30,6 +41,7 @@ fileConfig(config.config_file_name)
 # target_metadata = mymodel.Base.metadata
 target_metadata = Base.metadata
 
+
 # other values from the config, defined by the needs of env.py,
 # can be acquired:
 # my_important_option = config.get_main_option("my_important_option")
@@ -77,6 +89,7 @@ def run_migrations_online():
         with context.begin_transaction():
             context.run_migrations()
 
+
 if context.is_offline_mode():
     run_migrations_offline()
 else:

alembic/versions/2228d49c383f_add_cascade_rules_to_userportal.py

@@ -0,0 +1,41 @@
+"""Add cascade rules to UserPortal
+
+Revision ID: 2228d49c383f
+Revises: bcfefa1f1299
+Create Date: 2018-05-31 11:11:59.482112
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = '2228d49c383f'
+down_revision = 'bcfefa1f1299'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    try:
+        with op.batch_alter_table("user_portal") as batch_op:
+            batch_op.drop_constraint("user_portal_user_fkey", type_="foreignkey")
+            batch_op.drop_constraint("user_portal_portal_fkey", type_="foreignkey")
+            batch_op.create_foreign_key("user_portal_user_fkey", "user", ["user"], ["tgid"],
+                                        onupdate="CASCADE", ondelete="CASCADE")
+            batch_op.create_foreign_key("user_portal_portal_fkey", "portal",
+                                        ["portal", "portal_receiver"], ["tgid", "tg_receiver"],
+                                        onupdate="CASCADE", ondelete="CASCADE")
+    except ValueError:
+        return
+
+
+def downgrade():
+    try:
+        with op.batch_alter_table("user_portal") as batch_op:
+                batch_op.drop_constraint("user_portal_user_fkey", type_="foreignkey")
+                batch_op.drop_constraint("user_portal_portal_fkey", type_="foreignkey")
+                batch_op.create_foreign_key("user_portal_user_fkey", "portal",
+                                            ["portal", "portal_receiver"], ["tgid", "tg_receiver"])
+                batch_op.create_foreign_key("user_portal_portal_fkey", "user", ["user"], ["tgid"])
+    except ValueError:
+        return

alembic/versions/6ca3d74d51e4_move_state_store_to_main_database.py

@@ -0,0 +1,129 @@
+"""Move state store to main database
+
+Revision ID: 6ca3d74d51e4
+Revises: 2228d49c383f
+Create Date: 2018-06-26 21:31:26.911307
+
+"""
+from alembic import context, op
+import sqlalchemy.orm as orm
+import sqlalchemy as sa
+import json
+import re
+
+from mautrix_telegram.config import Config
+from mautrix_telegram.base import Base
+
+# revision identifiers, used by Alembic.
+revision = "6ca3d74d51e4"
+down_revision = "2228d49c383f"
+branch_labels = None
+depends_on = None
+
+
+class RoomState(Base):
+    query = None
+    __tablename__ = "mx_room_state"
+    __table_args__ = {"extend_existing": True}
+
+    room_id = sa.Column(sa.String, primary_key=True)
+    power_levels = sa.Column("power_levels", sa.Text, nullable=True)
+
+
+class UserProfile(Base):
+    query = None
+    __tablename__ = "mx_user_profile"
+    __table_args__ = {"extend_existing": True}
+
+    room_id = sa.Column(sa.String, primary_key=True)
+    user_id = sa.Column(sa.String, primary_key=True)
+    membership = sa.Column(sa.String, nullable=False, default="leave")
+    displayname = sa.Column(sa.String, nullable=True)
+    avatar_url = sa.Column(sa.String, nullable=True)
+
+
+class Puppet(Base):
+    query = None
+    __tablename__ = "puppet"
+    __table_args__ = {"extend_existing": True}
+
+    id = sa.Column(sa.Integer, primary_key=True)
+    displayname = sa.Column(sa.String, nullable=True)
+    displayname_source = sa.Column(sa.Integer, nullable=True)
+    username = sa.Column(sa.String, nullable=True)
+    photo_id = sa.Column(sa.String, nullable=True)
+    is_bot = sa.Column(sa.Boolean, nullable=True)
+    matrix_registered = sa.Column(sa.Boolean, nullable=False, default=False)
+
+
+def upgrade():
+    op.add_column("puppet", sa.Column("matrix_registered", sa.Boolean(), nullable=False,
+                                      server_default=sa.sql.expression.false()))
+    op.create_table("mx_room_state",
+                    sa.Column("room_id", sa.String(), nullable=False),
+                    sa.Column("power_levels", sa.Text(), nullable=True),
+                    sa.PrimaryKeyConstraint("room_id"))
+    op.create_table("mx_user_profile",
+                    sa.Column("room_id", sa.String(), nullable=False),
+                    sa.Column("user_id", sa.String(), nullable=False),
+                    sa.Column("membership", sa.String(), nullable=False,
+                              default="leave"),
+                    sa.Column("displayname", sa.String(), nullable=True),
+                    sa.Column("avatar_url", sa.String(), nullable=True),
+                    sa.PrimaryKeyConstraint("room_id", "user_id"))
+
+    conn = op.get_bind()
+    session = orm.sessionmaker(bind=conn)
+    session = orm.scoping.scoped_session(session)
+    Puppet.query = session.query_property()
+
+    try:
+        with open("mx-state.json") as file:
+            data = json.load(file)
+    except FileNotFoundError:
+        return
+    if not data:
+        return
+    registrations = data.get("registrations", [])
+
+    mxtg_config_path = context.get_x_argument(as_dictionary=True).get("config", "config.yaml")
+    mxtg_config = Config(mxtg_config_path, None, None)
+    mxtg_config.load()
+
+    username_template = mxtg_config.get("bridge.username_template", "telegram_{userid}")
+    hs_domain = mxtg_config["homeserver.domain"]
+    localpart = username_template.format(userid="(.+)")
+    mxid_regex = re.compile("@{}:{}".format(localpart, hs_domain))
+    for user in registrations:
+        match = mxid_regex.match(user)
+        if not match:
+            continue
+
+        puppet = Puppet.query.get(match.group(1))
+        if not puppet:
+            continue
+
+        puppet.matrix_registered = True
+        session.merge(puppet)
+    session.commit()
+
+    user_profiles = [UserProfile(room_id=room, user_id=user,
+                                 membership=member.get("membership", "leave"),
+                                 displayname=member.get("displayname", None),
+                                 avatar_url=member.get("avatar_url", None))
+                     for room, members in data.get("members", {}).items()
+                     for user, member in members.items()]
+    session.add_all(user_profiles)
+    session.commit()
+
+    room_state = [RoomState(room_id=room, power_levels=json.dumps(levels))
+                  for room, levels in data.get("power_levels", {}).items()]
+    session.add_all(room_state)
+    session.commit()
+
+
+def downgrade():
+    op.drop_table("mx_user_profile")
+    op.drop_table("mx_room_state")
+    with op.batch_alter_table("puppet") as batch_op:
+        batch_op.drop_column("matrix_registered")

alembic/versions/7d47d84380b6_add_timestamp_to_telegramfile.py

@@ -17,7 +17,7 @@ depends_on = None
 
 def upgrade():
     op.add_column('telegram_file',
-                  sa.Column('timestamp', sa.BigInteger(), nullable=False, default=0,
+                  sa.Column('timestamp', sa.BigInteger(), nullable=True, default=0,
                             server_default="0"))
 
 

alembic/versions/97d2a942bcf8_initial_revision.py

@@ -50,9 +50,13 @@ def upgrade():
                     sa.Column('portal', sa.Integer),
                     sa.Column('portal_receiver', sa.Integer),
                     sa.PrimaryKeyConstraint('user', 'portal', 'portal_receiver'),
-                    sa.ForeignKeyConstraint(("user",), ("user.tgid",)),
+                    sa.ForeignKeyConstraint(("user",), ("user.tgid",),
+                                            name="user_portal_user_fkey",
+                                            onupdate="CASCADE", ondelete="CASCADE"),
                     sa.ForeignKeyConstraint(("portal", "portal_receiver"),
-                                            ("portal.tgid", "portal.tg_receiver")))
+                                            ("portal.tgid", "portal.tg_receiver"),
+                                            name="user_portal_portal_fkey",
+                                            onupdate="CASCADE", ondelete="CASCADE"))
     op.create_table('message',
                     sa.Column('mxid', sa.String),
                     sa.Column('mx_room', sa.String),

alembic/versions/d5f7b8b4b456_add_access_token_and_custom_mxid_fields_.py

@@ -0,0 +1,26 @@
+"""Add access_token and custom_mxid fields for puppets
+
+Revision ID: d5f7b8b4b456
+Revises: 6ca3d74d51e4
+Create Date: 2018-07-20 12:09:30.277960
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = "d5f7b8b4b456"
+down_revision = "6ca3d74d51e4"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.add_column("puppet", sa.Column("access_token", sa.String(), nullable=True))
+    op.add_column("puppet", sa.Column("custom_mxid", sa.String(), nullable=True))
+
+
+def downgrade():
+    with op.batch_alter_table("puppet") as batch_op:
+        batch_op.drop_column("custom_mxid")
+        batch_op.drop_column("access_token")

docker-run.sh

@@ -1,22 +1,22 @@
-#!/bin/bash
+#!/bin/sh
 
-# Define functions
+# Define functions.
 function fixperms {
-	chown -R ${UID}:${GID} /data /opt/mautrixtelegram
+	chown -R $UID:$GID /data /opt/mautrix-telegram
 }
 
-
-# Go into env
-cd /opt/mautrixtelegram
-export FFMPEG_BINARY=/usr/bin/ffmpeg
+cd /opt/mautrix-telegram
 
 # Replace database path in config.
 sed -i "s#sqlite:///mautrix-telegram.db#sqlite:////data/mautrix-telegram.db#" /data/config.yaml
 
+if [ -f /data/mx-state.json ]; then
+	ln -s /data/mx-state.json
+fi
 # Check that database is in the right state
 alembic -x config=/data/config.yaml upgrade head
 
-if [[ ! -f /data/config.yaml ]]; then
+if [ ! -f /data/config.yaml ]; then
 	cp example-config.yaml /data/config.yaml
 	echo "Didn't find a config file."
 	echo "Copied default config file to /data/config.yaml"
@@ -26,14 +26,14 @@ if [[ ! -f /data/config.yaml ]]; then
 	exit
 fi
 
-if [[ ! -f /data/registration.yaml ]]; then
+if [ ! -f /data/registration.yaml ]; then
 	python3 -m mautrix_telegram -g -c /data/config.yaml -r /data/registration.yaml
 	echo "Didn't find a registration file."
-	echo "Generated ode for you."
+	echo "Generated one for you."
 	echo "Copy that over to synapses app service directory."
 	fixperms
 	exit
 fi
 
 fixperms
-exec su-exec ${UID}:${GID} python3 -m mautrix_telegram -c /data/config.yaml
+exec su-exec $UID:$GID python3 -m mautrix_telegram -c /data/config.yaml

docker/root/etc/s6.d/.s6-svscan/finish

@@ -1 +0,0 @@
-#!/bin/sh

docker/root/etc/s6.d/mautrix-telegram/finish

@@ -1,2 +0,0 @@
-#!/bin/bash
-s6-svscanctl -t /etc/s6.d

example-config.yaml

@@ -11,15 +11,18 @@ homeserver:
 # Application service host/registration related details
 # Changing these values requires regeneration of the registration.
 appservice:
-    # The protocol the homeserver should use when connecting to this appservice.
-    # Usually "http" or "https".
-    protocol: http
+    # The address that the homeserver can use to connect to this appservice.
+    address: http://localhost:8080
 
-    # The hostname and port where the homeserver can find this appservice.
-    hostname: localhost
+    # The hostname and port where this appservice should listen.
+    hostname: 0.0.0.0
     port: 8080
+    # The maximum body size of appservice API requests (from the homeserver) in mebibytes
+    # Usually 1 is enough, but on high-traffic bridges you might need to increase this to avoid 413s
+    max_body_size: 1
 
-    # The full URI to the database.
+    # The full URI to the database. SQLite and Postgres are fully supported.
+    # Other DBMSes supported by SQLAlchemy may or may not work.
     database: sqlite:///mautrix-telegram.db
 
     # Public part of web server for out-of-Matrix interaction with the bridge.
@@ -34,14 +37,25 @@ appservice:
         # implicitly.
         external: https://example.com/public
 
-    # Whether or not to enable debug messages in the console.
-    debug: true
+    # Provisioning API part of the web server for automated portal creation and fetching information.
+    # Used by things like Dimension (https://dimension.t2bot.io/).
+    provisioning:
+        # Whether or not the provisioning API should be enabled.
+        enabled: true
+        # The prefix to use in the provisioning API endpoints.
+        prefix: /_matrix/provision/v1
+        # The shared secret to authorize users of the API.
+        # Set to "generate" to generate and save a new token.
+        shared_secret: generate
 
     # The unique ID of this appservice.
     id: telegram
     # Username of the appservice bot.
     bot_username: telegrambot
+    # Display name and avatar for bot. Set to "remove" to remove display name/avatar, leave empty
+    # to leave display name/avatar as-is.
     bot_displayname: Telegram bridge bot
+    bot_avatar: mxc://maunium.net/tJCRmUyJDsgRNgqhOgoiHWbX
 
     # Authentication tokens for AS <-> HS communication. Autogenerated; do not modify.
     as_token: "This value is generated when generating the registration"
@@ -92,6 +106,10 @@ bridge:
     # will not send any more members.
     # Defaults to no local limit (-> limited to 10000 by server)
     max_initial_member_sync: -1
+    # Whether or not to sync the member list in channels.
+    # If no channel admins have logged into the bridge, the bridge won't be able to sync the member
+    # list regardless of this setting.
+    sync_channel_members: true
     # The maximum number of simultaneous Telegram deletions to handle.
     # A large number of simultaneous redactions could put strain on your homeserver.
     max_telegram_delete: 10
@@ -114,12 +132,58 @@ bridge:
     # Whether or not to fetch and handle Telegram updates at startup from the time the bridge was down.
     # WARNING: Probably buggy, might get stuck in infinite loop.
     catch_up: false
+    # Whether or not to use /sync to get presence, read receipts and typing notifications when using
+    # your own Matrix account as the Matrix puppet for your Telegram account.
+    sync_with_custom_puppets: true
 
+    # Some config options related to Telegram message deduplication.
+    # The default values are usually fine, but some debug messages/warnings might recommend you
+    # change these.
+    deduplication:
+        # Whether or not to check the database if the message about to be sent is a duplicate.
+        pre_db_check: false
+        # The number of latest events to keep when checking for duplicates.
+        # You might need to increase this on high-traffic bridge instances.
+        cache_queue_length: 20
+
+    # The formats to use when sending messages to Telegram via the relay bot.
+    #
+    # Telegram doesn't have built-in emotes, so the m.emote format is also used for non-relaybot users.
+    #
+    # Available variables:
+    #   $sender_displayname    - The display name of the sender (e.g. Example User)
+    #   $sender_username       - The username (Matrix ID localpart) of the sender (e.g. exampleuser)
+    #   $sender_mxid           - The Matrix ID of the sender (e.g. @exampleuser:example.com)
+    #   $message               - The message content as HTML
+    message_formats:
+        m.text: "<b>$sender_displayname</b>: $message"
+        m.emote: "* <b>$sender_displayname</b> $message"
+        m.file: "<b>$sender_displayname</b> sent a file: $message"
+        m.image: "<b>$sender_displayname</b> sent an image: $message"
+        m.audio: "<b>$sender_displayname</b> sent an audio file: $message"
+        m.video: "<b>$sender_displayname</b> sent a video: $message"
+        m.location: "<b>$sender_displayname</b> sent a location: $message"
+
+    # The formats to use when sending state events to Telegram via the relay bot.
+    #
+    # Variables from `message_formats` that have the `sender_` prefix are available without the prefix.
+    # In name_change events, `$prev_displayname` is the previous displayname.
+    #
+    # Set format to an empty string to disable the messages for that event.
+    state_event_formats:
+        join: "<b>$displayname</b> joined the room."
+        leave: "<b>$displayname</b> left the room."
+        name_change: "<b>$prev_displayname</b> changed their name to <b>$displayname</b>"
+
+    # Filter rooms that can/can't be bridged. Can also be managed using the `filter` and
+    # `filter-mode` management commands.
+    #
+    # Filters do not affect direct chats.
+    # An empty blacklist will essentially disable the filter.
     filter:
         # Filter mode to use. Either "blacklist" or "whitelist".
-        # If the mode is "blacklist", the listed chats will never be bridged. An empty blacklist disables the filter.
+        # If the mode is "blacklist", the listed chats will never be bridged.
         # If the mode is "whitelist", only the listed chats can be bridged.
-        # Direct chats are not affected.
         mode: blacklist
         # The list of group/channel IDs to filter.
         list: []
@@ -130,7 +194,9 @@ bridge:
     # Permissions for using the bridge.
     # Permitted values:
     #   relaybot - Only use the bridge via the relaybot, no access to commands.
-    #       full - Full access to use the bridge via relaybot or logging in with Telegram account.
+    #       user - Relaybot level + access to commands to create bridges.
+    #  puppeting - User level + logging in with a Telegram account.
+    #       full - Full access to use the bridge, i.e. previous levels + Matrix login.
     #      admin - Full access to use the bridge and some extra administration commands.
     # Permitted keys:
     #        * - All Matrix users
@@ -138,8 +204,8 @@ bridge:
     #     mxid - Specific user
     permissions:
         "*": "relaybot"
+        "public.example.com": "user"
         "example.com": "full"
-        "public.example.com": "full"
         "@admin:example.com": "admin"
 
     # Options related to the message relay Telegram bot.
@@ -148,6 +214,8 @@ bridge:
         authless_portals: true
         # Whether or not to allow Telegram group admins to use the bot commands.
         whitelist_group_admins: true
+        # Whether or not to ignore incoming events sent by the relay bot.
+        ignore_own_incoming_events: true
         # List of usernames/user IDs who are also allowed to use the bot commands.
         whitelist:
         - myusername
@@ -160,3 +228,46 @@ telegram:
     api_hash: tjyd5yge35lbodk1xwzw2jstp90k55qz
     # (Optional) Create your own bot at https://t.me/BotFather
     bot_token: disabled
+    # Telethon proxy configuration.
+    # You must install PySocks from pip for proxies to work.
+    proxy:
+        # Allowed types: disabled, socks4, socks5, http
+        type: disabled
+        # Proxy IP address and port.
+        address: 127.0.0.1
+        port: 1080
+        # Whether or not to perform DNS resolving remotely.
+        rdns: true
+        # Proxy authentication (optional).
+        username: ""
+        password: ""
+
+# Python logging configuration.
+#
+# See section 16.7.2 of the Python documentation for more info:
+# https://docs.python.org/3.6/library/logging.config.html#configuration-dictionary-schema
+logging:
+    version: 1
+    formatters:
+        precise:
+            format: "[%(asctime)s] [%(levelname)s@%(name)s] %(message)s"
+    handlers:
+        file:
+            class: logging.handlers.RotatingFileHandler
+            formatter: precise
+            filename: ./mautrix-telegram.log
+            maxBytes: 10485760
+            backupCount: 10
+        console:
+            class: logging.StreamHandler
+            formatter: precise
+    loggers:
+        mau:
+            level: DEBUG
+        telethon:
+            level: DEBUG
+        aiohttp:
+            level: INFO
+    root:
+        level: DEBUG
+        handlers: [file, console]

mautrix_telegram/__init__.py

@@ -1,2 +1,2 @@
-__version__ = "0.2.0rc4"
+__version__ = "0.3.0rc3"
 __author__ = "Tulir Asokan <tulir@maunium.net>"

mautrix_telegram/__main__.py

@@ -14,36 +14,33 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Optional
 import argparse
-import sys
-import logging
 import asyncio
+import logging.config
+import sys
 
-import sqlalchemy as sql
 from sqlalchemy import orm
+import sqlalchemy as sql
 
-from alchemysession import AlchemySessionContainer
 from mautrix_appservice import AppService
+from alchemysession import AlchemySessionContainer
 
-from .base import Base
-from .config import Config
-from .matrix import MatrixHandler
-
-from .db import init as init_db
+from .web.provisioning import ProvisioningAPI
+from .web.public import PublicBridgeWebsite
 from .abstract_user import init as init_abstract_user
-from .user import init as init_user, User
+from .base import Base
 from .bot import init as init_bot
+from .config import Config
+from .context import Context
+from .db import init as init_db
+from .formatter import init as init_formatter
+from .matrix import MatrixHandler
 from .portal import init as init_portal
 from .puppet import init as init_puppet
-from .formatter import init as init_formatter
-from .public import PublicBridgeWebsite
-from .context import Context
-
-log = logging.getLogger("mau")
-time_formatter = logging.Formatter("[%(asctime)s] [%(levelname)s@%(name)s] %(message)s")
-handler = logging.StreamHandler()
-handler.setFormatter(time_formatter)
-log.addHandler(handler)
+from .sqlstatestore import SQLStateStore
+from .user import User, init as init_user
+from . import __version__
 
 parser = argparse.ArgumentParser(
     description="A Matrix-Telegram puppeting bridge.",
@@ -69,34 +66,44 @@ if args.generate_registration:
     print(f"Registration generated and saved to {config.registration_path}")
     sys.exit(0)
 
-if config["appservice.debug"]:
-    telethon_log = logging.getLogger("telethon")
-    telethon_log.addHandler(handler)
-    telethon_log.setLevel(logging.DEBUG)
-    log.setLevel(logging.DEBUG)
-    log.debug("Debug messages enabled.")
+logging.config.dictConfig(config["logging"])
+log = logging.getLogger("mau.init")  # type: logging.Logger
+log.debug(f"Initializing mautrix-telegram {__version__}")
 
-db_engine = sql.create_engine(config.get("appservice.database", "sqlite:///mautrix-telegram.db"))
+db_engine = sql.create_engine(config["appservice.database"] or "sqlite:///mautrix-telegram.db")
 db_factory = orm.sessionmaker(bind=db_engine)
 db_session = orm.scoping.scoped_session(db_factory)
 Base.metadata.bind = db_engine
 
-telethon_session_container = AlchemySessionContainer(engine=db_engine, session=db_session,
-                                                     table_base=Base, table_prefix="telethon_",
-                                                     manage_tables=False)
+session_container = AlchemySessionContainer(engine=db_engine, session=db_session,
+                                            table_base=Base, table_prefix="telethon_",
+                                            manage_tables=False)
 
-loop = asyncio.get_event_loop()
+loop = asyncio.get_event_loop()  # type: asyncio.AbstractEventLoop
 
+state_store = SQLStateStore(db_session)
+mebibyte = 1024 ** 2
 appserv = AppService(config["homeserver.address"], config["homeserver.domain"],
                      config["appservice.as_token"], config["appservice.hs_token"],
                      config["appservice.bot_username"], log="mau.as", loop=loop,
-                     verify_ssl=config["homeserver.verify_ssl"])
+                     verify_ssl=config["homeserver.verify_ssl"], state_store=state_store,
+                     real_user_content_key="net.maunium.telegram.puppet",
+                     aiohttp_params={
+                         "client_max_size": config["appservice.max_body_size"] * mebibyte
+                     })
 
-context = Context(appserv, db_session, config, loop, None, None, telethon_session_container)
+context = Context(appserv, db_session, config, loop, session_container)
 
 if config["appservice.public.enabled"]:
-    public = PublicBridgeWebsite(loop)
-    appserv.app.add_subapp(config.get("appservice.public.prefix", "/public"), public.app)
+    public_website = PublicBridgeWebsite(loop)
+    appserv.app.add_subapp(config["appservice.public.prefix"] or "/public", public_website.app)
+    context.public_website = public_website
+
+if config["appservice.provisioning.enabled"]:
+    provisioning_api = ProvisioningAPI(context)
+    appserv.app.add_subapp(config["appservice.provisioning.prefix"] or "/_matrix/provisioning",
+                           provisioning_api.app)
+    context.provisioning_api = provisioning_api
 
 with appserv.run(config["appservice.hostname"], config["appservice.port"]) as start:
     init_db(db_session)
@@ -105,16 +112,22 @@ with appserv.run(config["appservice.hostname"], config["appservice.port"]) as st
     context.mx = MatrixHandler(context)
     init_formatter(context)
     init_portal(context)
-    init_puppet(context)
-    startup_actions = init_user(context) + [start, context.mx.init_as_bot()]
+    startup_actions = (init_puppet(context) +
+                       init_user(context) +
+                       [start,
+                        context.mx.init_as_bot()])
 
     if context.bot:
         startup_actions.append(context.bot.start())
 
     try:
+        log.debug("Initialization complete, running startup actions")
         loop.run_until_complete(asyncio.gather(*startup_actions, loop=loop))
+        log.debug("Startup actions complete, now running forever")
         loop.run_forever()
     except KeyboardInterrupt:
-        for user in User.by_tgid.values():
-            user.stop()
+        log.debug("Keyboard interrupt received, stopping clients")
+        loop.run_until_complete(
+            asyncio.gather(*[user.stop() for user in User.by_tgid.values()], loop=loop))
+        log.debug("Clients stopped, shutting down")
         sys.exit(0)

mautrix_telegram/abstract_user.py

@@ -14,37 +14,86 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Tuple, Optional, List, Union, TYPE_CHECKING
+from abc import ABC, abstractmethod
+import asyncio
+import logging
 import platform
-import os
 
-from telethon.tl.types import *
-from mautrix_appservice import MatrixRequestError
+from sqlalchemy import orm
+from telethon.tl.types import Channel, ChannelForbidden, Chat, ChatForbidden, Message, \
+    MessageActionChannelMigrateFrom, MessageService, PeerUser, TypeUpdate, \
+    UpdateChannelPinnedMessage, UpdateChatAdmins, UpdateChatParticipantAdmin, \
+    UpdateChatParticipants, UpdateChatUserTyping, UpdateDeleteChannelMessages, \
+    UpdateDeleteMessages, UpdateEditChannelMessage, UpdateEditMessage, UpdateNewChannelMessage, \
+    UpdateNewMessage, UpdateReadHistoryOutbox, UpdateShortChatMessage, UpdateShortMessage, \
+    UpdateUserName, UpdateUserPhoto, UpdateUserStatus, UpdateUserTyping, User, UserStatusOffline, \
+    UserStatusOnline
+
+from mautrix_appservice import MatrixRequestError, AppService
+from alchemysession import AlchemySessionContainer
 
-from .tgclient import MautrixTelegramClient
-from .db import Message as DBMessage
 from . import portal as po, puppet as pu, __version__
+from .db import Message as DBMessage
+from .tgclient import MautrixTelegramClient
 
-config = None
+if TYPE_CHECKING:
+    from .context import Context
+    from .config import Config
+    from .bot import Bot
+
+config = None  # type: Config
 # Value updated from config in init()
-MAX_DELETIONS = 10
+MAX_DELETIONS = 10  # type: int
+
+UpdateMessage = Union[UpdateShortChatMessage, UpdateShortMessage, UpdateNewChannelMessage,
+                      UpdateNewMessage, UpdateEditMessage, UpdateEditChannelMessage]
+UpdateMessageContent = Union[UpdateShortMessage, UpdateShortChatMessage, Message, MessageService]
 
 
-class AbstractUser:
-    session_container = None
-    loop = None
-    log = None
-    db = None
-    az = None
+class AbstractUser(ABC):
+    session_container = None  # type: AlchemySessionContainer
+    loop = None  # type: asyncio.AbstractEventLoop
+    log = None  # type: logging.Logger
+    db = None  # type: orm.Session
+    az = None  # type: AppService
+    bot = None  # type: Bot
+    ignore_incoming_bot_events = True  # type: bool
 
     def __init__(self):
-        self.connected = False
-        self.whitelisted = False
-        self.client = None
-        self.tgid = None
-        self.mxid = None
-        self.is_relaybot = False
+        self.is_admin = False  # type: bool
+        self.matrix_puppet_whitelisted = False  # type: bool
+        self.puppet_whitelisted = False  # type: bool
+        self.whitelisted = False  # type: bool
+        self.relaybot_whitelisted = False  # type: bool
+        self.client = None  # type: MautrixTelegramClient
+        self.tgid = None  # type: int
+        self.mxid = None  # type: str
+        self.is_relaybot = False  # type: bool
+        self.is_bot = False  # type: bool
 
-    async def _init_client(self):
+    @property
+    def connected(self) -> bool:
+        return self.client and self.client.is_connected()
+
+    @property
+    def _proxy_settings(self) -> Optional[Tuple[int, str, str, str, str, str]]:
+        proxy_type = config["telegram.proxy.type"].lower()
+        if proxy_type == "disabled":
+            return None
+        elif proxy_type == "socks4":
+            proxy_type = 1
+        elif proxy_type == "socks5":
+            proxy_type = 2
+        elif proxy_type == "http":
+            proxy_type = 3
+
+        return (proxy_type,
+                config["telegram.proxy.address"], config["telegram.proxy.port"],
+                config["telegram.proxy.rdns"],
+                config["telegram.proxy.username"], config["telegram.proxy.password"])
+
+    def _init_client(self):
         self.log.debug(f"Initializing client for {self.name}")
         device = f"{platform.system()} {platform.release()}"
         sysversion = MautrixTelegramClient.__version__
@@ -56,23 +105,36 @@ class AbstractUser:
                                             app_version=__version__,
                                             system_version=sysversion,
                                             device_model=device,
-                                            report_errors=False)
-        await self.client.add_event_handler(self._update_catch)
+                                            timeout=120,
+                                            proxy=self._proxy_settings)
+        self.client.add_event_handler(self._update_catch)
 
-    async def update(self, update):
+    @abstractmethod
+    async def update(self, update: TypeUpdate) -> bool:
         return False
 
+    @abstractmethod
     async def post_login(self):
         raise NotImplementedError()
 
-    async def _update_catch(self, update):
+    @abstractmethod
+    def register_portal(self, portal: po.Portal):
+        raise NotImplementedError()
+
+    @abstractmethod
+    def unregister_portal(self, portal: po.Portal):
+        raise NotImplementedError()
+
+    async def _update_catch(self, update: TypeUpdate):
         try:
             if not await self.update(update):
                 await self._update(update)
         except Exception:
             self.log.exception("Failed to handle Telegram update")
 
-    async def _get_dialogs(self, limit=None):
+    async def get_dialogs(self, limit: int = None) -> List[Union[Chat, Channel]]:
+        if self.is_bot:
+            return []
         dialogs = await self.client.get_dialogs(limit=limit)
         return [dialog.entity for dialog in dialogs if (
             not isinstance(dialog.entity, (User, ChatForbidden, ChannelForbidden))
@@ -80,37 +142,46 @@ class AbstractUser:
                      and (dialog.entity.deactivated or dialog.entity.left)))]
 
     @property
-    def name(self):
+    @abstractmethod
+    def name(self) -> str:
         raise NotImplementedError()
 
-    @property
-    def logged_in(self):
-        return self.client and self.client.is_user_authorized()
+    async def is_logged_in(self) -> bool:
+        return self.client and await self.client.is_user_authorized()
 
-    @property
-    def has_full_access(self):
-        return self.logged_in and self.whitelisted
+    async def has_full_access(self, allow_bot: bool = False) -> bool:
+        return (self.puppet_whitelisted
+                and (not self.is_bot or allow_bot)
+                and await self.is_logged_in())
 
-    async def start(self):
+    async def start(self, delete_unless_authenticated: bool = False) -> "AbstractUser":
         if not self.client:
-            await self._init_client()
-        self.connected = await self.client.connect()
-
-    async def ensure_started(self, even_if_no_session=False):
-        if not self.whitelisted:
-            return self
-        elif not self.connected and (even_if_no_session or os.path.exists(f"{self.name}.session")):
-            return await self.start()
+            self._init_client()
+        await self.client.connect()
+        self.log.debug("%s connected: %s", self.mxid, self.connected)
         return self
 
-    def stop(self):
-        self.client.disconnect()
+    async def ensure_started(self, even_if_no_session=False) -> "AbstractUser":
+        if not self.puppet_whitelisted:
+            return self
+        self.log.debug("ensure_started(%s, connected=%s, even_if_no_session=%s, session_count=%s)",
+                       self.mxid, self.connected, even_if_no_session,
+                       self.session_container.Session.query.filter(
+                           self.session_container.Session.session_id == self.mxid).count())
+        should_connect = (even_if_no_session or
+                          self.session_container.Session.query.filter(
+                              self.session_container.Session.session_id == self.mxid).count() > 0)
+        if not self.connected and should_connect:
+            await self.start(delete_unless_authenticated=not even_if_no_session)
+        return self
+
+    async def stop(self):
+        await self.client.disconnect()
         self.client = None
-        self.connected = False
 
     # region Telegram update handling
 
-    async def _update(self, update):
+    async def _update(self, update: TypeUpdate):
         if isinstance(update, (UpdateShortChatMessage, UpdateShortMessage, UpdateNewChannelMessage,
                                UpdateNewMessage, UpdateEditMessage, UpdateEditChannelMessage)):
             await self.update_message(update)
@@ -135,17 +206,19 @@ class AbstractUser:
         else:
             self.log.debug("Unhandled update: %s", update)
 
-    async def update_pinned_messages(self, update):
+    @staticmethod
+    async def update_pinned_messages(update: UpdateChannelPinnedMessage):
         portal = po.Portal.get_by_tgid(update.channel_id)
         if portal and portal.mxid:
             await portal.receive_telegram_pin_id(update.id)
 
-    async def update_participants(self, update):
+    @staticmethod
+    async def update_participants(update: UpdateChatParticipants):
         portal = po.Portal.get_by_tgid(update.participants.chat_id)
         if portal and portal.mxid:
             await portal.update_telegram_participants(update.participants.participants)
 
-    async def update_read_receipt(self, update):
+    async def update_read_receipt(self, update: UpdateReadHistoryOutbox):
         if not isinstance(update.peer, PeerUser):
             self.log.debug("Unexpected read receipt peer: %s", update.peer)
             return
@@ -162,7 +235,7 @@ class AbstractUser:
         puppet = pu.Puppet.get(update.peer.user_id)
         await puppet.intent.mark_read(portal.mxid, message.mxid)
 
-    async def update_admin(self, update):
+    async def update_admin(self, update: Union[UpdateChatAdmins, UpdateChatParticipantAdmin]):
         # TODO duplication not checked
         portal = po.Portal.get_by_tgid(update.chat_id, peer_type="chat")
         if isinstance(update, UpdateChatAdmins):
@@ -172,7 +245,7 @@ class AbstractUser:
         else:
             self.log.warning("Unexpected admin status update: %s", update)
 
-    async def update_typing(self, update):
+    async def update_typing(self, update: Union[UpdateUserTyping, UpdateChatUserTyping]):
         if isinstance(update, UpdateUserTyping):
             portal = po.Portal.get_by_tgid(update.user_id, self.tgid, "user")
         else:
@@ -180,7 +253,7 @@ class AbstractUser:
         sender = pu.Puppet.get(update.user_id)
         await portal.handle_telegram_typing(sender, update)
 
-    async def update_others_info(self, update):
+    async def update_others_info(self, update: Union[UpdateUserName, UpdateUserPhoto]):
         # TODO duplication not checked
         puppet = pu.Puppet.get(update.user_id)
         if isinstance(update, UpdateUserName):
@@ -192,17 +265,19 @@ class AbstractUser:
         else:
             self.log.warning("Unexpected other user info update: %s", update)
 
-    async def update_status(self, update):
+    async def update_status(self, update: UpdateUserStatus):
         puppet = pu.Puppet.get(update.user_id)
         if isinstance(update.status, UserStatusOnline):
-            await puppet.intent.set_presence("online")
+            await puppet.default_mxid_intent.set_presence("online")
         elif isinstance(update.status, UserStatusOffline):
-            await puppet.intent.set_presence("offline")
+            await puppet.default_mxid_intent.set_presence("offline")
         else:
             self.log.warning("Unexpected user status update: %s", update)
         return
 
-    def get_message_details(self, update):
+    def get_message_details(self, update: UpdateMessage) -> Tuple[UpdateMessageContent,
+                                                                  Optional[pu.Puppet],
+                                                                  Optional[po.Portal]]:
         if isinstance(update, UpdateShortChatMessage):
             portal = po.Portal.get_by_tgid(update.chat_id, peer_type="chat")
             sender = pu.Puppet.get(update.from_id)
@@ -225,7 +300,7 @@ class AbstractUser:
         return update, sender, portal
 
     @staticmethod
-    async def _try_redact(portal, message):
+    async def _try_redact(portal: po.Portal, message: DBMessage):
         if not portal:
             return
         try:
@@ -233,7 +308,7 @@ class AbstractUser:
         except MatrixRequestError:
             pass
 
-    async def delete_message(self, update):
+    async def delete_message(self, update: UpdateDeleteMessages):
         if len(update.messages) > MAX_DELETIONS:
             return
 
@@ -249,7 +324,7 @@ class AbstractUser:
                 await self._try_redact(portal, message)
         self.db.commit()
 
-    async def delete_channel_message(self, update):
+    async def delete_channel_message(self, update: UpdateDeleteChannelMessages):
         if len(update.messages) > MAX_DELETIONS:
             return
 
@@ -265,8 +340,11 @@ class AbstractUser:
             await self._try_redact(portal, message)
         self.db.commit()
 
-    async def update_message(self, original_update):
+    async def update_message(self, original_update: UpdateMessage):
         update, sender, portal = self.get_message_details(original_update)
+        if self.ignore_incoming_bot_events and self.bot and sender.id == self.bot.tgid:
+            self.log.debug(f"Ignoring relaybot-sent message %s to %s", update, portal.tgid_log)
+            return
 
         if isinstance(update, MessageService):
             if isinstance(update.action, MessageActionChannelMigrateFrom):
@@ -291,8 +369,9 @@ class AbstractUser:
     # endregion
 
 
-def init(context):
+def init(context: "Context"):
     global config, MAX_DELETIONS
-    AbstractUser.az, AbstractUser.db, config, AbstractUser.loop, _ = context
-    AbstractUser.session_container = context.telethon_session_container
+    AbstractUser.az, AbstractUser.db, config, AbstractUser.loop, AbstractUser.relaybot = context
+    AbstractUser.ignore_incoming_bot_events = config["bridge.relaybot.ignore_own_incoming_events"]
+    AbstractUser.session_container = context.session_container
     MAX_DELETIONS = config.get("bridge.max_telegram_delete", 10)

mautrix_telegram/base.py

@@ -1,2 +1,2 @@
 from sqlalchemy.ext.declarative import declarative_base
-Base = declarative_base()
+Base = declarative_base()  # type: declarative_base

mautrix_telegram/bot.py

@@ -14,7 +14,7 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Awaitable, Callable
+from typing import Awaitable, Callable, Pattern, Dict, TYPE_CHECKING
 import logging
 import re
 
@@ -27,24 +27,31 @@ from .abstract_user import AbstractUser
 from .db import BotChat
 from . import puppet as pu, portal as po, user as u
 
-config = None
+if TYPE_CHECKING:
+    from .config import Config
+
+config = None  # type: Config
 
 ReplyFunc = Callable[[str], Awaitable[Message]]
 
 
 class Bot(AbstractUser):
-    log = logging.getLogger("mau.bot")
-    mxid_regex = re.compile("@.+:.+")
+    log = logging.getLogger("mau.bot")  # type: logging.Logger
+    mxid_regex = re.compile("@.+:.+")  # type: Pattern
 
     def __init__(self, token: str):
         super().__init__()
-        self.token = token
-        self.whitelisted = True
-        self.username = None
-        self.is_relaybot = True
-        self.chats = {chat.id: chat.type for chat in BotChat.query.all()}
-        self.tg_whitelist = []
-        self.whitelist_group_admins = config["bridge.relaybot.whitelist_group_admins"] or False
+        self.token = token  # type: str
+        self.puppet_whitelisted = True  # type: bool
+        self.whitelisted = True  # type: bool
+        self.relaybot_whitelisted = True  # type: bool
+        self.username = None  # type: str
+        self.is_relaybot = True  # type: bool
+        self.is_bot = True  # type: bool
+        self.chats = {chat.id: chat.type for chat in BotChat.query.all()}  # type: Dict[int, str]
+        self.tg_whitelist = []  # type: List[int]
+        self.whitelist_group_admins = (config["bridge.relaybot.whitelist_group_admins"]
+                                       or False)  # type: bool
 
     async def init_permissions(self):
         whitelist = config["bridge.relaybot.whitelist"] or []
@@ -58,9 +65,9 @@ class Bot(AbstractUser):
             if isinstance(id, int):
                 self.tg_whitelist.append(id)
 
-    async def start(self):
-        await super().start()
-        if not self.logged_in:
+    async def start(self, delete_unless_authenticated: bool = False) -> "Bot":
+        await super().start(delete_unless_authenticated)
+        if not await self.is_logged_in():
             await self.client.sign_in(bot_token=self.token)
         await self.post_login()
         return self
@@ -115,7 +122,7 @@ class Bot(AbstractUser):
             self.db.delete(existing_chat)
             self.db.commit()
 
-    async def _can_use_commands(self, chat, tgid):
+    async def _can_use_commands(self, chat: TypePeer, tgid: int) -> bool:
         if tgid in self.tg_whitelist:
             return True
 
@@ -135,7 +142,7 @@ class Bot(AbstractUser):
                     if p.user_id == tgid:
                         return isinstance(p, (ChatParticipantCreator, ChatParticipantAdmin))
 
-    async def check_can_use_commands(self, event: Message, reply: ReplyFunc):
+    async def check_can_use_commands(self, event: Message, reply: ReplyFunc) -> bool:
         if not await self._can_use_commands(event.to_id, event.from_id):
             await reply("You do not have the permission to use that command.")
             return False
@@ -168,7 +175,7 @@ class Bot(AbstractUser):
         user = await u.User.get_by_mxid(mxid).ensure_started()
         if not user.relaybot_whitelisted:
             return await reply("That user is not whitelisted to use the bridge.")
-        elif user.logged_in:
+        elif await user.is_logged_in():
             displayname = f"@{user.username}" if user.username else user.displayname
             return await reply("That user seems to be logged in. "
                                f"Just invite [{displayname}](tg://user?id={user.tgid})")
@@ -200,8 +207,7 @@ class Bot(AbstractUser):
 
     async def handle_command(self, message: Message):
         def reply(reply_text):
-            return self.client.send_message(message.to_id, reply_text, markdown=True,
-                                            reply_to=message.id)
+            return self.client.send_message(message.to_id, reply_text, reply_to=message.id)
 
         text = message.message
 
@@ -260,7 +266,7 @@ class Bot(AbstractUser):
         return "bot"
 
 
-def init(context):
+def init(context) -> Optional[Bot]:
     global config
     config = context.config
     token = config["telegram.bot_token"]

mautrix_telegram/commands/__init__.py

@@ -1,2 +1,5 @@
-from .handler import command_handler, CommandHandler, CommandEvent
+from .handler import (command_handler, command_handlers as _command_handlers,
+                      CommandHandler, CommandProcessor, CommandEvent,
+                      SECTION_GENERAL, SECTION_AUTH, SECTION_CREATING_PORTALS,
+                      SECTION_PORTAL_MANAGEMENT, SECTION_MISC, SECTION_ADMIN)
 from . import clean_rooms, auth, meta, telegram, portal

mautrix_telegram/commands/auth.py

@@ -14,28 +14,31 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Dict
 import asyncio
 
 from telethon.errors import *
 
-from . import command_handler
+from . import command_handler, CommandEvent, SECTION_AUTH
 from .. import puppet as pu
 from ..util import format_duration
 
 
-@command_handler(needs_auth=False)
-async def ping(evt):
-    if not evt.sender.logged_in:
-        return await evt.reply("You're not logged in.")
-    me = await evt.sender.client.get_me()
+@command_handler(needs_auth=False,
+                 help_section=SECTION_AUTH,
+                 help_text="Check if you're logged into Telegram.")
+async def ping(evt: CommandEvent):
+    me = await evt.sender.client.get_me() if await evt.sender.is_logged_in() else None
     if me:
         return await evt.reply(f"You're logged in as @{me.username}")
     else:
         return await evt.reply("You're not logged in.")
 
 
-@command_handler()
-async def ping_bot(evt):
+@command_handler(needs_auth=False, needs_puppeting=False,
+                 help_section=SECTION_AUTH,
+                 help_text="Get the info of the message relay Telegram bot.")
+async def ping_bot(evt: CommandEvent):
     if not evt.tgbot:
         return await evt.reply("Telegram message relay bot not configured.")
     bot_info = await evt.tgbot.client.get_me()
@@ -46,14 +49,76 @@ async def ping_bot(evt):
                            "To use the bot, simply invite it to a portal room.")
 
 
-@command_handler(needs_auth=False, management_only=True)
-def register(evt):
-    return evt.reply("Not yet implemented.")
+@command_handler(needs_auth=True, needs_matrix_puppeting=True,
+                 help_section=SECTION_AUTH,
+                 help_text="Revert your Telegram account's Matrix puppet to use the default Matrix "
+                           "account.")
+async def logout_matrix(evt: CommandEvent):
+    puppet = pu.Puppet.get(evt.sender.tgid)
+    if not puppet.is_real_user:
+        return await evt.reply("You are not logged in with your Matrix account.")
+    await puppet.switch_mxid(None, None)
+    await evt.reply("Reverted your Telegram account's Matrix puppet back to the default.")
 
 
-@command_handler(needs_auth=False, management_only=True)
-async def register(evt):
-    if evt.sender.logged_in:
+@command_handler(needs_auth=True, management_only=True, needs_matrix_puppeting=True,
+                 help_section=SECTION_AUTH,
+                 help_text="Replace your Telegram account's Matrix puppet with your own Matrix "
+                           "account")
+async def login_matrix(evt: CommandEvent):
+    puppet = pu.Puppet.get(evt.sender.tgid)
+    if puppet.is_real_user:
+        return await evt.reply("You have already logged in with your Matrix account. "
+                               "Log out with `$cmdprefix+sp logout-matrix` first.")
+    allow_matrix_login = evt.config.get("bridge.allow_matrix_login", True)
+    if allow_matrix_login:
+        evt.sender.command_status = {
+            "next": enter_matrix_token,
+            "action": "Matrix login",
+        }
+    if evt.config["appservice.public.enabled"]:
+        prefix = evt.config["appservice.public.external"]
+        url = f"{prefix}/matrix-login?token={evt.public_website.make_token(evt.sender.mxid, '/matrix-login')}"
+        if allow_matrix_login:
+            return await evt.reply(
+                "This bridge instance allows you to log in inside or outside Matrix.\n\n"
+                "If you would like to log in within Matrix, please send your Matrix access token "
+                "here.\n"
+                f"If you would like to log in outside of Matrix, [click here]({url}).\n\n"
+                "Logging in outside of Matrix is recommended, because in-Matrix login would save "
+                "your access token in the message history.")
+        return await evt.reply("This bridge instance does not allow logging in inside Matrix.\n\n"
+                               f"Please visit [the login page]({url}) to log in.")
+    elif allow_matrix_login:
+        return await evt.reply(
+            "This bridge instance does not allow you to log in outside of Matrix.\n\n"
+            "Please send your Matrix access token here to log in.")
+    return await evt.reply("This bridge instance has been configured to not allow logging in.")
+
+
+async def enter_matrix_token(evt: CommandEvent):
+    evt.sender.command_status = None
+
+    puppet = pu.Puppet.get(evt.sender.tgid)
+    if puppet.is_real_user:
+        return await evt.reply("You have already logged in with your Matrix account. "
+                               "Log out with `$cmdprefix+sp logout-matrix` first.")
+
+    resp = await puppet.switch_mxid(" ".join(evt.args), evt.sender.mxid)
+    if resp == 2:
+        return await evt.reply("You can only log in as your own Matrix user.")
+    elif resp == 1:
+        return await evt.reply("Failed to verify access token.")
+    return await evt.reply(
+        f"Replaced your Telegram account's Matrix puppet with {puppet.custom_mxid}.")
+
+
+@command_handler(needs_auth=False, management_only=True,
+                 help_section=SECTION_AUTH,
+                 help_args="<_phone_> <_full name_>",
+                 help_text="Register to Telegram")
+async def register(evt: CommandEvent):
+    if await evt.sender.is_logged_in():
         return await evt.reply("You are already logged in.")
     elif len(evt.args) < 1:
         return await evt.reply("**Usage:** `$cmdprefix+sp register <phone> <full name>`")
@@ -71,7 +136,7 @@ async def register(evt):
     })
 
 
-async def enter_code_register(evt):
+async def enter_code_register(evt: CommandEvent):
     if len(evt.args) == 0:
         return await evt.reply("**Usage:** `$cmdprefix+sp <code>`")
     try:
@@ -97,36 +162,41 @@ async def enter_code_register(evt):
                                "Check console for more details.")
 
 
-@command_handler(needs_auth=False, management_only=True)
-async def login(evt):
-    if evt.sender.logged_in:
+@command_handler(needs_auth=False, management_only=True,
+                 help_section=SECTION_AUTH,
+                 help_text="Get instructions on how to log in.")
+async def login(evt: CommandEvent):
+    if await evt.sender.is_logged_in():
         return await evt.reply("You are already logged in.")
 
     allow_matrix_login = evt.config.get("bridge.allow_matrix_login", True)
     if allow_matrix_login:
         evt.sender.command_status = {
-            "next": enter_phone,
+            "next": enter_phone_or_token,
             "action": "Login",
         }
 
     if evt.config["appservice.public.enabled"]:
         prefix = evt.config["appservice.public.external"]
-        url = f"{prefix}/login?mxid={evt.sender.mxid}"
-        if evt.config.get("bridge.allow_matrix_login", True):
-            return await evt.reply("\n\n".join((
-                "This bridge instance allows you to log in inside or outside Matrix.",
-                "If you would like to log in within Matrix, please send your phone number here.",
-                f"If you would like to log in outside of Matrix, [click here]({url}).")))
+        url = f"{prefix}/login?token={evt.public_website.make_token(evt.sender.mxid, '/login')}"
+        if allow_matrix_login:
+            return await evt.reply(
+                "This bridge instance allows you to log in inside or outside Matrix.\n\n"
+                "If you would like to log in within Matrix, please send your phone number or bot "
+                "auth token here.\n"
+                f"If you would like to log in outside of Matrix, [click here]({url}).\n\n"
+                "Logging in outside of Matrix is recommended if you have two-factor authentication "
+                "enabled, because in-Matrix login would save your password in the message history.")
         return await evt.reply("This bridge instance does not allow logging in inside Matrix.\n\n"
                                f"Please visit [the login page]({url}) to log in.")
     elif allow_matrix_login:
         return await evt.reply(
             "This bridge instance does not allow you to log in outside of Matrix.\n\n"
-            "Please send your phone number here to start the login process.")
+            "Please send your phone number or bot auth token here to start the login process.")
     return await evt.reply("This bridge instance has been configured to not allow logging in.")
 
 
-async def request_code(evt, phone_number, next_status):
+async def request_code(evt: CommandEvent, phone_number: str, next_status: Dict[str, str]):
     ok = False
     try:
         await evt.sender.ensure_started(even_if_no_session=True)
@@ -158,44 +228,37 @@ async def request_code(evt, phone_number, next_status):
 
 
 @command_handler(needs_auth=False)
-async def enter_phone(evt):
+async def enter_phone_or_token(evt: CommandEvent):
     if len(evt.args) == 0:
-        return await evt.reply("**Usage:** `$cmdprefix+sp enter-phone <phone>`")
+        return await evt.reply("**Usage:** `$cmdprefix+sp enter-phone-or-token <phone-or-token>`")
     elif not evt.config.get("bridge.allow_matrix_login", True):
         return await evt.reply("This bridge instance does not allow in-Matrix login. "
                                "Please use `$cmdprefix+sp login` to get login instructions")
 
-    phone_number = evt.args[0]
-    await request_code(evt, phone_number, {
-        "next": enter_code,
-        "action": "Login",
-    })
+    # phone numbers don't contain colons but telegram bot auth tokens do
+    if evt.args[0].find(":") > 0:
+        try:
+            await sign_in(evt, bot_token=evt.args[0])
+        except Exception:
+            evt.log.exception("Error sending auth token")
+            return await evt.reply("Unhandled exception while sending auth token. "
+                                   "Check console for more details.")
+    else:
+        await request_code(evt, evt.args[0], {
+            "next": enter_code,
+            "action": "Login",
+        })
 
 
 @command_handler(needs_auth=False)
-async def enter_code(evt):
+async def enter_code(evt: CommandEvent):
     if len(evt.args) == 0:
         return await evt.reply("**Usage:** `$cmdprefix+sp enter-code <code>`")
     elif not evt.config.get("bridge.allow_matrix_login", True):
         return await evt.reply("This bridge instance does not allow in-Matrix login. "
                                "Please use `$cmdprefix+sp login` to get login instructions")
     try:
-        await evt.sender.ensure_started(even_if_no_session=True)
-        user = await evt.sender.client.sign_in(code=evt.args[0])
-        asyncio.ensure_future(evt.sender.post_login(user), loop=evt.loop)
-        evt.sender.command_status = None
-        return await evt.reply(f"Successfully logged in as @{user.username}")
-    except PhoneCodeExpiredError:
-        return await evt.reply("Phone code expired. Try again with `$cmdprefix+sp login`.")
-    except PhoneCodeInvalidError:
-        return await evt.reply("Invalid phone code.")
-    except SessionPasswordNeededError:
-        evt.sender.command_status = {
-            "next": enter_password,
-            "action": "Login (password entry)",
-        }
-        return await evt.reply("Your account has two-factor authentication. "
-                               "Please send your password here.")
+        await sign_in(evt, code=evt.args[0])
     except Exception:
         evt.log.exception("Error sending phone code")
         return await evt.reply("Unhandled exception while sending code. "
@@ -203,30 +266,50 @@ async def enter_code(evt):
 
 
 @command_handler(needs_auth=False)
-async def enter_password(evt):
+async def enter_password(evt: CommandEvent):
     if len(evt.args) == 0:
         return await evt.reply("**Usage:** `$cmdprefix+sp enter-password <password>`")
     elif not evt.config.get("bridge.allow_matrix_login", True):
         return await evt.reply("This bridge instance does not allow in-Matrix login. "
                                "Please use `$cmdprefix+sp login` to get login instructions")
     try:
-        await evt.sender.ensure_started(even_if_no_session=True)
-        user = await evt.sender.client.sign_in(password=" ".join(evt.args))
-        asyncio.ensure_future(evt.sender.post_login(user), loop=evt.loop)
-        evt.sender.command_status = None
-        return await evt.reply(f"Successfully logged in as @{user.username}")
-    except PasswordHashInvalidError:
-        return await evt.reply("Incorrect password.")
+        await sign_in(evt, password=" ".join(evt.args))
+    except AccessTokenInvalidError:
+        return await evt.reply("That bot token is not valid.")
+    except AccessTokenExpiredError:
+        return await evt.reply("That bot token has expired.")
     except Exception:
         evt.log.exception("Error sending password")
         return await evt.reply("Unhandled exception while sending password. "
                                "Check console for more details.")
 
 
-@command_handler(needs_auth=False)
-async def logout(evt):
-    if not evt.sender.logged_in:
-        return await evt.reply("You're not logged in.")
+async def sign_in(evt: CommandEvent, **sign_in_info):
+    try:
+        await evt.sender.ensure_started(even_if_no_session=True)
+        user = await evt.sender.client.sign_in(**sign_in_info)
+        asyncio.ensure_future(evt.sender.post_login(user), loop=evt.loop)
+        evt.sender.command_status = None
+        return await evt.reply(f"Successfully logged in as @{user.username}")
+    except PhoneCodeExpiredError:
+        return await evt.reply("Phone code expired. Try again with `$cmdprefix+sp login`.")
+    except PhoneCodeInvalidError:
+        return await evt.reply("Invalid phone code.")
+    except PasswordHashInvalidError:
+        return await evt.reply("Incorrect password.")
+    except SessionPasswordNeededError:
+        evt.sender.command_status = {
+            "next": enter_password,
+            "action": "Login (password entry)",
+        }
+        return await evt.reply("Your account has two-factor authentication. "
+                               "Please send your password here.")
+
+
+@command_handler(needs_auth=True,
+                 help_section=SECTION_AUTH,
+                 help_text="Log out from Telegram.")
+async def logout(evt: CommandEvent):
     if await evt.sender.log_out():
         return await evt.reply("Logged out successfully.")
     return await evt.reply("Failed to log out.")

mautrix_telegram/commands/clean_rooms.py

@@ -14,17 +14,23 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from mautrix_appservice import MatrixRequestError
+from typing import Tuple, List
 
-from . import command_handler
+from mautrix_appservice import MatrixRequestError, IntentAPI
+
+from . import command_handler, CommandEvent, SECTION_ADMIN
 from .. import puppet as pu, portal as po
 
+ManagementRoomList = List[Tuple[str, str]]
+RoomIDList = List[str]
 
-async def _find_rooms(intent):
-    management_rooms = []
-    unidentified_rooms = []
-    portals = []
-    empty_portals = []
+
+async def _find_rooms(intent: IntentAPI) -> Tuple[ManagementRoomList, RoomIDList,
+                                                  List["po.Portal"], List["po.Portal"]]:
+    management_rooms = []  # type: ManagementRoomList
+    unidentified_rooms = []  # type: RoomIDList
+    portals = []  # type: List[po.Portal]
+    empty_portals = []  # type: List[po.Portal]
 
     rooms = await intent.get_joined_rooms()
     for room in rooms:
@@ -52,12 +58,10 @@ async def _find_rooms(intent):
     return management_rooms, unidentified_rooms, portals, empty_portals
 
 
-@command_handler(needs_admin=True, needs_auth=False, name="clean-rooms")
-async def clean_rooms(evt):
-    if not evt.is_management:
-        return await evt.reply("`clean-rooms` is a particularly spammy command. Please don't "
-                               "run it in non-management rooms.")
-
+@command_handler(needs_admin=True, needs_auth=False, management_only=True, name="clean-rooms",
+                 help_section=SECTION_ADMIN,
+                 help_text="Clean up unused portal/management rooms.")
+async def clean_rooms(evt: CommandEvent):
     management_rooms, unidentified_rooms, portals, empty_portals = await _find_rooms(evt.az.intent)
 
     reply = ["#### Management rooms (M)"]
@@ -90,7 +94,7 @@ async def clean_rooms(evt):
                "where `range` is the range (e.g. `5-21`) prefixed with the first letter of"
                "the group name."),
               "",
-              ("Please note that you will have to re-run `$cmdprefix+sp cleanrooms` "
+              ("Please note that you will have to re-run `$cmdprefix+sp clean-rooms` "
                "between each use of the commands above.")]
 
     evt.sender.command_status = {
@@ -102,7 +106,9 @@ async def clean_rooms(evt):
     return await evt.reply("\n".join(reply))
 
 
-async def set_rooms_to_clean(evt, management_rooms, unidentified_rooms, portals, empty_portals):
+async def set_rooms_to_clean(evt, management_rooms: ManagementRoomList,
+                             unidentified_rooms: RoomIDList, portals: List["po.Portal"],
+                             empty_portals: List["po.Portal"]):
     command = evt.args[0]
     rooms_to_clean = []
     if command == "clean-recommended":
@@ -112,7 +118,7 @@ async def set_rooms_to_clean(evt, management_rooms, unidentified_rooms, portals,
             return await evt.reply("**Usage:** `$cmdprefix+sp clean-groups [M][A][U][I]")
         groups_to_clean = evt.args[1]
         if "M" in groups_to_clean:
-            rooms_to_clean += management_rooms
+            rooms_to_clean += [room_id for (room_id, user_id) in management_rooms]
         if "A" in groups_to_clean:
             rooms_to_clean += portals
         if "U" in groups_to_clean:
@@ -126,7 +132,7 @@ async def set_rooms_to_clean(evt, management_rooms, unidentified_rooms, portals,
             start, end = range.split("-")
             start, end = int(start), int(end)
             if group == "M":
-                group = management_rooms
+                group = [room_id for (room_id, user_id) in management_rooms]
             elif group == "A":
                 group = portals
             elif group == "U":

mautrix_telegram/commands/handler.py

@@ -14,42 +14,38 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import List, Dict, Callable, Optional
+from collections import namedtuple
 import markdown
 import logging
 
 from telethon.errors import FloodWaitError
 
 from ..util import format_duration
+from .. import user as u, context as c
 
-command_handlers = {}
+command_handlers = {}  # type: Dict[str, CommandHandler]
 
+HelpSection = namedtuple("HelpSection", "name order description")
 
-def command_handler(needs_auth=True, management_only=False, needs_admin=False, name=None):
-    def decorator(func):
-        def wrapper(evt):
-            if management_only and not evt.is_management:
-                return evt.reply(f"`{evt.command}` is a restricted command:"
-                                 "you may only run it in management rooms.")
-            elif needs_auth and not evt.sender.logged_in:
-                return evt.reply("This command requires you to be logged in.")
-            elif needs_admin and not evt.sender.is_admin:
-                return evt.reply("This is command requires administrator privileges.")
-            return func(evt)
-
-        command_handlers[name or func.__name__.replace("_", "-")] = wrapper
-        return wrapper
-
-    return decorator
+SECTION_GENERAL = HelpSection("General", 0, "")
+SECTION_AUTH = HelpSection("Authentication", 10, "")
+SECTION_CREATING_PORTALS = HelpSection("Creating portals", 20, "")
+SECTION_PORTAL_MANAGEMENT = HelpSection("Portal management", 30, "")
+SECTION_MISC = HelpSection("Miscellaneous", 40, "")
+SECTION_ADMIN = HelpSection("Administration", 50, "")
 
 
 class CommandEvent:
-    def __init__(self, handler, room, sender, command, args, is_management, is_portal):
-        self.az = handler.az
-        self.log = handler.log
-        self.loop = handler.loop
-        self.tgbot = handler.tgbot
-        self.config = handler.config
-        self.command_prefix = handler.command_prefix
+    def __init__(self, processor: "CommandProcessor", room: str, sender: u.User, command: str,
+                 args: List[str], is_management: bool, is_portal: bool):
+        self.az = processor.az
+        self.log = processor.log
+        self.loop = processor.loop
+        self.tgbot = processor.tgbot
+        self.config = processor.config
+        self.public_website = processor.public_website
+        self.command_prefix = processor.command_prefix
         self.room_id = room
         self.sender = sender
         self.command = command
@@ -57,7 +53,7 @@ class CommandEvent:
         self.is_management = is_management
         self.is_portal = is_portal
 
-    def reply(self, message, allow_html=False, render_markdown=True):
+    def reply(self, message: str, allow_html: bool = False, render_markdown: bool = True):
         message = message.replace("$cmdprefix+sp ",
                                   "" if self.is_management else f"{self.command_prefix} ")
         message = message.replace("$cmdprefix", self.command_prefix)
@@ -70,17 +66,86 @@ class CommandEvent:
 
 
 class CommandHandler:
+    def __init__(self, handler: Callable[[CommandEvent], None], needs_auth: bool,
+                 needs_puppeting: bool, needs_matrix_puppeting: bool, needs_admin: bool,
+                 management_only: bool, name: str, help_text: str, help_args: str,
+                 help_section: HelpSection):
+        self._handler = handler
+        self.needs_auth = needs_auth
+        self.needs_puppeting = needs_puppeting
+        self.needs_matrix_puppeting = needs_matrix_puppeting
+        self.needs_admin = needs_admin
+        self.management_only = management_only
+        self.name = name
+        self._help_text = help_text
+        self._help_args = help_args
+        self.help_section = help_section
+
+    async def get_permission_error(self, evt: CommandEvent) -> Optional[str]:
+        if self.management_only and not evt.is_management:
+            return (f"`{evt.command}` is a restricted command: "
+                    "you may only run it in management rooms.")
+        elif self.needs_puppeting and not evt.sender.puppet_whitelisted:
+            return "This command requires puppeting privileges."
+        elif self.needs_matrix_puppeting and not evt.sender.matrix_puppet_whitelisted:
+            return "This command requires Matrix puppeting privileges."
+        elif self.needs_admin and not evt.sender.is_admin:
+            return "This command requires administrator privileges."
+        elif self.needs_auth and not await evt.sender.is_logged_in():
+            return "This command requires you to be logged in."
+        return None
+
+    def has_permission(self, is_management: bool, puppet_whitelisted: bool,
+                       matrix_puppet_whitelisted: bool, is_admin: bool, is_logged_in: bool) -> bool:
+        return ((not self.management_only or is_management) and
+                (not self.needs_puppeting or puppet_whitelisted) and
+                (not self.needs_matrix_puppeting or matrix_puppet_whitelisted) and
+                (not self.needs_admin or is_admin) and
+                (not self.needs_auth or is_logged_in))
+
+    async def __call__(self, evt: CommandEvent):
+        error = await self.get_permission_error(evt)
+        if error is not None:
+            return await evt.reply(error)
+        return await self._handler(evt)
+
+    @property
+    def has_help(self) -> bool:
+        return bool(self.help_section) and bool(self._help_text)
+
+    @property
+    def help(self) -> str:
+        return f"**{self.name}** {self._help_args} - {self._help_text}"
+
+
+def command_handler(_func: Optional[Callable[[CommandEvent], None]] = None, *, needs_auth=True,
+                    needs_puppeting=True, needs_matrix_puppeting=False, needs_admin=False,
+                    management_only=False, name=None, help_text="", help_args="",
+                    help_section=None):
+    input_name = name
+
+    def decorator(func: Callable[[CommandEvent], None]):
+        name = input_name or func.__name__.replace("_", "-")
+        handler = CommandHandler(func, needs_auth, needs_puppeting, needs_matrix_puppeting,
+                                 needs_admin, management_only, name, help_text, help_args,
+                                 help_section)
+        command_handlers[handler.name] = handler
+        return handler
+
+    return decorator if _func is None else decorator(_func)
+
+
+class CommandProcessor:
     log = logging.getLogger("mau.commands")
 
-    def __init__(self, context):
+    def __init__(self, context: c.Context):
         self.az, self.db, self.config, self.loop, self.tgbot = context
+        self.public_website = context.public_website
         self.command_prefix = self.config["bridge.command_prefix"]
 
-    # region Utility functions for handling commands
-
-    async def handle(self, room, sender, command, args, is_management, is_portal):
-        evt = CommandEvent(self, room, sender, command, args,
-                           is_management, is_portal)
+    async def handle(self, room: str, sender: u.User, command: str, args: List[str],
+                     is_management: bool, is_portal: bool):
+        evt = CommandEvent(self, room, sender, command, args, is_management, is_portal)
         orig_command = command
         command = command.lower()
         try:
@@ -97,7 +162,7 @@ class CommandHandler:
         except FloodWaitError as e:
             return await evt.reply(f"Flood error: Please wait {format_duration(e.seconds)}")
         except Exception:
-            self.log.exception("Fatal error handling command "
+            self.log.exception("Unhandled error while handling command "
                                f"{evt.command} {' '.join(args)} from {sender.mxid}")
-            return await evt.reply("Fatal error while handling command. "
+            return await evt.reply("Unhandled error while handling command. "
                                    "Check logs for more details.")

mautrix_telegram/commands/meta.py

@@ -14,11 +14,13 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from . import command_handler
+from . import command_handler, CommandEvent, _command_handlers, SECTION_GENERAL
 
 
-@command_handler(needs_auth=False)
-def cancel(evt):
+@command_handler(needs_auth=False, needs_puppeting=False,
+                 help_section=SECTION_GENERAL,
+                 help_text="Cancel an ongoing action (such as login)")
+def cancel(evt: CommandEvent):
     if evt.sender.command_status:
         action = evt.sender.command_status["action"]
         evt.sender.command_status = None
@@ -27,62 +29,41 @@ def cancel(evt):
         return evt.reply("No ongoing command.")
 
 
-@command_handler(needs_auth=False)
-def unknown_command(evt):
+@command_handler(needs_auth=False, needs_puppeting=False)
+def unknown_command(evt: CommandEvent):
     return evt.reply("Unknown command. Try `$cmdprefix+sp help` for help.")
 
 
-@command_handler(needs_auth=False)
-def help(evt):
+help_cache = {}
+
+
+async def _get_help_text(evt: CommandEvent):
+    cache_key = (evt.is_management, evt.sender.puppet_whitelisted,
+                 evt.sender.matrix_puppet_whitelisted, evt.sender.is_admin,
+                 await evt.sender.is_logged_in())
+    if cache_key not in help_cache:
+        help = {}
+        for handler in _command_handlers.values():
+            if handler.has_help and handler.has_permission(*cache_key):
+                help.setdefault(handler.help_section, [])
+                help[handler.help_section].append(handler.help + "  ")
+        help = sorted(help.items(), key=lambda item: item[0].order)
+        help = ["#### {}\n{}\n".format(key.name, "\n".join(value)) for key, value in help]
+        help_cache[cache_key] = "\n".join(help)
+    return help_cache[cache_key]
+
+
+def _get_management_status(evt: CommandEvent):
     if evt.is_management:
-        management_status = ("This is a management room: prefixing commands "
-                             "with `$cmdprefix` is not required.\n")
+        return "This is a management room: prefixing commands with `$cmdprefix` is not required."
     elif evt.is_portal:
-        management_status = ("**This is a portal room**: you must always "
-                             "prefix commands with `$cmdprefix`.\n"
-                             "Management commands will not be sent to Telegram.")
-    else:
-        management_status = ("**This is not a management room**: you must "
-                             "prefix commands with `$cmdprefix`.\n")
-    help = """\n
-#### Generic bridge commands
-**help**   - Show this help message.  
-**cancel** - Cancel an ongoing action (such as login).
+        return ("**This is a portal room**: you must always prefix commands with `$cmdprefix`.\n"
+                "Management commands will not be sent to Telegram.")
+    return "**This is not a management room**: you must prefix commands with `$cmdprefix`."
 
-#### Authentication
-**login**  - Request an authentication code.  
-**logout** - Log out from Telegram.  
-**ping**   - Check if you're logged into Telegram.
 
-#### Miscellaneous things
-**search** [_-r|--remote_] <_query_> - Search your contacts or the Telegram servers for users.  
-**sync** [`chats`|`contacts`|`me`]   - Synchronize your chat portals, contacts and/or own info.  
-**ping-bot**                         - Get info of the message relay Telegram bot.  
-**set-pl** <_level_> [_mxid_]        - Set a temporary power level without affecting Telegram.
-
-#### Initiating chats
-**pm** <_identifier_> - Open a private chat with the given Telegram user. The identifier is either
-                        the internal user ID, the username or the phone number.  
-**join** <_link_>     - Join a chat with an invite link.  
-**create** [_type_]   - Create a Telegram chat of the given type for the current Matrix room. The
-                        type is either `group`, `supergroup` or `channel` (defaults to `group`).
-
-#### Portal management  
-**upgrade**                 - Upgrade a normal Telegram group to a supergroup.  
-**invite-link**             - Get a Telegram invite link to the current chat.  
-**delete-portal**           - Remove all users from the current portal room and forget the portal.
-                              Only works for group chats; to delete a private chat portal, simply
-                              leave the room.  
-**unbridge**                - Remove puppets from the current portal room and forget the portal.  
-**bridge** [_id_]           - Bridge the current Matrix room to the Telegram chat with the given
-                              ID. The ID must be the prefixed version that you get with the `/id`
-                              command of the Telegram-side bot.  
-**group-name** <_name_|`-`> - Change the username of a supergroup/channel. To disable, use a dash
-                             (`-`) as the name.  
-**clean-rooms**             - Clean up unused portal/management rooms.
-
-**filter** <`whitelist`|`blacklist`> <_chat ID_> - Allow or disallow bridging a specific chat.  
-**filter-mode** <`whitelist`|`blacklist`>      - Change whether the bridge will allow or disallow
-                                                 bridging rooms by default.
-"""
-    return evt.reply(management_status + help)
+@command_handler(needs_auth=False, needs_puppeting=False,
+                 help_section=SECTION_GENERAL,
+                 help_text="Show this help message.")
+async def help(evt: CommandEvent):
+    return await evt.reply(_get_management_status(evt) + "\n" + await _get_help_text(evt))

mautrix_telegram/commands/portal.py

@@ -14,16 +14,22 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Optional, Callable
 import asyncio
 
 from telethon.errors import *
-from mautrix_appservice import MatrixRequestError
+from telethon.tl.types import ChatForbidden, ChannelForbidden
+from mautrix_appservice import MatrixRequestError, IntentAPI
 
-from .. import portal as po
-from . import command_handler, CommandEvent
+from .. import portal as po, user as u
+from . import (command_handler, CommandEvent,
+               SECTION_ADMIN, SECTION_CREATING_PORTALS, SECTION_PORTAL_MANAGEMENT)
 
 
-@command_handler(needs_admin=True, needs_auth=False, name="set-pl")
+@command_handler(needs_admin=True, needs_auth=False, name="set-pl",
+                 help_section=SECTION_ADMIN,
+                 help_args="<_level_> [_mxid_]",
+                 help_text="Set a temporary power level without affecting Telegram.")
 async def set_power_level(evt: CommandEvent):
     try:
         level = int(evt.args[0])
@@ -41,7 +47,8 @@ async def set_power_level(evt: CommandEvent):
         return await evt.reply("Failed to set power level.")
 
 
-@command_handler()
+@command_handler(help_section=SECTION_PORTAL_MANAGEMENT,
+                 help_text="Get a Telegram invite link to the current chat.")
 async def invite_link(evt: CommandEvent):
     portal = po.Portal.get_by_mxid(evt.room_id)
     if not portal:
@@ -59,7 +66,7 @@ async def invite_link(evt: CommandEvent):
         return await evt.reply("You don't have the permission to create an invite link.")
 
 
-async def _has_access_to(room, intent, sender, event, default=50):
+async def user_has_power_level(room: str, intent, sender: u.User, event: str, default: int = 50):
     if sender.is_admin:
         return True
     # Make sure the state store contains the power levels.
@@ -72,7 +79,8 @@ async def _has_access_to(room, intent, sender, event, default=50):
                                               default=default)
 
 
-async def _get_portal_and_check_permission(evt, permission, action=None):
+async def _get_portal_and_check_permission(evt: CommandEvent, permission: str,
+                                           action: Optional[str] = None):
     room_id = evt.args[0] if len(evt.args) > 0 else evt.room_id
 
     portal = po.Portal.get_by_mxid(room_id)
@@ -80,13 +88,14 @@ async def _get_portal_and_check_permission(evt, permission, action=None):
         that_this = "This" if room_id == evt.room_id else "That"
         return await evt.reply(f"{that_this} is not a portal room."), False
 
-    if not await _has_access_to(portal.mxid, evt.az.intent, evt.sender, permission):
+    if not await user_has_power_level(portal.mxid, evt.az.intent, evt.sender, permission):
         action = action or f"{permission.replace('_', ' ')}s"
         return await evt.reply(f"You do not have the permissions to {action} that portal."), False
     return portal, True
 
 
-def _get_portal_murder_function(action, room_id, function, command, completed_message):
+def _get_portal_murder_function(action: str, room_id: str, function: Callable, command: str,
+                                completed_message: str):
     async def post_confirm(confirm):
         confirm.sender.command_status = None
         if len(confirm.args) > 0 and confirm.args[0] == f"confirm-{command}":
@@ -102,9 +111,13 @@ def _get_portal_murder_function(action, room_id, function, command, completed_me
     }
 
 
-@command_handler(needs_auth=False)
+@command_handler(needs_auth=False, needs_puppeting=False,
+                 help_section=SECTION_PORTAL_MANAGEMENT,
+                 help_text="Remove all users from the current portal room and forget the portal. "
+                           "Only works for group chats; to delete a private chat portal, simply "
+                           "leave the room.")
 async def delete_portal(evt: CommandEvent):
-    portal, ok = await _get_portal_and_check_permission(evt, "delete_portal")
+    portal, ok = await _get_portal_and_check_permission(evt, "unbridge")
     if not ok:
         return
 
@@ -114,12 +127,18 @@ async def delete_portal(evt: CommandEvent):
     return await evt.reply("Please confirm deletion of portal "
                            f"[{portal.alias or portal.mxid}](https://matrix.to/#/{portal.mxid}) "
                            f"to Telegram chat \"{portal.title}\" "
-                           "by typing `$cmdprefix+sp confirm-delete`")
+                           "by typing `$cmdprefix+sp confirm-delete`"
+                           "\n\n"
+                           "**WARNING:** If the bridge bot has the power level to do so, **this "
+                           "will kick ALL users** in the room. If you just want to remove the "
+                           "bridge, use `$cmdprefix+sp unbridge` instead.")
 
 
-@command_handler(needs_auth=False)
+@command_handler(needs_auth=False, needs_puppeting=False,
+                 help_section=SECTION_PORTAL_MANAGEMENT,
+                 help_text="Remove puppets from the current portal room and forget the portal.")
 async def unbridge(evt: CommandEvent):
-    portal, ok = await _get_portal_and_check_permission(evt, "unbridge_room")
+    portal, ok = await _get_portal_and_check_permission(evt, "unbridge")
     if not ok:
         return
 
@@ -131,7 +150,12 @@ async def unbridge(evt: CommandEvent):
                            "by typing `$cmdprefix+sp confirm-unbridge`")
 
 
-@command_handler(needs_auth=False)
+@command_handler(needs_auth=False, needs_puppeting=False,
+                 help_section=SECTION_PORTAL_MANAGEMENT,
+                 help_args="[_id_]",
+                 help_text="Bridge the current Matrix room to the Telegram chat with the given "
+                           "ID. The ID must be the prefixed version that you get with the `/id` "
+                           "command of the Telegram-side bot.")
 async def bridge(evt: CommandEvent):
     if len(evt.args) == 0:
         return await evt.reply("**Usage:** "
@@ -143,8 +167,8 @@ async def bridge(evt: CommandEvent):
     if portal:
         return await evt.reply(f"{that_this} room is already a portal room.")
 
-    if not await _has_access_to(room_id, evt.az.intent, evt.sender, "bridge"):
-        return await evt.reply("You do not have the permissions to bridge that room.")
+    if not await user_has_power_level(room_id, evt.az.intent, evt.sender, "bridge"):
+        return await evt.reply(f"You do not have the permissions to bridge {that_this} room.")
 
     # The /id bot command provides the prefixed ID, so we assume
     tgid = evt.args[0]
@@ -163,13 +187,13 @@ async def bridge(evt: CommandEvent):
     portal = po.Portal.get_by_tgid(tgid, peer_type=peer_type)
     if not portal.allow_bridging():
         return await evt.reply("This bridge doesn't allow bridging that Telegram chat.\n"
-                               "If you're the bridge admin, try"
+                               "If you're the bridge admin, try "
                                "`$cmdprefix+sp whitelist <Telegram chat ID>` first.")
     if portal.mxid:
         has_portal_message = (
             "That Telegram chat already has a portal at "
             f"[{portal.alias or portal.mxid}](https://matrix.to/#/{portal.mxid}). ")
-        if not await _has_access_to(portal.mxid, evt.az.intent, evt.sender, "unbridge"):
+        if not await user_has_power_level(portal.mxid, evt.az.intent, evt.sender, "unbridge"):
             return await evt.reply(f"{has_portal_message}"
                                    "Additionally, you do not have the permissions to unbridge "
                                    "that room.")
@@ -198,7 +222,7 @@ async def bridge(evt: CommandEvent):
                            "chat to this room, use `$cmdprefix+sp continue`")
 
 
-async def cleanup_old_portal_while_bridging(evt, portal):
+async def cleanup_old_portal_while_bridging(evt: CommandEvent, portal: "po.Portal"):
     if not portal.mxid:
         await evt.reply("The portal seems to have lost its Matrix room between you"
                         "calling `$cmdprefix+sp bridge` and this command.\n\n"
@@ -247,21 +271,28 @@ async def confirm_bridge(evt: CommandEvent):
         return await evt.reply("Please use `$cmdprefix+sp continue` to confirm the bridging or "
                                "`$cmdprefix+sp cancel` to cancel.")
 
-    user = evt.sender if evt.sender.logged_in else evt.tgbot
+    is_logged_in = await evt.sender.is_logged_in()
+    user = evt.sender if is_logged_in else evt.tgbot
     try:
         entity = await user.client.get_entity(portal.peer)
     except Exception:
         evt.log.exception("Failed to get_entity(%s) for manual bridging.", portal.peer)
-        if evt.sender.logged_in:
+        if is_logged_in:
             return await evt.reply("Failed to get info of telegram chat. "
                                    "You are logged in, are you in that chat?")
         else:
             return await evt.reply("Failed to get info of telegram chat. "
                                    "You're not logged in, is the relay bot in the chat?")
+    if isinstance(entity, (ChatForbidden, ChannelForbidden)):
+        if is_logged_in:
+            return await evt.reply("You don't seem to be in that chat.")
+        else:
+            return await evt.reply("The bot doesn't seem to be in that chat.")
+
     direct = False
 
     portal.mxid = bridge_to_mxid
-    portal.title, portal.about, levels = await _get_initial_state(evt)
+    portal.title, portal.about, levels = await get_initial_state(evt.az.intent, evt.room_id)
     portal.photo_id = ""
     portal.save()
 
@@ -271,24 +302,32 @@ async def confirm_bridge(evt: CommandEvent):
     return await evt.reply("Bridging complete. Portal synchronization should begin momentarily.")
 
 
-async def _get_initial_state(evt: CommandEvent):
-    state = await evt.az.intent.get_room_state(evt.room_id)
+async def get_initial_state(intent: IntentAPI, room_id: str):
+    state = await intent.get_room_state(room_id)
     title = None
     about = None
     levels = None
     for event in state:
-        if event["type"] == "m.room.name":
-            title = event["content"]["name"]
-        elif event["type"] == "m.room.topic":
-            about = event["content"]["topic"]
-        elif event["type"] == "m.room.power_levels":
-            levels = event["content"]
-        elif event["type"] == "m.room.canonical_alias":
-            title = title or event["content"]["alias"]
+        try:
+            if event["type"] == "m.room.name":
+                title = event["content"]["name"]
+            elif event["type"] == "m.room.topic":
+                about = event["content"]["topic"]
+            elif event["type"] == "m.room.power_levels":
+                levels = event["content"]
+            elif event["type"] == "m.room.canonical_alias":
+                title = title or event["content"]["alias"]
+        except KeyError:
+            # Some state event probably has empty content
+            pass
     return title, about, levels
 
 
-@command_handler()
+@command_handler(help_section=SECTION_CREATING_PORTALS,
+                 help_args="[_type_]",
+                 help_text="Create a Telegram chat of the given type for the current Matrix room. "
+                           "The type is either `group`, `supergroup` or `channel` (defaults to "
+                           "`group`).")
 async def create(evt: CommandEvent):
     type = evt.args[0] if len(evt.args) > 0 else "group"
     if type not in {"chat", "group", "supergroup", "channel"}:
@@ -298,7 +337,10 @@ async def create(evt: CommandEvent):
     if po.Portal.get_by_mxid(evt.room_id):
         return await evt.reply("This is already a portal room.")
 
-    title, about, levels = await _get_initial_state(evt)
+    if not await user_has_power_level(evt.room_id, evt.az.intent, evt.sender, "bridge"):
+        return await evt.reply("You do not have the permissions to bridge this room.")
+
+    title, about, levels = await get_initial_state(evt.az.intent, evt.room_id)
     if not title:
         return await evt.reply("Please set a title before creating a Telegram chat.")
 
@@ -319,7 +361,8 @@ async def create(evt: CommandEvent):
     return await evt.reply(f"Telegram chat created. ID: {portal.tgid}")
 
 
-@command_handler()
+@command_handler(help_section=SECTION_PORTAL_MANAGEMENT,
+                 help_text="Upgrade a normal Telegram group to a supergroup.")
 async def upgrade(evt: CommandEvent):
     portal = po.Portal.get_by_mxid(evt.room_id)
     if not portal:
@@ -338,7 +381,10 @@ async def upgrade(evt: CommandEvent):
         return await evt.reply(e.args[0])
 
 
-@command_handler()
+@command_handler(help_section=SECTION_PORTAL_MANAGEMENT,
+                 help_args="<_name_|`-`>",
+                 help_text="Change the username of a supergroup/channel. "
+                           "To disable, use a dash (`-`) as the name.")
 async def group_name(evt: CommandEvent):
     if len(evt.args) == 0:
         return await evt.reply("**Usage:** `$cmdprefix+sp group-name <name/->`")
@@ -370,7 +416,11 @@ async def group_name(evt: CommandEvent):
         return await evt.reply("Invalid username")
 
 
-@command_handler(needs_admin=True)
+@command_handler(needs_admin=True,
+                 help_section=SECTION_ADMIN,
+                 help_args="<`whitelist`|`blacklist`>",
+                 help_text="Change whether the bridge will allow or disallow bridging rooms by "
+                           "default.")
 async def filter_mode(evt: CommandEvent):
     try:
         mode = evt.args[0]
@@ -392,7 +442,10 @@ async def filter_mode(evt: CommandEvent):
                                "`!filter blacklist <chat ID>`.")
 
 
-@command_handler(needs_admin=True)
+@command_handler(needs_admin=True,
+                 help_section=SECTION_ADMIN,
+                 help_args="<`whitelist`|`blacklist`> <_chat ID_>",
+                 help_text="Allow or disallow bridging a specific chat.")
 async def filter(evt: CommandEvent):
     try:
         action = evt.args[0]

mautrix_telegram/commands/telegram.py

@@ -20,11 +20,13 @@ from telethon.tl.functions.messages import ImportChatInviteRequest, CheckChatInv
 from telethon.tl.functions.channels import JoinChannelRequest
 
 from .. import puppet as pu, portal as po
-from . import command_handler
+from . import command_handler, CommandEvent, SECTION_MISC, SECTION_CREATING_PORTALS
 
 
-@command_handler()
-async def search(evt):
+@command_handler(help_section=SECTION_MISC,
+                 help_args="[_-r|--remote_] <_query_>",
+                 help_text="Search your contacts or the Telegram servers for users.")
+async def search(evt: CommandEvent):
     if len(evt.args) == 0:
         return await evt.reply("**Usage:** `$cmdprefix+sp search [-r|--remote] <query>`")
 
@@ -59,8 +61,14 @@ async def search(evt):
     return await evt.reply("\n".join(reply))
 
 
-@command_handler(name="pm")
-async def private_message(evt):
+@command_handler(name="pm",
+                 help_section=SECTION_CREATING_PORTALS,
+                 help_args="<_identifier_>",
+                 help_text="Open a private chat with the given Telegram user. The identifier is "
+                           "either the internal user ID, the username or the phone number. "
+                           "**N.B.** The phone numbers you start chats with must already be in "
+                           "your contacts.")
+async def private_message(evt: CommandEvent):
     if len(evt.args) == 0:
         return await evt.reply("**Usage:** `$cmdprefix+sp pm <user identifier>`")
 
@@ -79,7 +87,7 @@ async def private_message(evt):
                            f"{pu.Puppet.get_displayname(user, False)}")
 
 
-async def _join(evt, arg):
+async def _join(evt: CommandEvent, arg: str):
     if arg.startswith("joinchat/"):
         invite_hash = arg[len("joinchat/"):]
         try:
@@ -99,8 +107,10 @@ async def _join(evt, arg):
         return await evt.sender.client(JoinChannelRequest(channel)), None
 
 
-@command_handler()
-async def join(evt):
+@command_handler(help_section=SECTION_CREATING_PORTALS,
+                 help_args="<_link_>",
+                 help_text="Join a chat with an invite link.")
+async def join(evt: CommandEvent):
     if len(evt.args) == 0:
         return await evt.reply("**Usage:** `$cmdprefix+sp join <invite link>`")
 
@@ -124,8 +134,10 @@ async def join(evt):
             return await evt.reply(f"Created room for {portal.title}")
 
 
-@command_handler()
-async def sync(evt):
+@command_handler(help_section=SECTION_MISC,
+                 help_args="[`chats`|`contacts`|`me`]",
+                 help_text="Synchronize your chat portals, contacts and/or own info.")
+async def sync(evt: CommandEvent):
     if len(evt.args) > 0:
         sync_only = evt.args[0]
         if sync_only not in ("chats", "contacts", "me"):

mautrix_telegram/config.py

@@ -14,6 +14,7 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Tuple, Any, Optional
 from ruamel.yaml import YAML
 from ruamel.yaml.comments import CommentedMap
 import random
@@ -24,28 +25,28 @@ yaml.indent(4)
 
 
 class DictWithRecursion:
-    def __init__(self, data=None):
-        self._data = data or CommentedMap()
+    def __init__(self, data: CommentedMap = None):
+        self._data = data or CommentedMap()  # type: CommentedMap
 
-    def _recursive_get(self, data, key, default_value):
+    def _recursive_get(self, data: CommentedMap, key: str, default_value: Any) -> Any:
         if '.' in key:
             key, next_key = key.split('.', 1)
             next_data = data.get(key, CommentedMap())
             return self._recursive_get(next_data, next_key, default_value)
         return data.get(key, default_value)
 
-    def get(self, key, default_value, allow_recursion=True):
+    def get(self, key: str, default_value: Any, allow_recursion: bool = True) -> Any:
         if allow_recursion and '.' in key:
             return self._recursive_get(self._data, key, default_value)
         return self._data.get(key, default_value)
 
-    def __getitem__(self, key):
+    def __getitem__(self, key: str) -> Any:
         return self.get(key, None)
 
-    def __contains__(self, key):
+    def __contains__(self, key: str) -> bool:
         return self[key] is not None
 
-    def _recursive_set(self, data, key, value):
+    def _recursive_set(self, data: CommentedMap, key: str, value: Any):
         if '.' in key:
             key, next_key = key.split('.', 1)
             if key not in data:
@@ -55,16 +56,16 @@ class DictWithRecursion:
             return
         data[key] = value
 
-    def set(self, key, value, allow_recursion=True):
+    def set(self, key: str, value: Any, allow_recursion: bool = True):
         if allow_recursion and '.' in key:
             self._recursive_set(self._data, key, value)
             return
         self._data[key] = value
 
-    def __setitem__(self, key, value):
+    def __setitem__(self, key: str, value: Any):
         self.set(key, value)
 
-    def _recursive_del(self, data, key):
+    def _recursive_del(self, data: CommentedMap, key: str):
         if '.' in key:
             key, next_key = key.split('.', 1)
             if key not in data:
@@ -78,7 +79,7 @@ class DictWithRecursion:
         except KeyError:
             pass
 
-    def delete(self, key, allow_recursion=True):
+    def delete(self, key: str, allow_recursion: bool = True):
         if allow_recursion and '.' in key:
             self._recursive_del(self._data, key)
             return
@@ -88,23 +89,23 @@ class DictWithRecursion:
         except KeyError:
             pass
 
-    def __delitem__(self, key):
+    def __delitem__(self, key: str):
         self.delete(key)
 
 
 class Config(DictWithRecursion):
-    def __init__(self, path, registration_path, base_path):
+    def __init__(self, path: str, registration_path: str, base_path: str):
         super().__init__()
-        self.path = path
-        self.registration_path = registration_path
-        self.base_path = base_path
-        self._registration = None
+        self.path = path  # type: str
+        self.registration_path = registration_path  # type: str
+        self.base_path = base_path  # type: str
+        self._registration = None  # type: dict
 
     def load(self):
         with open(self.path, 'r') as stream:
             self._data = yaml.load(stream)
 
-    def load_base(self):
+    def load_base(self) -> Optional[DictWithRecursion]:
         try:
             with open(self.base_path, 'r') as stream:
                 return DictWithRecursion(yaml.load(stream))
@@ -120,7 +121,7 @@ class Config(DictWithRecursion):
                 yaml.dump(self._registration, stream)
 
     @staticmethod
-    def _new_token():
+    def _new_token() -> str:
         return "".join(random.choice(string.ascii_lowercase + string.digits) for _ in range(64))
 
     def update(self):
@@ -132,10 +133,11 @@ class Config(DictWithRecursion):
             if from_path in self:
                 base[to_path or from_path] = self[from_path]
 
-        def copy_dict(from_path, to_path=None):
+        def copy_dict(from_path, to_path=None, override_existing_map=True):
             if from_path in self:
                 to_path = to_path or from_path
-                base[to_path] = CommentedMap()
+                if override_existing_map or to_path not in base:
+                    base[to_path] = CommentedMap()
                 for key, value in self[from_path].items():
                     base[to_path][key] = value
 
@@ -143,9 +145,15 @@ class Config(DictWithRecursion):
         copy("homeserver.verify_ssl")
         copy("homeserver.domain")
 
-        copy("appservice.protocol")
+        if "appservice.protocol" in self and "appservice.address" not in self:
+            protocol, hostname, port = (self["appservice.protocol"], self["appservice.hostname"],
+                                        self["appservice.port"])
+            base["appservice.address"] = f"{protocol}://{hostname}:{port}"
+        else:
+            copy("appservice.address")
         copy("appservice.hostname")
         copy("appservice.port")
+        copy("appservice.max_body_size")
 
         copy("appservice.database")
 
@@ -153,11 +161,16 @@ class Config(DictWithRecursion):
         copy("appservice.public.prefix")
         copy("appservice.public.external")
 
-        copy("appservice.debug")
+        copy("appservice.provisioning.enabled")
+        copy("appservice.provisioning.prefix")
+        copy("appservice.provisioning.shared_secret")
+        if base["appservice.provisioning.shared_secret"] == "generate":
+            base["appservice.provisioning.shared_secret"] = self._new_token()
 
         copy("appservice.id")
         copy("appservice.bot_username")
         copy("appservice.bot_displayname")
+        copy("appservice.bot_avatar")
 
         copy("appservice.as_token")
         copy("appservice.hs_token")
@@ -173,6 +186,7 @@ class Config(DictWithRecursion):
         copy("bridge.bridge_notices")
         copy("bridge.bot_messages_as_notices")
         copy("bridge.max_initial_member_sync")
+        copy("bridge.sync_channel_members")
         copy("bridge.max_telegram_delete")
         copy("bridge.allow_matrix_login")
         copy("bridge.inline_images")
@@ -180,6 +194,17 @@ class Config(DictWithRecursion):
         copy("bridge.public_portals")
         copy("bridge.native_stickers")
         copy("bridge.catch_up")
+        copy("bridge.sync_with_custom_puppets")
+
+        copy("bridge.deduplication.pre_db_check")
+        copy("bridge.deduplication.cache_queue_length")
+
+        if "bridge.message_formats.m_text" in self:
+            del self["bridge.message_formats"]
+        copy_dict("bridge.message_formats", override_existing_map=False)
+        copy("bridge.state_event_formats.join")
+        copy("bridge.state_event_formats.leave")
+        copy("bridge.state_event_formats.name_change")
 
         copy("bridge.filter.mode")
         copy("bridge.filter.list")
@@ -205,22 +230,39 @@ class Config(DictWithRecursion):
             copy("bridge.relaybot.authless_portals")
             copy("bridge.relaybot.whitelist_group_admins")
             copy("bridge.relaybot.whitelist")
+            copy("bridge.relaybot.ignore_own_incoming_events")
 
         copy("telegram.api_id")
         copy("telegram.api_hash")
         copy("telegram.bot_token")
+        copy("telegram.proxy.type")
+        copy("telegram.proxy.address")
+        copy("telegram.proxy.port")
+        copy("telegram.proxy.rdns")
+        copy("telegram.proxy.username")
+        copy("telegram.proxy.password")
+
+        if "appservice.debug" in self and "logging" not in self:
+            level = "DEBUG" if self["appservice.debug"] else "INFO"
+            base["logging.root.level"] = level
+            base["logging.loggers.mau.level"] = level
+            base["logging.loggers.telethon.level"] = level
+        else:
+            copy("logging")
 
         self._data = base._data
         self.save()
 
-    def _get_permissions(self, key):
+    def _get_permissions(self, key: str) -> Tuple[bool, bool, bool, bool, bool, bool]:
         level = self["bridge.permissions"].get(key, "")
         admin = level == "admin"
-        whitelisted = level == "full" or admin
-        relaybot = level == "relaybot" or whitelisted
-        return relaybot, whitelisted, admin
+        matrix_puppeting = level == "full" or admin
+        puppeting = level == "puppeting" or matrix_puppeting
+        user = level == "user" or puppeting
+        relaybot = level == "relaybot" or user
+        return relaybot, user, puppeting, matrix_puppeting, admin, level
 
-    def get_permissions(self, mxid):
+    def get_permissions(self, mxid: str) -> Tuple[bool, bool, bool, bool, bool, bool]:
         permissions = self["bridge.permissions"] or {}
         if mxid in permissions:
             return self._get_permissions(mxid)
@@ -242,10 +284,8 @@ class Config(DictWithRecursion):
         self.set("appservice.as_token", self._new_token())
         self.set("appservice.hs_token", self._new_token())
 
-        url = (f"{self['appservice.protocol']}://"
-               f"{self['appservice.hostname']}:{self['appservice.port']}")
         self._registration = {
-            "id": self.get("appservice.id", "telegram"),
+            "id": self["appservice.id"] or "telegram",
             "as_token": self["appservice.as_token"],
             "hs_token": self["appservice.hs_token"],
             "namespaces": {
@@ -258,7 +298,7 @@ class Config(DictWithRecursion):
                     "regex": f"#{alias_format}:{homeserver}"
                 }]
             },
-            "url": url,
+            "url": self["appservice.address"],
             "sender_localpart": self["appservice.bot_username"],
             "rate_limited": False
         }

mautrix_telegram/context.py

@@ -14,17 +14,34 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import TYPE_CHECKING, Optional
+
+if TYPE_CHECKING:
+    import asyncio
+
+    from sqlalchemy.orm import scoped_session
+
+    from alchemysession import AlchemySessionContainer
+    from mautrix_appservice import AppService
+
+    from .web import PublicBridgeWebsite, ProvisioningAPI
+    from .config import Config
+    from .bot import Bot
+    from .matrix import MatrixHandler
 
 
 class Context:
-    def __init__(self, az, db, config, loop, bot, mx, telethon_session_container):
-        self.az = az
-        self.db = db
-        self.config = config
-        self.loop = loop
-        self.bot = bot
-        self.mx = mx
-        self.telethon_session_container = telethon_session_container
+    def __init__(self, az: "AppService", db: "scoped_session", config: "Config",
+                 loop: "asyncio.AbstractEventLoop", session_container: "AlchemySessionContainer"):
+        self.az = az  # type: AppService
+        self.db = db  # type: scoped_session
+        self.config = config  # type: Config
+        self.loop = loop  # type: asyncio.AbstractEventLoop
+        self.bot = None  # type: Optional[Bot]
+        self.mx = None  # type: MatrixHandler
+        self.session_container = session_container  # type: AlchemySessionContainer
+        self.public_website = None  # type: PublicBridgeWebsite
+        self.provisioning_api = None  # type: ProvisioningAPI
 
     def __iter__(self):
         yield self.az

mautrix_telegram/db.py

@@ -15,20 +15,22 @@
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 from sqlalchemy import (Column, UniqueConstraint, ForeignKey, ForeignKeyConstraint, Integer,
-                        BigInteger, String, Boolean)
-from sqlalchemy.orm import relationship
+                        BigInteger, String, Boolean, Text)
+from sqlalchemy.sql import expression
+from sqlalchemy.orm import relationship, Query
+import json
 
 from .base import Base
 
 
 class Portal(Base):
-    query = None
+    query = None  # type: Query
     __tablename__ = "portal"
 
     # Telegram chat information
     tgid = Column(Integer, primary_key=True)
     tg_receiver = Column(Integer, primary_key=True)
-    peer_type = Column(String)
+    peer_type = Column(String, nullable=False)
     megagroup = Column(Boolean)
 
     # Matrix portal information
@@ -42,7 +44,7 @@ class Portal(Base):
 
 
 class Message(Base):
-    query = None
+    query = None  # type: Query
     __tablename__ = "message"
 
     mxid = Column(String)
@@ -54,32 +56,76 @@ class Message(Base):
 
 
 class UserPortal(Base):
-    query = None
+    query = None  # type: Query
     __tablename__ = "user_portal"
 
-    user = Column(Integer, ForeignKey("user.tgid"), primary_key=True)
+    user = Column(Integer, ForeignKey("user.tgid", onupdate="CASCADE", ondelete="CASCADE"),
+                  primary_key=True)
     portal = Column(Integer, primary_key=True)
     portal_receiver = Column(Integer, primary_key=True)
 
     __table_args__ = (ForeignKeyConstraint(("portal", "portal_receiver"),
-                                           ("portal.tgid", "portal.tg_receiver")),)
+                                           ("portal.tgid", "portal.tg_receiver"),
+                                           onupdate="CASCADE", ondelete="CASCADE"),)
 
 
 class User(Base):
-    query = None
+    query = None  # type: Query
     __tablename__ = "user"
 
     mxid = Column(String, primary_key=True)
     tgid = Column(Integer, nullable=True, unique=True)
     tg_username = Column(String, nullable=True)
-    saved_contacts = Column(Integer, default=0)
+    saved_contacts = Column(Integer, default=0, nullable=False)
     contacts = relationship("Contact", uselist=True,
                             cascade="save-update, merge, delete, delete-orphan")
     portals = relationship("Portal", secondary="user_portal")
 
 
+class RoomState(Base):
+    query = None  # type: Query
+    __tablename__ = "mx_room_state"
+
+    room_id = Column(String, primary_key=True)
+    _power_levels_text = Column("power_levels", Text, nullable=True)
+    _power_levels_json = None
+
+    @property
+    def has_power_levels(self):
+        return bool(self._power_levels_text)
+
+    @property
+    def power_levels(self):
+        if not self._power_levels_json and self._power_levels_text:
+            self._power_levels_json = json.loads(self._power_levels_text)
+        return self._power_levels_json or {}
+
+    @power_levels.setter
+    def power_levels(self, val):
+        self._power_levels_json = val
+        self._power_levels_text = json.dumps(val)
+
+
+class UserProfile(Base):
+    query = None  # type: Query
+    __tablename__ = "mx_user_profile"
+
+    room_id = Column(String, primary_key=True)
+    user_id = Column(String, primary_key=True)
+    membership = Column(String, nullable=False, default="leave")
+    displayname = Column(String, nullable=True)
+    avatar_url = Column(String, nullable=True)
+
+    def dict(self):
+        return {
+            "membership": self.membership,
+            "displayname": self.displayname,
+            "avatar_url": self.avatar_url,
+        }
+
+
 class Contact(Base):
-    query = None
+    query = None  # type: Query
     __tablename__ = "contact"
 
     user = Column(Integer, ForeignKey("user.tgid"), primary_key=True)
@@ -87,27 +133,30 @@ class Contact(Base):
 
 
 class Puppet(Base):
-    query = None
+    query = None  # type: Query
     __tablename__ = "puppet"
 
     id = Column(Integer, primary_key=True)
+    custom_mxid = Column(String, nullable=True)
+    access_token = Column(String, nullable=True)
     displayname = Column(String, nullable=True)
     displayname_source = Column(Integer, nullable=True)
     username = Column(String, nullable=True)
     photo_id = Column(String, nullable=True)
     is_bot = Column(Boolean, nullable=True)
+    matrix_registered = Column(Boolean, nullable=False, server_default=expression.false())
 
 
 # Fucking Telegram not telling bots what chats they are in 3:<
 class BotChat(Base):
-    query = None
+    query = None  # type: Query
     __tablename__ = "bot_chat"
     id = Column(Integer, primary_key=True)
     type = Column(String, nullable=False)
 
 
 class TelegramFile(Base):
-    query = None
+    query = None  # type: Query
     __tablename__ = "telegram_file"
 
     id = Column(String, primary_key=True)
@@ -130,3 +179,5 @@ def init(db_session):
     Puppet.query = db_session.query_property()
     BotChat.query = db_session.query_property()
     TelegramFile.query = db_session.query_property()
+    UserProfile.query = db_session.query_property()
+    RoomState.query = db_session.query_property()

mautrix_telegram/formatter/__init__.py

@@ -1,9 +1,9 @@
 from .from_matrix import (matrix_reply_to_telegram, matrix_to_telegram, matrix_text_to_telegram,
                           init_mx)
 from .from_telegram import (telegram_reply_to_matrix, telegram_to_matrix, init_tg)
-from ..context import Context
+from .. import context as c
 
 
-def init(context: Context):
+def init(context: c.Context):
     init_mx(context)
     init_tg(context)

mautrix_telegram/formatter/from_matrix/__init__.py

@@ -0,0 +1,156 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2018 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Optional, List, Tuple, Callable, Pattern, Match, TYPE_CHECKING
+import re
+import logging
+
+from telethon.tl.types import (MessageEntityMention, MessageEntityMentionName, MessageEntityItalic,
+                               TypeMessageEntity)
+
+from ... import puppet as pu
+from ...db import Message as DBMessage
+from ..util import (add_surrogates, remove_surrogates, trim_reply_fallback_html,
+                    trim_reply_fallback_text)
+from .parser_common import ParsedMessage
+
+try:
+    from mautrix_telegram.formatter.from_matrix.parser_lxml import parse_html
+except ImportError:
+    from mautrix_telegram.formatter.from_matrix.parser_htmlparser import parse_html
+
+if TYPE_CHECKING:
+    from ...context import Context
+
+log = logging.getLogger("mau.fmt.mx")  # type: logging.Logger
+should_bridge_plaintext_highlights = False  # type: bool
+
+command_regex = re.compile(r"^!([A-Za-z0-9@]+)")  # type: Pattern
+not_command_regex = re.compile(r"^\\(![A-Za-z0-9@]+)")  # type: Pattern
+plain_mention_regex = None  # type: Pattern
+
+
+def plain_mention_to_html(match: Match) -> str:
+    puppet = pu.Puppet.find_by_displayname(match.group(2))
+    if puppet:
+        return (f"{match.group(1)}"
+                f"<a href='https://matrix.to/#/{puppet.mxid}'>"
+                f"{puppet.displayname}"
+                "</a>")
+    return "".join(match.groups())
+
+
+def cut_long_message(message: str, entities: List[TypeMessageEntity]) -> ParsedMessage:
+    if len(message) > 4096:
+        message = message[0:4082] + " [message cut]"
+        new_entities = []
+        for entity in entities:
+            if entity.offset > 4082:
+                continue
+            if entity.offset + entity.length > 4082:
+                entity.length = 4082 - entity.offset
+            new_entities.append(entity)
+        new_entities.append(MessageEntityItalic(4082, len(" [message cut]")))
+        entities = new_entities
+    return message, entities
+
+
+class FormatError(Exception):
+    pass
+
+
+def matrix_to_telegram(html: str) -> ParsedMessage:
+    try:
+        html = command_regex.sub(r"<command>\1</command>", html)
+        html = html.replace("\t", " " * 4)
+        html = not_command_regex.sub(r"\1", html)
+        if should_bridge_plaintext_highlights:
+            html = plain_mention_regex.sub(plain_mention_to_html, html)
+
+        html = add_surrogates(html)
+        text, entities = parse_html(add_surrogates(html))
+        text = remove_surrogates(text.strip())
+        text, entities = cut_long_message(text, entities)
+
+        return text, entities
+    except Exception as e:
+        raise FormatError(f"Failed to convert Matrix format: {html}") from e
+
+
+def matrix_reply_to_telegram(content: dict, tg_space: int, room_id: Optional[str] = None
+                             ) -> Optional[int]:
+    try:
+        reply = content["m.relates_to"]["m.in_reply_to"]
+        room_id = room_id or reply["room_id"]
+        event_id = reply["event_id"]
+
+        try:
+            if content["format"] == "org.matrix.custom.html":
+                content["formatted_body"] = trim_reply_fallback_html(content["formatted_body"])
+        except KeyError:
+            pass
+        content["body"] = trim_reply_fallback_text(content["body"])
+
+        message = DBMessage.query.filter(DBMessage.mxid == event_id,
+                                         DBMessage.tg_space == tg_space,
+                                         DBMessage.mx_room == room_id).one_or_none()
+        if message:
+            return message.tgid
+    except KeyError:
+        pass
+    return None
+
+
+def matrix_text_to_telegram(text: str) -> ParsedMessage:
+    text = command_regex.sub(r"/\1", text)
+    text = text.replace("\t", " " * 4)
+    text = not_command_regex.sub(r"\1", text)
+    if should_bridge_plaintext_highlights:
+        entities, pmr_replacer = plain_mention_to_text()
+        text = plain_mention_regex.sub(pmr_replacer, text)
+    else:
+        entities = []
+    return text, entities
+
+
+def plain_mention_to_text() -> Tuple[List[TypeMessageEntity], Callable[[str], str]]:
+    entities = []
+
+    def replacer(match) -> str:
+        puppet = pu.Puppet.find_by_displayname(match.group(2))
+        if puppet:
+            offset = match.start()
+            length = match.end() - offset
+            if puppet.username:
+                entity = MessageEntityMention(offset, length)
+                text = f"@{puppet.username}"
+            else:
+                entity = MessageEntityMentionName(offset, length, user_id=puppet.tgid)
+                text = puppet.displayname
+            entities.append(entity)
+            return text
+        return "".join(match.groups())
+
+    return entities, replacer
+
+
+def init_mx(context: "Context"):
+    global plain_mention_regex, should_bridge_plaintext_highlights
+    config = context.config
+    dn_template = config.get("bridge.displayname_template", "{displayname} (Telegram)")
+    dn_template = re.escape(dn_template).replace(re.escape("{displayname}"), "[^>]+")
+    plain_mention_regex = re.compile(f"(\s|^)({dn_template})")
+    should_bridge_plaintext_highlights = config["bridge.plaintext_highlights"] or False

mautrix_telegram/formatter/from_matrix/parser_common.py

@@ -0,0 +1,31 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2018 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+import re
+from typing import List, Tuple, Pattern
+from telethon.tl.types import TypeMessageEntity
+
+
+class MatrixParserCommon:
+    mention_regex = re.compile("https://matrix.to/#/(@.+:.+)")  # type: Pattern
+    room_regex = re.compile("https://matrix.to/#/(#.+:.+)")  # type: Pattern
+    block_tags = ("br", "p", "pre", "blockquote",
+                  "ol", "ul", "li",
+                  "h1", "h2", "h3", "h4", "h5", "h6",
+                  "div", "hr", "table")  # type: Tuple[str, ...]
+
+
+ParsedMessage = Tuple[str, List[TypeMessageEntity]]

mautrix_telegram/formatter/from_matrix/parser_htmlparser.py

@@ -14,52 +14,43 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import (Optional, List, Tuple, Type, Dict, Any, Deque, Match)
 from html import unescape
 from html.parser import HTMLParser
 from collections import deque
-from typing import Optional, List, Tuple, Type, Callable, Dict, Any
 import math
-import re
-import logging
 
-from telethon.tl.types import (MessageEntityMention,
-                               InputMessageEntityMentionName, MessageEntityEmail,
+from telethon.tl.types import (MessageEntityMention, MessageEntityMentionName, MessageEntityEmail,
                                MessageEntityUrl, MessageEntityTextUrl, MessageEntityBold,
                                MessageEntityItalic, MessageEntityCode, MessageEntityPre,
-                               MessageEntityBotCommand, InputUser, TypeMessageEntity)
+                               MessageEntityBotCommand, TypeMessageEntity)
 
-from ..context import Context
-from .. import user as u, puppet as pu, portal as po
-from ..db import Message as DBMessage
-from .util import (add_surrogates, remove_surrogates, trim_reply_fallback_html,
-                   trim_reply_fallback_text, html_to_unicode)
-
-log = logging.getLogger("mau.fmt.mx")
-should_bridge_plaintext_highlights = False
+from ... import user as u, puppet as pu, portal as po
+from ..util import html_to_unicode
+from .parser_common import MatrixParserCommon, ParsedMessage
 
 
-class MatrixParser(HTMLParser):
-    mention_regex = re.compile("https://matrix.to/#/(@.+:.+)")
-    room_regex = re.compile("https://matrix.to/#/(#.+:.+)")
-    block_tags = ("br", "p", "pre", "blockquote",
-                  "ol", "ul", "li",
-                  "h1", "h2", "h3", "h4", "h5", "h6",
-                  "div", "hr", "table")
+def parse_html(html: str) -> ParsedMessage:
+    parser = MatrixParser()
+    parser.feed(html)
+    return parser.text, parser.entities
 
+
+class MatrixParser(HTMLParser, MatrixParserCommon):
     def __init__(self):
-        super().__init__()
-        self.text = ""
-        self.entities = []
-        self._building_entities = {}
-        self._list_counter = 0
-        self._open_tags = deque()
-        self._open_tags_meta = deque()
-        self._line_is_new = True
-        self._list_entry_is_new = False
+        super(MatrixParser, self).__init__()
+        self.text = ""  # type: str
+        self.entities = []  # type: List[TypeMessageEntity]
+        self._building_entities = {}  # type: Dict[str, TypeMessageEntity]
+        self._list_counter = 0  # type: int
+        self._open_tags = deque()  # type: Deque[str]
+        self._open_tags_meta = deque()  # type: Deque[Any]
+        self._line_is_new = True  # type: bool
+        self._list_entry_is_new = False  # type: bool
 
     def _parse_url(self, url: str, args: Dict[str, Any]
                    ) -> Tuple[Optional[Type[TypeMessageEntity]], Optional[str]]:
-        mention = self.mention_regex.match(url)
+        mention = self.mention_regex.match(url)  # type: Match
         if mention:
             mxid = mention.group(1)
             user = (pu.Puppet.get_by_mxid(mxid)
@@ -69,12 +60,12 @@ class MatrixParser(HTMLParser):
             if user.username:
                 return MessageEntityMention, f"@{user.username}"
             elif user.tgid:
-                args["user_id"] = InputUser(user.tgid, 0)
-                return InputMessageEntityMentionName, user.displayname or None
+                args["user_id"] = user.tgid
+                return MessageEntityMentionName, user.displayname or None
             else:
                 return None, None
 
-        room = self.room_regex.match(url)
+        room = self.room_regex.match(url)  # type: Match
         if room:
             username = po.Portal.get_username_from_mx_alias(room.group(1))
             portal = po.Portal.find_by_username(username)
@@ -94,8 +85,8 @@ class MatrixParser(HTMLParser):
         self._open_tags_meta.appendleft(0)
 
         attrs = dict(attrs)
-        entity_type = None
-        args = {}
+        entity_type = None  # type: type(TypeMessageEntity)
+        args = {}  # type: Dict[str, Any]
         if tag in ("strong", "b"):
             entity_type = MessageEntityBold
         elif tag in ("em", "i"):
@@ -243,97 +234,3 @@ class MatrixParser(HTMLParser):
 
         if tag in self.block_tags and tag != "br" and "blockquote" not in self._open_tags:
             self._newline(allow_multi=tag == "br")
-
-
-command_regex = re.compile(r"^!([A-Za-z0-9@]+)")
-not_command_regex = re.compile(r"^\\(![A-Za-z0-9@]+)")
-plain_mention_regex = None
-
-
-def plain_mention_to_html(match):
-    puppet = pu.Puppet.find_by_displayname(match.group(2))
-    if puppet:
-        return (f"{match.group(1)}"
-                f"<a href='https://matrix.to/#/{puppet.mxid}'>"
-                f"{puppet.displayname}"
-                "</a>")
-    return "".join(match.groups())
-
-
-def matrix_to_telegram(html: str) -> Tuple[str, List[TypeMessageEntity]]:
-    try:
-        parser = MatrixParser()
-        html = command_regex.sub(r"<command>\1</command>", html)
-        html = not_command_regex.sub(r"\1", html)
-        if should_bridge_plaintext_highlights:
-            html = plain_mention_regex.sub(plain_mention_to_html, html)
-        parser.feed(add_surrogates(html))
-        return remove_surrogates(parser.text.strip()), parser.entities
-    except Exception:
-        log.exception("Failed to convert Matrix format:\nhtml=%s", html)
-
-
-def matrix_reply_to_telegram(content: dict, tg_space: int, room_id: Optional[str] = None
-                             ) -> Optional[int]:
-    try:
-        reply = content["m.relates_to"]["m.in_reply_to"]
-        room_id = room_id or reply["room_id"]
-        event_id = reply["event_id"]
-
-        try:
-            if content["format"] == "org.matrix.custom.html":
-                content["formatted_body"] = trim_reply_fallback_html(content["formatted_body"])
-        except KeyError:
-            pass
-        content["body"] = trim_reply_fallback_text(content["body"])
-
-        message = DBMessage.query.filter(DBMessage.mxid == event_id,
-                                         DBMessage.tg_space == tg_space,
-                                         DBMessage.mx_room == room_id).one_or_none()
-        if message:
-            return message.tgid
-    except KeyError:
-        pass
-    return None
-
-
-def matrix_text_to_telegram(text: str) -> Tuple[str, List[TypeMessageEntity]]:
-    text = command_regex.sub(r"/\1", text)
-    text = not_command_regex.sub(r"\1", text)
-    if should_bridge_plaintext_highlights:
-        entities, pmr_replacer = plain_mention_to_text()
-        text = plain_mention_regex.sub(pmr_replacer, text)
-    else:
-        entities = []
-    return text, entities
-
-
-def plain_mention_to_text() -> Tuple[List[TypeMessageEntity], Callable[[str], str]]:
-    entities = []
-
-    def replacer(match):
-        puppet = pu.Puppet.find_by_displayname(match.group(2))
-        if puppet:
-            offset = match.start()
-            length = match.end() - offset
-            if puppet.username:
-                entity = MessageEntityMention(offset, length)
-                text = f"@{puppet.username}"
-            else:
-                entity = InputMessageEntityMentionName(offset, length,
-                                                       user_id=InputUser(puppet.tgid, 0))
-                text = puppet.displayname
-            entities.append(entity)
-            return text
-        return "".join(match.groups())
-
-    return entities, replacer
-
-
-def init_mx(context: Context):
-    global plain_mention_regex, should_bridge_plaintext_highlights
-    config = context.config
-    dn_template = config.get("bridge.displayname_template", "{displayname} (Telegram)")
-    dn_template = re.escape(dn_template).replace(re.escape("{displayname}"), "[^>]+")
-    plain_mention_regex = re.compile(f"(\s|^)({dn_template})")
-    should_bridge_plaintext_highlights = config["bridge.plaintext_highlights"] or False

mautrix_telegram/formatter/from_matrix/parser_lxml.py

@@ -0,0 +1,343 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2018 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Optional, List, Tuple, Union, Callable
+from lxml import html
+
+from telethon.tl.types import (MessageEntityMention as Mention,
+                               MessageEntityMentionName as MentionName, MessageEntityEmail as Email,
+                               MessageEntityUrl as URL, MessageEntityTextUrl as TextURL,
+                               MessageEntityBold as Bold, MessageEntityItalic as Italic,
+                               MessageEntityCode as Code, MessageEntityPre as Pre,
+                               MessageEntityBotCommand as Command, TypeMessageEntity,
+                               InputMessageEntityMentionName as InputMentionName)
+
+from ... import user as u, puppet as pu, portal as po
+from ..util import html_to_unicode
+from .parser_common import MatrixParserCommon, ParsedMessage
+
+
+def parse_html(html: str) -> ParsedMessage:
+    return MatrixParser.parse(html)
+
+
+class Entity:
+    @staticmethod
+    def copy(entity: TypeMessageEntity) -> Optional[TypeMessageEntity]:
+        if not entity:
+            return None
+        kwargs = {
+            "offset": entity.offset,
+            "length": entity.length,
+        }
+        if isinstance(entity, Pre):
+            kwargs["language"] = entity.language
+        elif isinstance(entity, TextURL):
+            kwargs["url"] = entity.url
+        elif isinstance(entity, (MentionName, InputMentionName)):
+            kwargs["user_id"] = entity.user_id
+        return entity.__class__(**kwargs)
+
+    @classmethod
+    def adjust(cls, entity: Union[TypeMessageEntity, List[TypeMessageEntity]],
+               func: Callable[[TypeMessageEntity], None]
+               ) -> Union[Optional[TypeMessageEntity], List[TypeMessageEntity]]:
+        if isinstance(entity, list):
+            return [Entity.adjust(element, func) for element in entity if entity]
+        elif not entity:
+            return None
+        entity = cls.copy(entity)
+        func(entity)
+        if entity.offset < 0:
+            entity.length += entity.offset
+            entity.offset = 0
+        return entity
+
+
+def offset_diff(amount: int):
+    def func(entity: TypeMessageEntity):
+        entity.offset += amount
+
+    return func
+
+
+def offset_length_multiply(amount: int):
+    def func(entity: TypeMessageEntity):
+        entity.offset *= amount
+        entity.length *= amount
+
+    return func
+
+
+class TelegramMessage:
+    def __init__(self, text: str = "", entities: Optional[List[TypeMessageEntity]] = None):
+        self.text = text  # type: str
+        self.entities = entities or []  # type: List[TypeMessageEntity]
+
+    def offset_entities(self, offset: int) -> "TelegramMessage":
+        def apply_offset(entity: TypeMessageEntity, inner_offset: int
+                         ) -> Optional[TypeMessageEntity]:
+            entity = Entity.copy(entity)
+            entity.offset += inner_offset
+            if entity.offset < 0:
+                entity.offset = 0
+            elif entity.offset > len(self.text):
+                return None
+            elif entity.offset + entity.length > len(self.text):
+                entity.length = len(self.text) - entity.offset
+            return entity
+
+        self.entities = [apply_offset(entity, offset) for entity in self.entities if entity]
+        self.entities = [x for x in self.entities if x is not None]
+        return self
+
+    def append(self, *args: Union[str, "TelegramMessage"]) -> "TelegramMessage":
+        for msg in args:
+            if isinstance(msg, str):
+                msg = TelegramMessage(text=msg)
+            self.entities += Entity.adjust(msg.entities, offset_diff(len(self.text)))
+            self.text += msg.text
+        return self
+
+    def prepend(self, *args: Union[str, "TelegramMessage"]) -> "TelegramMessage":
+        for msg in args:
+            if isinstance(msg, str):
+                msg = TelegramMessage(text=msg)
+            self.entities = msg.entities + Entity.adjust(self.entities, offset_diff(len(msg.text)))
+            self.text = msg.text + self.text
+        return self
+
+    def format(self, entity_type: type(TypeMessageEntity), offset: int = None, length: int = None,
+               **kwargs) -> "TelegramMessage":
+        self.entities.append(entity_type(offset=offset or 0,
+                                         length=length if length is not None else len(self.text),
+                                         **kwargs))
+        return self
+
+    def concat(self, *args: Union[str, "TelegramMessage"]) -> "TelegramMessage":
+        return TelegramMessage().append(self, *args)
+
+    def trim(self) -> "TelegramMessage":
+        orig_len = len(self.text)
+        self.text = self.text.lstrip()
+        diff = orig_len - len(self.text)
+        self.text = self.text.rstrip()
+        self.offset_entities(-diff)
+        return self
+
+    def split(self, separator, max_items: int = 0) -> List["TelegramMessage"]:
+        text_parts = self.text.split(separator, max_items - 1)
+        output = []  # type: List[TelegramMessage]
+
+        offset = 0
+        for part in text_parts:
+            msg = TelegramMessage(part)
+            for entity in self.entities:
+                start_in_range = len(part) > entity.offset - offset >= 0
+                end_in_range = len(part) >= entity.offset - offset + entity.length > 0
+                if start_in_range and end_in_range:
+                    msg.entities.append(Entity.adjust(entity, offset_diff(-offset)))
+            output.append(msg)
+
+            offset += len(part)
+            offset += len(separator)
+
+        return output
+
+    @staticmethod
+    def join(items: List[Union[str, "TelegramMessage"]], separator: str = " ") -> "TelegramMessage":
+        main = TelegramMessage()
+        for msg in items:
+            if isinstance(msg, str):
+                msg = TelegramMessage(text=msg)
+            main.entities += Entity.adjust(msg.entities, offset_diff(len(main.text)))
+            main.text += msg.text + separator
+        main.text = main.text[:-len(separator)]
+        return main
+
+
+class MatrixParser(MatrixParserCommon):
+    @classmethod
+    def list_to_tmessage(cls, node: html.HtmlElement, strip_linebreaks) -> TelegramMessage:
+        ordered = node.tag == "ol"
+        tagged_children = cls.node_to_tagged_tmessages(node, strip_linebreaks)
+        counter = 1
+        indent_length = 0
+        if ordered:
+            try:
+                counter = int(node.attrib.get("start", "1"))
+            except ValueError:
+                counter = 1
+
+            longest_index = counter - 1 + len(tagged_children)
+            indent_length = len(str(longest_index))
+        indent = (indent_length + 4) * " "
+        children = []  # type: List[TelegramMessage]
+        for child, tag in tagged_children:
+            if tag != "li":
+                continue
+
+            if ordered:
+                prefix = f"{counter}. "
+                counter += 1
+            else:
+                prefix = "● "
+            child = child.prepend(prefix)
+            parts = child.split("\n")
+            parts = parts[:1] + [part.prepend(indent) for part in parts[1:]]
+            child = TelegramMessage.join(parts, "\n")
+            children.append(child)
+        return TelegramMessage.join(children, "\n")
+
+    @classmethod
+    def blockquote_to_tmessage(cls, node: html.HtmlElement, strip_linebreaks) -> TelegramMessage:
+        msg = cls.tag_aware_parse_node(node, strip_linebreaks)
+        children = msg.trim().split("\n")
+        children = [child.prepend("> ") for child in children]
+        return TelegramMessage.join(children, "\n")
+
+    @classmethod
+    def header_to_tmessage(cls, node: html.HtmlElement, strip_linebreaks) -> TelegramMessage:
+        children = cls.node_to_tmessages(node, strip_linebreaks)
+        length = int(node.tag[1])
+        prefix = "#" * length + " "
+        return TelegramMessage.join(children, "").prepend(prefix)
+
+    @classmethod
+    def basic_format_to_tmessage(cls, node: html.HtmlElement, strip_linebreaks) -> TelegramMessage:
+        msg = cls.tag_aware_parse_node(node, strip_linebreaks)
+        if node.tag in ("b", "strong"):
+            msg.format(Bold)
+        elif node.tag in ("i", "em"):
+            msg.format(Italic)
+        elif node.tag == "command":
+            msg.format(Command)
+        elif node.tag in ("s", "del"):
+            msg.text = html_to_unicode(msg.text, "\u0336")
+        elif node.tag in ("u", "ins"):
+            msg.text = html_to_unicode(msg.text, "\u0332")
+
+        if node.tag in ("s", "del", "u", "ins"):
+            msg.entities = Entity.adjust(msg.entities, offset_length_multiply(2))
+
+        return msg
+
+    @classmethod
+    def link_to_tstring(cls, node: html.HtmlElement, strip_linebreaks) -> TelegramMessage:
+        msg = cls.tag_aware_parse_node(node, strip_linebreaks)
+        href = node.attrib.get("href", "")
+        if not href:
+            return msg
+
+        if href.startswith("mailto:"):
+            return TelegramMessage(href[len("mailto:"):]).format(Email)
+
+        mention = cls.mention_regex.match(href)
+        if mention:
+            mxid = mention.group(1)
+            user = (pu.Puppet.get_by_mxid(mxid)
+                    or u.User.get_by_mxid(mxid, create=False))
+            if not user:
+                return msg
+            if user.username:
+                return TelegramMessage(f"@{user.username}").format(Mention)
+            elif user.tgid:
+                return TelegramMessage(user.displayname or msg.text).format(MentionName,
+                                                                            user_id=user.tgid)
+            return msg
+
+        room = cls.room_regex.match(href)
+        if room:
+            username = po.Portal.get_username_from_mx_alias(room.group(1))
+            portal = po.Portal.find_by_username(username)
+            if portal and portal.username:
+                return TelegramMessage(f"@{portal.username}").format(Mention)
+
+        return (msg.format(URL)
+                if msg.text == href
+                else msg.format(TextURL, url=href))
+
+    @classmethod
+    def node_to_tmessage(cls, node: html.HtmlElement, strip_linebreaks) -> TelegramMessage:
+        if node.tag == "blockquote":
+            return cls.blockquote_to_tmessage(node, strip_linebreaks)
+        elif node.tag in ("ol", "ul"):
+            return cls.list_to_tmessage(node, strip_linebreaks)
+        elif node.tag in ("h1", "h2", "h3", "h4", "h5", "h6"):
+            return cls.header_to_tmessage(node, strip_linebreaks)
+        elif node.tag == "br":
+            return TelegramMessage("\n")
+        elif node.tag in ("b", "strong", "i", "em", "s", "del", "u", "ins", "command"):
+            return cls.basic_format_to_tmessage(node, strip_linebreaks)
+        elif node.tag == "a":
+            return cls.link_to_tstring(node, strip_linebreaks)
+        elif node.tag == "p":
+            return cls.tag_aware_parse_node(node, strip_linebreaks).append("\n")
+        elif node.tag == "pre":
+            lang = ""
+            try:
+                if node[0].tag == "code":
+                    lang = node[0].attrib["class"][len("language-"):]
+                    node = node[0]
+            except (IndexError, KeyError):
+                pass
+            return cls.parse_node(node, strip_linebreaks=False).format(Pre, language=lang)
+        elif node.tag == "code":
+            return cls.parse_node(node, strip_linebreaks=False).format(Code)
+        return cls.tag_aware_parse_node(node, strip_linebreaks)
+
+    @staticmethod
+    def text_to_tmessage(text: str, strip_linebreaks: bool = True) -> TelegramMessage:
+        if strip_linebreaks:
+            text = text.replace("\n", "")
+        return TelegramMessage(text)
+
+    @classmethod
+    def node_to_tagged_tmessages(cls, node: html.HtmlElement, strip_linebreaks: bool = True
+                                 ) -> List[Tuple[TelegramMessage, str]]:
+        output = []
+
+        if node.text:
+            output.append((cls.text_to_tmessage(node.text, strip_linebreaks), "text"))
+        for child in node:
+            output.append((cls.node_to_tmessage(child, strip_linebreaks), child.tag))
+            if child.tail:
+                output.append((cls.text_to_tmessage(child.tail, strip_linebreaks), "text"))
+        return output
+
+    @classmethod
+    def node_to_tmessages(cls, node: html.HtmlElement, strip_linebreaks) -> List[TelegramMessage]:
+        return [msg for (msg, tag) in cls.node_to_tagged_tmessages(node, strip_linebreaks)]
+
+    @classmethod
+    def tag_aware_parse_node(cls, node: html.HtmlElement, strip_linebreaks) -> TelegramMessage:
+        msgs = cls.node_to_tagged_tmessages(node, strip_linebreaks)
+        output = TelegramMessage()
+        for msg, tag in msgs:
+            if tag in cls.block_tags:
+                msg = msg.append("\n").prepend("\n")
+            output = output.append(msg)
+        return output.trim()
+
+    @classmethod
+    def parse_node(cls, node: html.HtmlElement, strip_linebreaks) -> TelegramMessage:
+        return TelegramMessage.join(cls.node_to_tmessages(node, strip_linebreaks))
+
+    @classmethod
+    def parse(cls, data: str) -> ParsedMessage:
+        document = html.fromstring(f"<html>{data}</html>")
+        msg = cls.parse_node(document, strip_linebreaks=True)
+        return msg.text, msg.entities

mautrix_telegram/formatter/from_telegram.py

@@ -14,13 +14,8 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Optional, List, Tuple, TYPE_CHECKING
 from html import escape
-from typing import Optional, List, Tuple
-
-try:
-    from lxml.html.diff import htmldiff
-except ImportError:
-    htmldiff = None  # type: function
 import logging
 import re
 
@@ -34,16 +29,25 @@ from mautrix_appservice import MatrixRequestError
 from mautrix_appservice.intent_api import IntentAPI
 
 from .. import user as u, puppet as pu, portal as po
-from ..context import Context
 from ..db import Message as DBMessage
 from .util import (add_surrogates, remove_surrogates, trim_reply_fallback_html,
                    trim_reply_fallback_text, unicode_to_html)
 
-log = logging.getLogger("mau.fmt.tg")
-should_highlight_edits = False
+if TYPE_CHECKING:
+    from ..abstract_user import AbstractUser
+    from ..context import Context
+
+try:
+    from lxml.html.diff import htmldiff
+except ImportError:
+    htmldiff = None  # type: function
 
 
-def telegram_reply_to_matrix(evt: Message, source: u.User) -> dict:
+log = logging.getLogger("mau.fmt.tg")  # type: logging.Logger
+should_highlight_edits = False  # type: bool
+
+
+def telegram_reply_to_matrix(evt: Message, source: "AbstractUser") -> dict:
     if evt.reply_to_msg_id:
         space = (evt.to_id.channel_id
                  if isinstance(evt, Message) and isinstance(evt.to_id, PeerChannel)
@@ -79,7 +83,7 @@ async def _add_forward_header(source, text: str, html: Optional[str],
         if not fwd_from_text:
             user = await source.client.get_entity(PeerUser(fwd_from.from_id))
             if user:
-                fwd_from_text = pu.Puppet.get_displayname(user, format=False)
+                fwd_from_text = pu.Puppet.get_displayname(user, False)
                 fwd_from_html = f"<b>{fwd_from_text}</b>"
 
     if not fwd_from_text:
@@ -111,8 +115,9 @@ def highlight_edits(new_html: str, old_html: str) -> str:
     return new_html
 
 
-async def _add_reply_header(source: u.User, text: str, html: str, evt: Message, relates_to: dict,
-                            main_intent: IntentAPI, is_edit: bool) -> Tuple[str, str]:
+async def _add_reply_header(source: "AbstractUser", text: str, html: str, evt: Message,
+                            relates_to: dict, main_intent: IntentAPI, is_edit: bool
+                            ) -> Tuple[str, str]:
     space = (evt.to_id.channel_id
              if isinstance(evt, Message) and isinstance(evt.to_id, PeerChannel)
              else source.tgid)
@@ -143,7 +148,7 @@ async def _add_reply_header(source: u.User, text: str, html: str, evt: Message,
 
         if is_edit and should_highlight_edits:
             html = highlight_edits(html or escape(text), r_html_body)
-    except (ValueError, KeyError, MatrixRequestError) as e:
+    except (ValueError, KeyError, MatrixRequestError):
         r_sender_link = "unknown user"
         r_displayname = "unknown user"
         r_text_body = "Failed to fetch message"
@@ -155,8 +160,9 @@ async def _add_reply_header(source: u.User, text: str, html: str, evt: Message,
 
     r_keyword = "In reply to" if not is_edit else "Edit to"
     r_msg_link = f"<a href='https://matrix.to/#/{msg.mx_room}/{msg.mxid}'>{r_keyword}</a>"
-    html = (f"<mx-reply><blockquote>{r_msg_link} {r_sender_link}\n{r_html_body}</blockquote></mx-reply>"
-            + (html or escape(text)))
+    html = (
+        f"<mx-reply><blockquote>{r_msg_link} {r_sender_link}\n{r_html_body}</blockquote></mx-reply>"
+        + (html or escape(text)))
 
     lines = r_text_body.strip().split("\n")
     text_with_quote = f"> <{r_displayname}> {lines.pop(0)}"
@@ -168,7 +174,8 @@ async def _add_reply_header(source: u.User, text: str, html: str, evt: Message,
     return text_with_quote, html
 
 
-async def telegram_to_matrix(evt: Message, source: u.User, main_intent: Optional[IntentAPI] = None,
+async def telegram_to_matrix(evt: Message, source: "AbstractUser",
+                             main_intent: Optional[IntentAPI] = None,
                              is_edit: bool = False, prefix_text: Optional[str] = None,
                              prefix_html: Optional[str] = None) -> Tuple[str, str, dict]:
     text = add_surrogates(evt.message)
@@ -321,6 +328,6 @@ def _parse_url(html: List[str], entity_text: str, url: str) -> bool:
     return False
 
 
-def init_tg(context: Context):
+def init_tg(context: "Context"):
     global should_highlight_edits
     should_highlight_edits = htmldiff and context.config["bridge.highlight_edits"]

mautrix_telegram/formatter/util.py

@@ -14,8 +14,8 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Optional, Pattern
 from html import escape
-from typing import Optional
 import struct
 import re
 
@@ -47,7 +47,7 @@ def trim_reply_fallback_text(text: str) -> str:
 
 html_reply_fallback_regex = re.compile("^<mx-reply>"
                                        r"[\s\S]+?"
-                                       "</mx-reply>")
+                                       "</mx-reply>")  # type: Pattern
 
 
 def trim_reply_fallback_html(html: str) -> str:

mautrix_telegram/matrix.py

@@ -14,156 +14,187 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import List, Dict, Tuple, Set, Match
 import logging
+import asyncio
+import re
 
-from mautrix_appservice import MatrixRequestError
+from mautrix_appservice import MatrixRequestError, IntentError
 
-from .user import User
-from .portal import Portal
-from .puppet import Puppet
-from .commands import CommandHandler
+from . import user as u, portal as po, puppet as pu, commands as com
 
 
 class MatrixHandler:
-    log = logging.getLogger("mau.mx")
+    log = logging.getLogger("mau.mx")  # type: logging.Logger
 
     def __init__(self, context):
         self.az, self.db, self.config, _, self.tgbot = context
-        self.commands = CommandHandler(context)
+        self.commands = com.CommandProcessor(context)  # type: com.CommandProcessor
+        self.previously_typing = []  # type: List[str]
 
         self.az.matrix_event_handler(self.handle_event)
 
     async def init_as_bot(self):
-        await self.az.intent.set_display_name(
-            self.config.get("appservice.bot_displayname", "Telegram bridge bot"))
+        displayname = self.config["appservice.bot_displayname"]
+        if displayname:
+            try:
+                await self.az.intent.set_display_name(
+                    displayname if displayname != "remove" else "")
+            except asyncio.TimeoutError:
+                self.log.exception("TimeoutError when trying to set displayname")
 
-    async def handle_puppet_invite(self, room, puppet, inviter):
-        self.log.debug(f"{inviter} invited puppet for {puppet.tgid} to {room}")
-        if not inviter.logged_in:
-            await puppet.intent.error_and_leave(
-                room, text="Please log in before inviting Telegram puppets.")
+        avatar = self.config["appservice.bot_avatar"]
+        if avatar:
+            try:
+                await self.az.intent.set_avatar(avatar if avatar != "remove" else "")
+            except asyncio.TimeoutError:
+                self.log.exception("TimeoutError when trying to set avatar")
+
+    async def handle_puppet_invite(self, room_id, puppet: pu.Puppet, inviter: u.User):
+        intent = puppet.default_mxid_intent
+        self.log.debug(f"{inviter} invited puppet for {puppet.tgid} to {room_id}")
+        if not await inviter.is_logged_in():
+            await intent.error_and_leave(
+                room_id, text="Please log in before inviting Telegram puppets.")
             return
-        portal = Portal.get_by_mxid(room)
+        portal = po.Portal.get_by_mxid(room_id)
         if portal:
             if portal.peer_type == "user":
-                await puppet.intent.error_and_leave(
-                    room, text="You can not invite additional users to private chats.")
+                await intent.error_and_leave(
+                    room_id, text="You can not invite additional users to private chats.")
                 return
             await portal.invite_telegram(inviter, puppet)
-            await puppet.intent.join_room(room)
+            await intent.join_room(room_id)
             return
         try:
-            members = await self.az.intent.get_room_members(room)
+            members = await self.az.intent.get_room_members(room_id)
         except MatrixRequestError:
             members = []
         if self.az.bot_mxid not in members:
             if len(members) > 1:
-                await puppet.intent.error_and_leave(room, text=None, html=(
+                await intent.error_and_leave(room_id, text=None, html=(
                     f"Please invite "
                     f"<a href='https://matrix.to/#/{self.az.bot_mxid}'>the bridge bot</a> "
                     f"first if you want to create a Telegram chat."))
                 return
 
-            await puppet.intent.join_room(room)
-            portal = Portal.get_by_tgid(puppet.tgid, inviter.tgid, "user")
+            await intent.join_room(room_id)
+            portal = po.Portal.get_by_tgid(puppet.tgid, inviter.tgid, "user")
             if portal.mxid:
                 try:
-                    await puppet.intent.invite(portal.mxid, inviter.mxid)
-                    await puppet.intent.send_notice(room, text=None, html=(
+                    await intent.invite(portal.mxid, inviter.mxid)
+                    await intent.send_notice(room_id, text=None, html=(
                         "You already have a private chat with me: "
                         f"<a href='https://matrix.to/#/{portal.mxid}'>"
                         "Link to room"
                         "</a>"))
-                    await puppet.intent.leave_room(room)
+                    await intent.leave_room(room_id)
                     return
                 except MatrixRequestError:
                     pass
-            portal.mxid = room
+            portal.mxid = room_id
             portal.save()
             inviter.register_portal(portal)
-            await puppet.intent.send_notice(room, "Portal to private chat created.")
+            await intent.send_notice(room_id, "po.Portal to private chat created.")
         else:
-            await puppet.intent.join_room(room)
-            await puppet.intent.send_notice(room, "This puppet will remain inactive until a "
-                                                  "Telegram chat is created for this room.")
+            await intent.join_room(room_id)
+            await intent.send_notice(room_id, "This puppet will remain inactive until a "
+                                              "Telegram chat is created for this room.")
 
-    async def handle_invite(self, room, user, inviter):
-        self.log.debug(f"{inviter} invited {user} to {room}")
-        inviter = await User.get_by_mxid(inviter).ensure_started()
-        if user == self.az.bot_mxid:
-            await self.az.intent.join_room(room)
-            if not inviter.whitelisted:
-                await self.az.intent.send_notice(
-                    room, text=None,
-                    html="You are not whitelisted to use this bridge.<br/><br/>"
-                         "If you are the owner of this bridge, see the "
-                         "<code>bridge.permissions</code> section in your config file.")
-                await self.az.intent.leave_room(room)
-            return
+    async def accept_bot_invite(self, room_id: str, inviter: u.User):
+        tries = 0
+        while tries < 5:
+            try:
+                await self.az.intent.join_room(room_id)
+                break
+            except (IntentError, MatrixRequestError):
+                tries += 1
+                wait_for_seconds = (tries + 1) * 10
+                if tries < 5:
+                    self.log.exception(f"Failed to join room {room_id} with bridge bot, "
+                                       f"retrying in {wait_for_seconds} seconds...")
+                    await asyncio.sleep(wait_for_seconds)
+                else:
+                    self.log.exception("Failed to join room {room}, giving up.")
+                    return
+
+        if not inviter.whitelisted:
+            await self.az.intent.send_notice(
+                room_id, text=None,
+                html="You are not whitelisted to use this bridge.<br/><br/>"
+                     "If you are the owner of this bridge, see the "
+                     "<code>bridge.permissions</code> section in your config file.")
+            await self.az.intent.leave_room(room_id)
+
+    async def handle_invite(self, room_id: str, user_id: str, inviter_mxid: str):
+        self.log.debug(f"{inviter_mxid} invited {user_id} to {room_id}")
+        inviter = await u.User.get_by_mxid(inviter_mxid).ensure_started()
+        if user_id == self.az.bot_mxid:
+            return await self.accept_bot_invite(room_id, inviter)
         elif not inviter.whitelisted:
             return
 
-        puppet = Puppet.get_by_mxid(user)
+        puppet = pu.Puppet.get_by_mxid(user_id)
         if puppet:
-            await self.handle_puppet_invite(room, puppet, inviter)
+            await self.handle_puppet_invite(room_id, puppet, inviter)
             return
 
-        user = User.get_by_mxid(user, create=False)
+        user = u.User.get_by_mxid(user_id, create=False)
         if not user:
             return
         await user.ensure_started()
-        portal = Portal.get_by_mxid(room)
-        if user and user.has_full_access and portal:
+        portal = po.Portal.get_by_mxid(room_id)
+        if user and await user.has_full_access(allow_bot=True) and portal:
             await portal.invite_telegram(inviter, user)
             return
 
         # The rest can probably be ignored
 
-    async def handle_join(self, room, user, event_id):
-        user = await User.get_by_mxid(user).ensure_started()
+    async def handle_join(self, room_id: str, user_id: str, event_id: str):
+        user = await u.User.get_by_mxid(user_id).ensure_started()
 
-        portal = Portal.get_by_mxid(room)
+        portal = po.Portal.get_by_mxid(room_id)
         if not portal:
             return
 
         if not user.relaybot_whitelisted:
-            await portal.main_intent.kick(room, user.mxid,
+            await portal.main_intent.kick(room_id, user.mxid,
                                           "You are not whitelisted on this Telegram bridge.")
             return
-        elif not user.logged_in and not portal.has_bot:
-            await portal.main_intent.kick(room, user.mxid,
+        elif not await user.is_logged_in() and not portal.has_bot:
+            await portal.main_intent.kick(room_id, user.mxid,
                                           "This chat does not have a bot relaying "
                                           "messages for unauthenticated users.")
             return
 
-        self.log.debug(f"{user} joined {room}")
-        if user.logged_in or portal.has_bot:
+        self.log.debug(f"{user} joined {room_id}")
+        if await user.is_logged_in() or portal.has_bot:
             await portal.join_matrix(user, event_id)
 
-    async def handle_part(self, room, user, sender, event_id):
-        self.log.debug(f"{user} left {room}")
+    async def handle_part(self, room_id: str, user_id, sender_mxid: str, event_id: str):
+        self.log.debug(f"{user_id} left {room_id}")
 
-        sender = User.get_by_mxid(sender, create=False)
+        sender = u.User.get_by_mxid(sender_mxid, create=False)
         if not sender:
             return
         await sender.ensure_started()
 
-        portal = Portal.get_by_mxid(room)
+        portal = po.Portal.get_by_mxid(room_id)
         if not portal:
             return
 
-        puppet = Puppet.get_by_mxid(user)
+        puppet = pu.Puppet.get_by_mxid(user_id)
         if sender and puppet:
-            await portal.leave_matrix(puppet, sender)
+            await portal.leave_matrix(puppet, sender, event_id)
 
-        user = User.get_by_mxid(user, create=False)
+        user = u.User.get_by_mxid(user_id, create=False)
         if not user:
             return
         await user.ensure_started()
-        if user.logged_in or portal.has_bot:
+        if await user.is_logged_in() or portal.has_bot:
             await portal.leave_matrix(user, sender, event_id)
 
-    def is_command(self, message):
+    def is_command(self, message: dict) -> Tuple[bool, str]:
         text = message.get("body", "")
         prefix = self.config["bridge.command_prefix"]
         is_command = text.startswith(prefix)
@@ -172,19 +203,20 @@ class MatrixHandler:
         return is_command, text
 
     async def handle_message(self, room, sender, message, event_id):
-        self.log.debug(f"{sender} sent {message} to ${room}")
-
         is_command, text = self.is_command(message)
-        sender = await User.get_by_mxid(sender).ensure_started()
+        sender = await u.User.get_by_mxid(sender).ensure_started()
         if not sender.relaybot_whitelisted:
+            self.log.debug(f"Ignoring message \"{message}\" from {sender} to {room}:"
+                           " u.User is not whitelisted.")
             return
+        self.log.debug(f"Received Matrix event \"{message}\" from {sender} in {room}")
 
-        portal = Portal.get_by_mxid(room)
-        if not is_command and portal and (sender.logged_in or portal.has_bot):
+        portal = po.Portal.get_by_mxid(room)
+        if not is_command and portal and (await sender.is_logged_in() or portal.has_bot):
             await portal.handle_matrix_message(sender, message, event_id)
             return
 
-        if not sender.whitelisted or message["msgtype"] != "m.text":
+        if not sender.whitelisted or message.get("msgtype", "m.unknown") != "m.text":
             return
 
         try:
@@ -204,40 +236,45 @@ class MatrixHandler:
             await self.commands.handle(room, sender, command, args, is_management,
                                        is_portal=portal is not None)
 
-    async def handle_redaction(self, room, sender, event_id):
-        sender = await User.get_by_mxid(sender).ensure_started()
+    @staticmethod
+    async def handle_redaction(room_id: str, sender_mxid: str, event_id: str):
+        sender = await u.User.get_by_mxid(sender_mxid).ensure_started()
         if not sender.relaybot_whitelisted:
             return
 
-        portal = Portal.get_by_mxid(room)
+        portal = po.Portal.get_by_mxid(room_id)
         if not portal:
             return
 
         await portal.handle_matrix_deletion(sender, event_id)
 
-    async def handle_power_levels(self, room, sender, new, old):
-        portal = Portal.get_by_mxid(room)
-        sender = await User.get_by_mxid(sender).ensure_started()
-        if sender.has_full_access and portal:
+    @staticmethod
+    async def handle_power_levels(room_id: str, sender_mxid: str, new: dict, old: dict):
+        portal = po.Portal.get_by_mxid(room_id)
+        sender = await u.User.get_by_mxid(sender_mxid).ensure_started()
+        if await sender.has_full_access(allow_bot=True) and portal:
             await portal.handle_matrix_power_levels(sender, new["users"], old["users"])
 
-    async def handle_room_meta(self, type, room, sender, content):
-        portal = Portal.get_by_mxid(room)
-        sender = await User.get_by_mxid(sender).ensure_started()
-        if sender.has_full_access and portal:
+    @staticmethod
+    async def handle_room_meta(evt_type: str, room_id: str, sender_mxid: str, content: dict):
+        portal = po.Portal.get_by_mxid(room_id)
+        sender = await u.User.get_by_mxid(sender_mxid).ensure_started()
+        if await sender.has_full_access(allow_bot=True) and portal:
             handler, content_key = {
                 "m.room.name": (portal.handle_matrix_title, "name"),
                 "m.room.topic": (portal.handle_matrix_about, "topic"),
                 "m.room.avatar": (portal.handle_matrix_avatar, "url"),
-            }[type]
+            }[evt_type]
             if content_key not in content:
                 return
             await handler(sender, content[content_key])
 
-    async def handle_room_pin(self, room, sender, new_events, old_events):
-        portal = Portal.get_by_mxid(room)
-        sender = await User.get_by_mxid(sender).ensure_started()
-        if sender.has_full_access and portal:
+    @staticmethod
+    async def handle_room_pin(room_id: str, sender_mxid: str, new_events: Set[str],
+                              old_events: Set[str]):
+        portal = po.Portal.get_by_mxid(room_id)
+        sender = await u.User.get_by_mxid(sender_mxid).ensure_started()
+        if await sender.has_full_access(allow_bot=True) and portal:
             events = new_events - old_events
             if len(events) > 0:
                 # New event pinned, set that as pinned in Telegram.
@@ -246,45 +283,123 @@ class MatrixHandler:
                 # All pinned events removed, remove pinned event in Telegram.
                 await portal.handle_matrix_pin(sender, None)
 
-    def filter_matrix_event(self, event):
-        return (event["sender"] == self.az.bot_mxid
-                or Puppet.get_id_from_mxid(event["sender"]) is not None)
+    @staticmethod
+    async def handle_name_change(room_id: str, user_id: str, displayname: str,
+                                 prev_displayname: str, event_id: str):
+        portal = po.Portal.get_by_mxid(room_id)
+        if not portal or not portal.has_bot:
+            return
 
-    async def handle_event(self, evt):
+        user = await u.User.get_by_mxid(user_id).ensure_started()
+        if await user.needs_relaybot(portal):
+            await portal.name_change_matrix(user, displayname, prev_displayname, event_id)
+
+    @staticmethod
+    def parse_read_receipts(content: dict) -> Dict[str, str]:
+        return {user_id: event_id
+                for event_id, receipts in content.items()
+                for user_id in receipts.get("m.read", {})}
+
+    @staticmethod
+    async def handle_read_receipts(room_id: str, receipts: Dict[str, str]):
+        portal = po.Portal.get_by_mxid(room_id)
+        if not portal:
+            return
+
+        for user_id, event_id in receipts.items():
+            user = await u.User.get_by_mxid(user_id).ensure_started()
+            if not await user.is_logged_in():
+                continue
+            await portal.mark_read(user, event_id)
+
+    @staticmethod
+    async def handle_presence(user_id: str, presence: str):
+        user = await u.User.get_by_mxid(user_id).ensure_started()
+        if not await user.is_logged_in():
+            return
+        await user.set_presence(presence == "online")
+
+    async def handle_typing(self, room_id: str, now_typing: List[str]):
+        portal = po.Portal.get_by_mxid(room_id)
+        if not portal:
+            return
+
+        for user_id in set(self.previously_typing + now_typing):
+            is_typing = user_id in now_typing
+            was_typing = user_id in self.previously_typing
+            if is_typing and was_typing:
+                continue
+
+            user = await u.User.get_by_mxid(user_id).ensure_started()
+            if not await user.is_logged_in():
+                continue
+
+            await portal.set_typing(user, is_typing)
+
+        self.previously_typing = now_typing
+
+    def filter_matrix_event(self, event: dict):
+        sender = event.get("sender", None)
+        if not sender:
+            return False
+        return (sender == self.az.bot_mxid
+                or pu.Puppet.get_id_from_mxid(sender) is not None)
+
+    async def try_handle_event(self, evt: dict):
+        try:
+            await self.handle_event(evt)
+        except Exception:
+            self.log.exception("Error handling manually received Matrix event")
+
+    async def handle_event(self, evt: dict):
         if self.filter_matrix_event(evt):
             return
         self.log.debug("Received event: %s", evt)
-        type = evt["type"]
-        content = evt.get("content", {})
-        if type == "m.room.member":
-            prev_content = evt.get("unsigned", {}).get("prev_content", {})
-            membership = content.get("membership", "")
-            prev_membership = prev_content.get("membership", "leave")
+        evt_type = evt.get("type", "m.unknown")  # type: str
+        room_id = evt.get("room_id", None)  # type: str
+        event_id = evt.get("event_id", None)  # type: str
+        sender = evt.get("sender", None)  # type: str
+        content = evt.get("content", {})  # type: dict
+        if evt_type == "m.room.member":
+            state_key = evt["state_key"]  # type: str
+            prev_content = evt.get("unsigned", {}).get("prev_content", {})  # type: dict
+            membership = content.get("membership", "")  # type: str
+            prev_membership = prev_content.get("membership", "leave")  # type: str
             if membership == prev_membership:
-                # TODO handle displayname/avatar changes
-                pass
+                match = re.compile("@(.+):(.+)").match(state_key)  # type: Match
+                localpart = match.group(1)  # type: str
+                displayname = content.get("displayname", localpart)  # type: str
+                prev_displayname = prev_content.get("displayname", localpart)  # type: str
+                if displayname != prev_displayname:
+                    await self.handle_name_change(room_id, state_key, displayname,
+                                                  prev_displayname, event_id)
             elif membership == "invite":
-                await self.handle_invite(evt["room_id"], evt["state_key"], evt["sender"])
+                await self.handle_invite(room_id, state_key, sender)
             elif prev_membership == "join" and membership == "leave":
-                await self.handle_part(evt["room_id"], evt["state_key"], evt["sender"],
-                                       evt["event_id"])
+                await self.handle_part(room_id, state_key, sender, event_id)
             elif membership == "join":
-                await self.handle_join(evt["room_id"], evt["state_key"], evt["event_id"])
-        elif type in ("m.room.message", "m.sticker"):
-            if type != "m.room.message":
-                content["msgtype"] = type
-            await self.handle_message(evt["room_id"], evt["sender"], content, evt["event_id"])
-        elif type == "m.room.redaction":
-            await self.handle_redaction(evt["room_id"], evt["sender"], evt["redacts"])
-        elif type == "m.room.power_levels":
-            await self.handle_power_levels(evt["room_id"], evt["sender"], evt["content"],
-                                           evt["prev_content"])
-        elif type in ("m.room.name", "m.room.avatar", "m.room.topic"):
-            await self.handle_room_meta(type, evt["room_id"], evt["sender"], evt["content"])
-        elif type == "m.room.pinned_events":
+                await self.handle_join(room_id, state_key, event_id)
+        elif evt_type in ("m.room.message", "m.sticker"):
+            if evt_type != "m.room.message":
+                content["msgtype"] = evt_type
+            await self.handle_message(room_id, sender, content, event_id)
+        elif evt_type == "m.room.redaction":
+            await self.handle_redaction(room_id, sender, evt["redacts"])
+        elif evt_type == "m.room.power_levels":
+            prev_content = evt.get("unsigned", {}).get("prev_content", {})  # type: dict
+            await self.handle_power_levels(room_id, sender, evt["content"], prev_content)
+        elif evt_type in ("m.room.name", "m.room.avatar", "m.room.topic"):
+            await self.handle_room_meta(evt_type, room_id, sender, evt["content"])
+        elif evt_type == "m.room.pinned_events":
             new_events = set(evt["content"]["pinned"])
             try:
                 old_events = set(evt["unsigned"]["prev_content"]["pinned"])
             except KeyError:
                 old_events = set()
-            await self.handle_room_pin(evt["room_id"], evt["sender"], new_events, old_events)
+            await self.handle_room_pin(room_id, sender, new_events, old_events)
+        elif evt_type == "m.receipt":
+            await self.handle_read_receipts(room_id, self.parse_read_receipts(content))
+        elif evt_type == "m.presence":
+            await self.handle_presence(sender, content.get("presence", "offline"))
+        elif evt_type == "m.typing":
+            await self.handle_typing(room_id, content.get("user_ids", []))

mautrix_telegram/public/__init__.py

@@ -1,166 +0,0 @@
-# -*- coding: future_fstrings -*-
-# mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from aiohttp import web
-from mako.template import Template
-import asyncio
-import pkg_resources
-import logging
-
-from telethon.errors import *
-
-from ..user import User
-from ..commands.auth import enter_password
-from ..util import format_duration
-
-
-class PublicBridgeWebsite:
-    log = logging.getLogger("mau.public")
-
-    def __init__(self, loop):
-        self.loop = loop
-
-        self.login = Template(
-            pkg_resources.resource_string("mautrix_telegram", "public/login.html.mako"))
-
-        self.app = web.Application(loop=loop)
-        self.app.router.add_route("GET", "/login", self.get_login)
-        self.app.router.add_route("POST", "/login", self.post_login)
-        self.app.router.add_static("/",
-                                   pkg_resources.resource_filename("mautrix_telegram", "public/"))
-
-    async def get_login(self, request):
-        user = (User.get_by_mxid(request.rel_url.query["mxid"], create=False)
-                if "mxid" in request.rel_url.query else None)
-        if not user:
-            return self.render_login(
-                mxid=request.rel_url.query["mxid"] if "mxid" in request.rel_url.query else None,
-                state="request")
-        elif not user.whitelisted:
-            return self.render_login(mxid=user.mxid, error="You are not whitelisted.", status=403)
-        await user.ensure_started()
-        if not user.logged_in:
-            return self.render_login(mxid=user.mxid, state="request")
-
-        return self.render_login(mxid=user.mxid, username=user.username)
-
-    def render_login(self, status=200, username="", state="", error="", message="", mxid=""):
-        return web.Response(status=status, content_type="text/html",
-                            text=self.login.render(username=username, state=state, error=error,
-                                                   message=message, mxid=mxid))
-
-    async def post_login_phone(self, user, phone):
-        try:
-            await user.client.sign_in(phone or "+123")
-            return self.render_login(mxid=user.mxid, state="code", status=200,
-                                     message="Code requested successfully.")
-        except PhoneNumberInvalidError:
-            return self.render_login(mxid=user.mxid, state="request", status=400,
-                                     error="Invalid phone number.")
-        except PhoneNumberUnoccupiedError:
-            return self.render_login(mxid=user.mxid, state="request", status=404,
-                                     error="That phone number has not been registered.")
-        except PhoneNumberFloodError:
-            return self.render_login(
-                mxid=user.mxid, state="request", status=429,
-                error="Your phone number has been temporarily blocked for flooding. "
-                      "The ban is usually applied for around a day.")
-        except FloodWaitError as e:
-            return self.render_login(
-                mxid=user.mxid, state="request", status=429,
-                error="Your phone number has been temporarily blocked for flooding. "
-                      f"Please wait for {format_duration(e.seconds)} before trying again.")
-        except PhoneNumberBannedError:
-            return self.render_login(mxid=user.mxid, state="request", status=401,
-                                     error="Your phone number is banned from Telegram.")
-        except PhoneNumberAppSignupForbiddenError:
-            return self.render_login(mxid=user.mxid, state="request", status=401,
-                                     error="You have disabled 3rd party apps on your account.")
-        except Exception:
-            self.log.exception("Error requesting phone code")
-            return self.render_login(mxid=user.mxid, state="request", status=500,
-                                     error="Internal server error while requesting code.")
-
-    async def post_login_code(self, user, code, password_in_data):
-        try:
-            user_info = await user.client.sign_in(code=code)
-            asyncio.ensure_future(user.post_login(user_info), loop=self.loop)
-            if user.command_status and user.command_status["action"] == "Login":
-                user.command_status = None
-            return self.render_login(mxid=user.mxid, state="logged-in", status=200,
-                                     username=user_info.username)
-        except PhoneCodeInvalidError:
-            return self.render_login(mxid=user.mxid, state="code", status=403,
-                                     error="Incorrect phone code.")
-        except PhoneCodeExpiredError:
-            return self.render_login(mxid=user.mxid, state="code", status=403,
-                                     error="Phone code expired.")
-        except SessionPasswordNeededError:
-            if not password_in_data:
-                if user.command_status and user.command_status["action"] == "Login":
-                    user.command_status = {
-                        "next": enter_password,
-                        "action": "Login (password entry)",
-                    }
-                return self.render_login(
-                    mxid=user.mxid, state="password", status=200,
-                    message="Code accepted, but you have 2-factor authentication is enabled.")
-            return None
-        except Exception:
-            self.log.exception("Error sending phone code")
-            return self.render_login(mxid=user.mxid, state="code", status=500,
-                                     error="Internal server error while sending code.")
-
-    async def post_login_password(self, user, password):
-        try:
-            user_info = await user.client.sign_in(password=password)
-            asyncio.ensure_future(user.post_login(user_info), loop=self.loop)
-            if user.command_status and user.command_status["action"] == "Login (password entry)":
-                user.command_status = None
-            return self.render_login(mxid=user.mxid, state="logged-in", status=200,
-                                     username=user_info.username)
-        except (PasswordHashInvalidError, PasswordEmptyError):
-            return self.render_login(mxid=user.mxid, state="password", status=400,
-                                     error="Incorrect password.")
-        except Exception:
-            self.log.exception("Error sending password")
-            return self.render_login(mxid=user.mxid, state="password", status=500,
-                                     error="Internal server error while sending password.")
-
-    async def post_login(self, request):
-        data = await request.post()
-        if "mxid" not in data:
-            return self.render_login(error="Please enter your Matrix ID.", status=400)
-
-        user = await User.get_by_mxid(data["mxid"]).ensure_started(even_if_no_session=True)
-        if not user.whitelisted:
-            return self.render_login(mxid=user.mxid, error="You are not whitelisted.", status=403)
-        elif user.logged_in:
-            return self.render_login(mxid=user.mxid, username=user.username)
-
-        if "phone" in data:
-            return await self.post_login_phone(user, data["phone"])
-        elif "code" in data:
-            resp = await self.post_login_code(user, data["code"],
-                                              password_in_data="password" in data)
-            if resp or "password" not in data:
-                return resp
-        elif "password" not in data:
-            return self.render_login(error="No data given.", status=400)
-
-        if "password" in data:
-            return await self.post_login_password(user, data["password"])
-        return self.render_login(error="This should never happen.", status=500)

mautrix_telegram/puppet.py

@@ -14,49 +14,232 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Optional, Awaitable, Pattern, Dict, List, TYPE_CHECKING
 from difflib import SequenceMatcher
 import re
 import logging
+import asyncio
+
+from sqlalchemy import orm
 
 from telethon.tl.types import UserProfilePhoto
-from telethon.errors.rpc_error_list import LocationInvalidError
+from mautrix_appservice import AppService, IntentAPI, IntentError, MatrixRequestError
 
 from .db import Puppet as DBPuppet
 from . import util
 
-config = None
+if TYPE_CHECKING:
+    from .matrix import MatrixHandler
+    from .config import Config
+    from .context import Context
+
+config = None  # type: Config
 
 
 class Puppet:
-    log = logging.getLogger("mau.puppet")
-    db = None
-    az = None
-    mxid_regex = None
-    username_template = None
-    hs_domain = None
-    cache = {}
+    log = logging.getLogger("mau.puppet")  # type: logging.Logger
+    db = None  # type: orm.Session
+    az = None  # type: AppService
+    mx = None  # type: MatrixHandler
+    loop = None  # type: asyncio.AbstractEventLoop
+    mxid_regex = None  # type: Pattern
+    username_template = None  # type: str
+    hs_domain = None  # type: str
+    cache = {}  # type: Dict[str, Puppet]
+    by_custom_mxid = {}  # type: Dict[str, Puppet]
 
-    def __init__(self, id=None, username=None, displayname=None, displayname_source=None,
-                 photo_id=None, is_bot=None, db_instance=None):
+    def __init__(self, id=None, access_token=None, custom_mxid=None, username=None,
+                 displayname=None, displayname_source=None, photo_id=None, is_bot=None,
+                 is_registered=False, db_instance=None):
         self.id = id
-        self.mxid = self.get_mxid_from_id(self.id)
+        self.access_token = access_token
+        self.custom_mxid = custom_mxid
+        self.is_real_user = self.custom_mxid and self.access_token
+        self.default_mxid = self.get_mxid_from_id(self.id)
+        self.mxid = self.custom_mxid or self.default_mxid
 
         self.username = username
         self.displayname = displayname
         self.displayname_source = displayname_source
         self.photo_id = photo_id
         self.is_bot = is_bot
+        self.is_registered = is_registered
         self._db_instance = db_instance
 
-        self.intent = self.az.intent.user(self.mxid)
-        self.logged_in = True
+        self.default_mxid_intent = self.az.intent.user(self.default_mxid)
+        self.intent = None  # type: IntentAPI
+        self.refresh_intents()
 
         self.cache[id] = self
+        if self.custom_mxid:
+            self.by_custom_mxid[self.custom_mxid] = self
 
     @property
     def tgid(self):
         return self.id
 
+    @staticmethod
+    async def is_logged_in():
+        return True
+
+    # region Custom puppet management
+    def refresh_intents(self):
+        self.is_real_user = self.custom_mxid and self.access_token
+        self.intent = (self.az.intent.user(self.custom_mxid, self.access_token)
+                       if self.is_real_user else self.default_mxid_intent)
+
+    async def switch_mxid(self, access_token, mxid):
+        prev_mxid = self.custom_mxid
+        self.custom_mxid = mxid
+        self.access_token = access_token
+        self.refresh_intents()
+
+        err = await self.init_custom_mxid()
+        if err != 0:
+            return err
+
+        try:
+            del self.by_custom_mxid[prev_mxid]
+        except KeyError:
+            pass
+        self.mxid = self.custom_mxid or self.default_mxid
+        if self.mxid != self.default_mxid:
+            self.by_custom_mxid[self.mxid] = self
+            await self.leave_rooms_with_default_user()
+        self.save()
+        return 0
+
+    async def init_custom_mxid(self):
+        if not self.is_real_user:
+            return 0
+
+        mxid = await self.intent.whoami()
+        if not mxid or mxid != self.custom_mxid:
+            self.custom_mxid = None
+            self.access_token = None
+            self.refresh_intents()
+            if mxid != self.custom_mxid:
+                return 2
+            return 1
+        if config["bridge.sync_with_custom_puppets"]:
+            asyncio.ensure_future(self.sync(), loop=self.loop)
+        return 0
+
+    async def leave_rooms_with_default_user(self):
+        for room_id in await self.default_mxid_intent.get_joined_rooms():
+            try:
+                await self.default_mxid_intent.leave_room(room_id)
+                await self.intent.ensure_joined(room_id)
+            except (IntentError, MatrixRequestError):
+                pass
+
+    def create_sync_filter(self) -> Awaitable[str]:
+        return self.intent.client.create_filter(self.custom_mxid, {
+            "room": {
+                "include_leave": False,
+                "state": {
+                    "types": []
+                },
+                "timeline": {
+                    "types": [],
+                },
+                "ephemeral": {
+                    "types": ["m.typing", "m.receipt"],
+                },
+                "account_data": {
+                    "types": []
+                }
+            },
+            "account_data": {
+                "types": [],
+            },
+            "presence": {
+                "types": ["m.presence"],
+                "senders": [self.custom_mxid],
+            },
+        })
+
+    def filter_events(self, events):
+        new_events = []
+        for event in events:
+            evt_type = event.get("type", None)
+            event.setdefault("content", {})
+            if evt_type == "m.typing":
+                is_typing = self.custom_mxid in event["content"].get("user_ids", [])
+                event["content"]["user_ids"] = [self.custom_mxid] if is_typing else []
+            elif evt_type == "m.receipt":
+                val = None
+                evt = None
+                for event_id in event["content"]:
+                    try:
+                        val = event["content"][event_id]["m.read"][self.custom_mxid]
+                        evt = event_id
+                        break
+                    except KeyError:
+                        pass
+                if val and evt:
+                    event["content"] = {evt: {"m.read": {
+                        self.custom_mxid: val
+                    }}}
+                else:
+                    continue
+            new_events.append(event)
+        return new_events
+
+    def handle_sync(self, presence, ephemeral):
+        presence = [self.mx.try_handle_event(event) for event in presence]
+
+        for room_id, events in ephemeral.items():
+            for event in events:
+                event["room_id"] = room_id
+
+        ephemeral = [self.mx.try_handle_event(event)
+                     for events in ephemeral.values()
+                     for event in self.filter_events(events)]
+
+        events = ephemeral + presence
+        coro = asyncio.gather(*events, loop=self.loop)
+        asyncio.ensure_future(coro, loop=self.loop)
+
+    async def sync(self):
+        try:
+            await self._sync()
+        except Exception:
+            self.log.exception("Fatal error syncing")
+
+    async def _sync(self):
+        if not self.is_real_user:
+            self.log.warning("Called sync() for non-custom puppet.")
+            return
+        custom_mxid = self.custom_mxid
+        access_token_at_start = self.access_token
+        errors = 0
+        next_batch = None
+        filter_id = await self.create_sync_filter()
+        self.log.debug(f"Starting syncer for {custom_mxid} with sync filter {filter_id}.")
+        while access_token_at_start == self.access_token:
+            try:
+                sync_resp = await self.intent.client.sync(filter=filter_id, since=next_batch,
+                                                          set_presence="offline")
+                errors = 0
+                if next_batch is not None:
+                    presence = sync_resp.get("presence", {}).get("events", [])
+                    ephemeral = {room: data.get("ephemeral", {}).get("events", [])
+                                 for room, data
+                                 in sync_resp.get("rooms", {}).get("join", {}).items()}
+                    self.handle_sync(presence, ephemeral)
+                next_batch = sync_resp.get("next_batch", None)
+            except MatrixRequestError as e:
+                wait = min(errors, 11) ** 2
+                self.log.warning(f"Syncer for {custom_mxid} errored: {e}. "
+                                 f"Waiting for {wait} seconds...")
+                errors += 1
+                await asyncio.sleep(wait)
+        self.log.debug(f"Syncer for custom puppet {custom_mxid} stopped.")
+
+    # endregion
+    # region DB conversion
+
     @property
     def db_instance(self):
         if not self._db_instance:
@@ -64,24 +247,31 @@ class Puppet:
         return self._db_instance
 
     def new_db_instance(self):
-        return DBPuppet(id=self.id, username=self.username, displayname=self.displayname,
+        return DBPuppet(id=self.id, access_token=self.access_token, custom_mxid=self.custom_mxid,
+                        username=self.username, displayname=self.displayname,
                         displayname_source=self.displayname_source, photo_id=self.photo_id,
-                        is_bot=self.is_bot)
+                        is_bot=self.is_bot, matrix_registered=self.is_registered)
 
     @classmethod
     def from_db(cls, db_puppet):
-        return Puppet(db_puppet.id, db_puppet.username, db_puppet.displayname,
-                      db_puppet.displayname_source, db_puppet.photo_id, db_puppet.is_bot,
+        return Puppet(db_puppet.id, db_puppet.access_token, db_puppet.custom_mxid,
+                      db_puppet.username, db_puppet.displayname, db_puppet.displayname_source,
+                      db_puppet.photo_id, db_puppet.is_bot, db_puppet.matrix_registered,
                       db_instance=db_puppet)
 
     def save(self):
+        self.db_instance.access_token = self.access_token
+        self.db_instance.custom_mxid = self.custom_mxid
         self.db_instance.username = self.username
         self.db_instance.displayname = self.displayname
         self.db_instance.displayname_source = self.displayname_source
         self.db_instance.photo_id = self.photo_id
         self.db_instance.is_bot = self.is_bot
+        self.db_instance.matrix_registered = self.is_registered
         self.db.commit()
 
+    # endregion
+    # region Info updating
     def similarity(self, query):
         username_similarity = (SequenceMatcher(None, self.username, query).ratio()
                                if self.username else 0)
@@ -91,7 +281,7 @@ class Puppet:
         return round(similarity * 1000) / 10
 
     @staticmethod
-    def get_displayname(info, format=True):
+    def get_displayname(info, enable_format=True):
         data = {
             "phone number": info.phone if hasattr(info, "phone") else None,
             "username": info.username,
@@ -113,7 +303,7 @@ class Puppet:
         elif not name:
             name = info.id
 
-        if not format:
+        if not enable_format:
             return name
         return config.get("bridge.displayname_template", "{displayname} (Telegram)").format(
             displayname=name)
@@ -142,7 +332,7 @@ class Puppet:
 
         displayname = self.get_displayname(info)
         if displayname != self.displayname:
-            await self.intent.set_display_name(displayname)
+            await self.default_mxid_intent.set_display_name(displayname)
             self.displayname = displayname
             self.displayname_source = source.tgid
             return True
@@ -153,26 +343,30 @@ class Puppet:
     async def update_avatar(self, source, photo):
         photo_id = f"{photo.volume_id}-{photo.local_id}"
         if self.photo_id != photo_id:
-            file = await util.transfer_file_to_matrix(self.db, source.client, self.intent, photo)
+            file = await util.transfer_file_to_matrix(self.db, source.client,
+                                                      self.default_mxid_intent, photo)
             if file:
-                await self.intent.set_avatar(file.mxc)
+                await self.default_mxid_intent.set_avatar(file.mxc)
                 self.photo_id = photo_id
                 return True
         return False
 
+    # endregion
+    # region Getters
+
     @classmethod
-    def get(cls, id, create=True):
+    def get(cls, tgid, create=True) -> "Optional[Puppet]":
         try:
-            return cls.cache[id]
+            return cls.cache[tgid]
         except KeyError:
             pass
 
-        puppet = DBPuppet.query.get(id)
+        puppet = DBPuppet.query.get(tgid)
         if puppet:
             return cls.from_db(puppet)
 
         if create:
-            puppet = cls(id)
+            puppet = cls(tgid)
             cls.db.add(puppet.db_instance)
             cls.db.commit()
             return puppet
@@ -180,10 +374,34 @@ class Puppet:
         return None
 
     @classmethod
-    def get_by_mxid(cls, mxid, create=True):
+    def get_by_mxid(cls, mxid, create=True) -> "Optional[Puppet]":
         tgid = cls.get_id_from_mxid(mxid)
         return cls.get(tgid, create) if tgid else None
 
+    @classmethod
+    def get_by_custom_mxid(cls, mxid):
+        if not mxid:
+            raise ValueError("Matrix ID can't be empty")
+
+        try:
+            return cls.by_custom_mxid[mxid]
+        except KeyError:
+            pass
+
+        puppet = DBPuppet.query.filter(DBPuppet.custom_mxid == mxid).one_or_none()
+        if puppet:
+            puppet = cls.from_db(puppet)
+            return puppet
+
+        return None
+
+    @classmethod
+    def get_all_with_custom_mxid(cls):
+        return [cls.by_custom_mxid[puppet.mxid]
+                if puppet.custom_mxid in cls.by_custom_mxid
+                else cls.from_db(puppet)
+                for puppet in DBPuppet.query.filter(DBPuppet.custom_mxid is not None).all()]
+
     @classmethod
     def get_id_from_mxid(cls, mxid):
         match = cls.mxid_regex.match(mxid)
@@ -192,11 +410,11 @@ class Puppet:
         return None
 
     @classmethod
-    def get_mxid_from_id(cls, id):
-        return f"@{cls.username_template.format(userid=id)}:{cls.hs_domain}"
+    def get_mxid_from_id(cls, tgid):
+        return f"@{cls.username_template.format(userid=tgid)}:{cls.hs_domain}"
 
     @classmethod
-    def find_by_username(cls, username):
+    def find_by_username(cls, username) -> "Optional[Puppet]":
         if not username:
             return None
 
@@ -211,7 +429,7 @@ class Puppet:
         return None
 
     @classmethod
-    def find_by_displayname(cls, displayname):
+    def find_by_displayname(cls, displayname) -> "Optional[Puppet]":
         if not displayname:
             return None
 
@@ -224,12 +442,15 @@ class Puppet:
             return cls.from_db(puppet)
 
         return None
+    # endregion
 
 
-def init(context):
+def init(context: "Context") -> List[Awaitable[int]]:
     global config
-    Puppet.az, Puppet.db, config, _, _ = context
+    Puppet.az, Puppet.db, config, Puppet.loop, _ = context
+    Puppet.mx = context.mx
     Puppet.username_template = config.get("bridge.username_template", "telegram_{userid}")
     Puppet.hs_domain = config["homeserver"]["domain"]
-    localpart = Puppet.username_template.format(userid="(.+)")
-    Puppet.mxid_regex = re.compile(f"@{localpart}:{Puppet.hs_domain}")
+    Puppet.mxid_regex = re.compile(
+        f"@{Puppet.username_template.format(userid='(.+)')}:{Puppet.hs_domain}")
+    return [puppet.init_custom_mxid() for puppet in Puppet.get_all_with_custom_mxid()]

mautrix_telegram/scripts/dbms_migrate/__main__.py

@@ -0,0 +1,59 @@
+import argparse
+import sqlalchemy as sql
+from sqlalchemy import orm
+from sqlalchemy.ext.declarative import declarative_base
+
+from alchemysession import AlchemySessionContainer
+
+parser = argparse.ArgumentParser(description="mautrix-telegram dbms migration script",
+                                 prog="python -m mautrix_telegram.scripts.dbms_migrate")
+parser.add_argument("-f", "--from-url", type=str, required=True, metavar="<url>",
+                    help="the old database path")
+parser.add_argument("-t", "--to-url", type=str, required=True, metavar="<url>",
+                    help="the new database path")
+args = parser.parse_args()
+
+
+def connect(to):
+    import mautrix_telegram.base as base
+    base.Base = declarative_base()
+    from mautrix_telegram.db import (Portal, Message, UserPortal, User, RoomState, UserProfile,
+                                     Contact, Puppet, BotChat, TelegramFile)
+    db_engine = sql.create_engine(to)
+    db_factory = orm.sessionmaker(bind=db_engine)
+    db_session = orm.scoped_session(db_factory)  # type: orm.Session
+    base.Base.metadata.bind = db_engine
+    session_container = AlchemySessionContainer(engine=db_engine, session=db_session,
+                                                table_base=base.Base, table_prefix="telethon_",
+                                                manage_tables=False)
+
+    return db_session, {
+        "Version": session_container.Version,
+        "Session": session_container.Session,
+        "Entity": session_container.Entity,
+        "SentFile": session_container.SentFile,
+        "UpdateState": session_container.UpdateState,
+        "Portal": Portal,
+        "Message": Message,
+        "Puppet": Puppet,
+        "User": User,
+        "UserPortal": UserPortal,
+        "RoomState": RoomState,
+        "UserProfile": UserProfile,
+        "Contact": Contact,
+        "BotChat": BotChat,
+        "TelegramFile": TelegramFile,
+    }
+
+
+session, tables = connect(args.from_url)
+
+data = {}
+for name, table in tables.items():
+    data[name] = session.query(table).all()
+
+session, tables = connect(args.to_url)
+for name, table in tables.items():
+    for row in data[name]:
+        session.merge(row)
+session.commit()

mautrix_telegram/scripts/telematrix_import/__main__.py

@@ -0,0 +1,91 @@
+import argparse
+import sqlalchemy as sql
+from sqlalchemy import orm
+
+from mautrix_telegram.base import Base
+from mautrix_telegram.config import Config
+from mautrix_telegram.db import Portal, Message, Puppet, BotChat
+from .models import ChatLink, TgUser, MatrixUser, Message as TMMessage, Base as TelematrixBase
+
+parser = argparse.ArgumentParser(
+    description="mautrix-telegram telematrix import script",
+    prog="python -m mautrix_telegram.scripts.telematrix_import")
+parser.add_argument("-c", "--config", type=str, default="config.yaml",
+                    metavar="<path>", help="the path to your mautrix-telegram config file")
+parser.add_argument("-b", "--bot-id", type=int, required=True,
+                    metavar="<id>", help="the telegram user ID of your relay bot")
+parser.add_argument("-t", "--telematrix-database", type=str, default="sqlite:///database.db",
+                    metavar="<url>", help="your telematrix database URL")
+args = parser.parse_args()
+
+config = Config(args.config, None, None)
+config.load()
+
+mxtg_db_engine = sql.create_engine(config.get("appservice.database", "sqlite:///mautrix-telegram.db"))
+mxtg = orm.sessionmaker(bind=mxtg_db_engine)()
+Base.metadata.bind = mxtg_db_engine
+
+telematrix_db_engine = sql.create_engine(args.telematrix_database)
+telematrix = orm.sessionmaker(bind=telematrix_db_engine)()
+TelematrixBase.metadata.bind = telematrix_db_engine
+
+chat_links = telematrix.query(ChatLink).all()
+tg_users = telematrix.query(TgUser).all()
+mx_users = telematrix.query(MatrixUser).all()
+messages = telematrix.query(TMMessage).all()
+
+telematrix.close()
+telematrix_db_engine.dispose()
+
+portals = {}
+chats = {}
+messages = {}
+puppets = {}
+
+for chat_link in chat_links:
+    if type(chat_link.tg_room) is str:
+        print("Expected tg_room to be a number, got a string. Ignoring %s" % chat_link.tg_room)
+        continue
+    if chat_link.tg_room >= 0:
+        print("Unexpected unprefixed telegram chat ID: %s, ignoring..." % chat_link.tg_room)
+        continue
+    tgid = str(chat_link.tg_room)
+    if tgid.startswith("-100"):
+        tgid = int(tgid[4:])
+        peer_type = "channel"
+        megagroup = True
+    else:
+        tgid = -chat_link.tg_room
+        peer_type = "chat"
+        megagroup = False
+
+    portal = Portal(tgid=tgid, tg_receiver=tgid, peer_type=peer_type, megagroup=megagroup,
+                    mxid=chat_link.matrix_room)
+    bot_chat = BotChat(id=tgid, type=peer_type)
+    portals[chat_link.tg_room] = portal
+    chats[tgid] = bot_chat
+
+for tm_msg in messages:
+    try:
+        portal = portals[tm_msg.tg_group_id]
+    except KeyError:
+        print("Found message entry %d in unlinked chat %d, ignoring..." % (tm_msg.tg_message_id, tm_msg.tg_group_id))
+        continue
+    tg_space = portal.tgid if portal.peer_type == "channel" else args.bot_id
+    message = Message(mxid=tm_msg.matrix_event_id, mx_room=tm_msg.matrix_room_id,
+                      tgid=tm_msg.tg_message_id, tg_space=tg_space)
+    messages[tm_msg.matrix_event_id] = message
+
+for user in tg_users:
+    puppets[user.tg_id] = Puppet(id=user.tg_id, displayname=user.name, displayname_source=args.bot_id)
+
+for k, v in portals.items():
+    mxtg.add(v)
+for k, v in chats.items():
+    mxtg.add(v)
+for k, v in messages.items():
+    mxtg.add(v)
+for k, v in puppets.items():
+    mxtg.add(v)
+
+mxtg.commit()

mautrix_telegram/scripts/telematrix_import/models.py

@@ -0,0 +1,44 @@
+import sqlalchemy as sa
+from sqlalchemy.ext.declarative import declarative_base
+
+Base = declarative_base()
+
+
+class ChatLink(Base):
+    __tablename__ = 'chat_link'
+
+    id = sa.Column(sa.Integer, primary_key=True)
+    matrix_room = sa.Column(sa.String)
+    tg_room = sa.Column(sa.BigInteger)
+    active = sa.Column(sa.Boolean)
+
+
+class TgUser(Base):
+    __tablename__ = 'tg_user'
+
+    id = sa.Column(sa.Integer, primary_key=True)
+    tg_id = sa.Column(sa.BigInteger)
+    name = sa.Column(sa.String)
+    profile_pic_id = sa.Column(sa.String, nullable=True)
+
+
+class MatrixUser(Base):
+    __tablename__ = 'matrix_user'
+
+    id = sa.Column(sa.Integer, primary_key=True)
+    matrix_id = sa.Column(sa.String)
+    name = sa.Column(sa.String)
+
+
+class Message(Base):
+    """Describes a message in a room bridged between Telegram and Matrix"""
+    __tablename__ = "message"
+
+    id = sa.Column(sa.Integer, primary_key=True)
+    tg_group_id = sa.Column(sa.BigInteger)
+    tg_message_id = sa.Column(sa.BigInteger)
+
+    matrix_room_id = sa.Column(sa.String)
+    matrix_event_id = sa.Column(sa.String)
+
+    displayname = sa.Column(sa.String)

mautrix_telegram/sqlstatestore.py

@@ -0,0 +1,120 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2018 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Dict, Tuple
+
+from sqlalchemy import orm
+
+from mautrix_appservice import StateStore
+
+from . import puppet as pu
+from .db import RoomState, UserProfile
+
+
+class SQLStateStore(StateStore):
+    def __init__(self, db):
+        super().__init__()
+        self.db = db  # type: orm.Session
+        self.profile_cache = {}  # type: Dict[Tuple[str, str], UserProfile]
+        self.room_state_cache = {}  # type: Dict[str, RoomState]
+
+    @staticmethod
+    def is_registered(user: str) -> bool:
+        puppet = pu.Puppet.get_by_mxid(user)
+        return puppet.is_registered if puppet else False
+
+    @staticmethod
+    def registered(user: str):
+        puppet = pu.Puppet.get_by_mxid(user)
+        if puppet:
+            puppet.is_registered = True
+            puppet.save()
+
+    def update_state(self, event: dict):
+        event_type = event["type"]
+        if event_type == "m.room.power_levels":
+            self.set_power_levels(event["room_id"], event["content"])
+        elif event_type == "m.room.member":
+            self.set_member(event["room_id"], event["state_key"], event["content"])
+
+    def _get_user_profile(self, room_id: str, user_id: str, create: bool = True) -> UserProfile:
+        key = (room_id, user_id)
+        try:
+            return self.profile_cache[key]
+        except KeyError:
+            pass
+
+        profile = UserProfile.query.get(key)
+        if profile:
+            self.profile_cache[key] = profile
+        elif create:
+            profile = UserProfile(room_id=room_id, user_id=user_id)
+            self.db.add(profile)
+            self.db.commit()
+            self.profile_cache[key] = profile
+        return profile
+
+    def get_member(self, room: str, user: str) -> dict:
+        return self._get_user_profile(room, user).dict()
+
+    def set_member(self, room: str, user: str, member: dict):
+        profile = self._get_user_profile(room, user)
+        profile.membership = member.get("membership", profile.membership or "leave")
+        profile.displayname = member.get("displayname", profile.displayname)
+        profile.avatar_url = member.get("avatar_url", profile.avatar_url)
+        self.db.commit()
+
+    def set_membership(self, room: str, user: str, membership: str):
+        self.set_member(room, user, {
+            "membership": membership,
+        })
+
+    def _get_room_state(self, room_id: str, create: bool = True) -> RoomState:
+        try:
+            return self.room_state_cache[room_id]
+        except KeyError:
+            pass
+
+        room = RoomState.query.get(room_id)
+        if room:
+            self.room_state_cache[room_id] = room
+        elif create:
+            room = RoomState(room_id=room_id)
+            self.room_state_cache[room_id] = room
+        return room
+
+    def has_power_levels(self, room: str) -> bool:
+        return self._get_room_state(room).has_power_levels
+
+    def get_power_levels(self, room: str) -> dict:
+        return self._get_room_state(room).power_levels
+
+    def set_power_level(self, room: str, user: str, level: int):
+        room_state = self._get_room_state(room)
+        power_levels = room_state.power_levels
+        if not power_levels:
+            power_levels = {
+                "users": {},
+                "events": {},
+            }
+        power_levels[room]["users"][user] = level
+        room_state.power_levels = power_levels
+        self.db.commit()
+
+    def set_power_levels(self, room: str, content: dict):
+        state = self._get_room_state(room)
+        state.power_levels = content
+        self.db.commit()

mautrix_telegram/tgclient.py

@@ -14,44 +14,17 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from io import BytesIO
-
-from telethon import TelegramClient
-from telethon.tl.functions.messages import SendMessageRequest, SendMediaRequest
+from telethon import TelegramClient, utils
+from telethon.tl.functions.messages import SendMediaRequest
 from telethon.tl.types import *
-from telethon.extensions.markdown import parse as parse_md
+from telethon.tl import custom
 
 
 class MautrixTelegramClient(TelegramClient):
-    async def send_message(self, entity, message, reply_to=None, entities=None, markdown=False,
-                           link_preview=True):
-        entity = await self.get_input_entity(entity)
-
-        if markdown:
-            message, entities = parse_md(message)
-
-        request = SendMessageRequest(
-            peer=entity,
-            message=message,
-            entities=entities,
-            no_webpage=not link_preview,
-            reply_to_msg_id=self._get_message_id(reply_to)
-        )
-        result = await self(request)
-        if isinstance(result, UpdateShortSentMessage):
-            return Message(
-                id=result.id,
-                to_id=entity,
-                message=message,
-                date=result.date,
-                out=result.out,
-                media=result.media,
-                entities=result.entities
-            )
-
-        return self._get_response_message(request, result)
-
-    async def upload_file(self, file, mime_type=None, attributes=None, file_name=None):
+    async def upload_file_direct(self, file: bytes, mime_type: str = None,
+                                 attributes: List[TypeDocumentAttribute] = None,
+                                 file_name: str = None
+                                 ) -> Union[InputMediaUploadedDocument, InputMediaUploadedPhoto]:
         file_handle = await super().upload_file(file, file_name=file_name, use_cache=False)
 
         if mime_type == "image/png" or mime_type == "image/jpeg":
@@ -65,24 +38,12 @@ class MautrixTelegramClient(TelegramClient):
                 mime_type=mime_type or "application/octet-stream",
                 attributes=list(attr_dict.values()))
 
-    async def send_media(self, entity, media, caption=None, entities=None, reply_to=None):
+    async def send_media(self, entity: Union[TypeInputPeer, TypePeer],
+                         media: Union[TypeInputMedia, TypeMessageMedia],
+                         caption: str = None, entities: List[TypeMessageEntity] = None,
+                         reply_to: int = None) -> Optional[custom.Message]:
         entity = await self.get_input_entity(entity)
-        reply_to = self._get_message_id(reply_to)
+        reply_to = utils.get_message_id(reply_to)
         request = SendMediaRequest(entity, media, message=caption or "", entities=entities or [],
                                    reply_to_msg_id=reply_to)
-        return self._get_response_message(request, await self(request))
-
-    async def download_file_bytes(self, location):
-        if isinstance(location, Document):
-            location = InputDocumentFileLocation(location.id, location.access_hash,
-                                                 location.version)
-        elif not isinstance(location, (InputFileLocation, InputDocumentFileLocation)):
-            location = InputFileLocation(location.volume_id, location.local_id, location.secret)
-
-        file = BytesIO()
-
-        await self.download_file(location, file)
-
-        data = file.getvalue()
-        file.close()
-        return data
+        return self._get_response_message(request, await self(request), entity)

mautrix_telegram/user.py

@@ -14,103 +14,116 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Dict, Awaitable, Optional, Match, Tuple, TYPE_CHECKING
 import logging
 import asyncio
 import re
 
 from telethon.tl.types import *
+from telethon.tl.types import User as TLUser
 from telethon.tl.types.contacts import ContactsNotModified
 from telethon.tl.functions.contacts import GetContactsRequest, SearchRequest
+from telethon.tl.functions.account import UpdateStatusRequest
 from mautrix_appservice import MatrixRequestError
 
-from .db import User as DBUser, Contact as DBContact
+from .db import User as DBUser, Contact as DBContact, Portal as DBPortal
 from .abstract_user import AbstractUser
 from . import portal as po, puppet as pu
 
-config = None
+if TYPE_CHECKING:
+    from .config import Config
+    from .context import Context
+
+config = None  # type: Config
+
+SearchResults = List[Tuple["pu.Puppet", int]]
 
 
 class User(AbstractUser):
-    log = logging.getLogger("mau.user")
-    by_mxid = {}
-    by_tgid = {}
+    log = logging.getLogger("mau.user")  # type: logging.Logger
+    by_mxid = {}  # type: Dict[str, User]
+    by_tgid = {}  # type: Dict[int, User]
 
-    def __init__(self, mxid, tgid=None, username=None, db_contacts=None, saved_contacts=0,
-                 db_portals=None, db_instance=None):
+    def __init__(self, mxid: str, tgid: Optional[int] = None, username: Optional[str] = None,
+                 db_contacts: Optional[List[DBContact]] = None, saved_contacts: int = 0,
+                 is_bot: bool = False, db_portals: Optional[List[DBPortal]] = None,
+                 db_instance: Optional[DBUser] = None):
         super().__init__()
-        self.mxid = mxid
-        self.tgid = tgid
-        self.username = username
-        self.contacts = []
-        self.saved_contacts = saved_contacts
-        self.db_contacts = db_contacts
-        self.portals = {}
-        self.db_portals = db_portals
-        self._db_instance = db_instance
+        self.mxid = mxid  # type: str
+        self.tgid = tgid  # type: int
+        self.is_bot = is_bot  # type: bool
+        self.username = username  # type: str
+        self.contacts = []  # type: List[pu.Puppet]
+        self.saved_contacts = saved_contacts  # type: int
+        self.db_contacts = db_contacts  # type: List[DBContact]
+        self.portals = {}  # type: Dict[Tuple[int, int], po.Portal]
+        self.db_portals = db_portals  # type: List[DBPortal]
+        self._db_instance = db_instance  # type: DBUser
 
-        self.command_status = None
+        self.command_status = None  # type: dict
 
         (self.relaybot_whitelisted,
          self.whitelisted,
-         self.is_admin) = config.get_permissions(self.mxid)
+         self.puppet_whitelisted,
+         self.matrix_puppet_whitelisted,
+         self.is_admin,
+         self.permissions) = config.get_permissions(self.mxid)
 
         self.by_mxid[mxid] = self
         if tgid:
             self.by_tgid[tgid] = self
 
     @property
-    def name(self):
+    def name(self) -> str:
         return self.mxid
 
     @property
-    def displayname(self):
-        # TODO show better username
-        match = re.compile("@(.+):(.+)").match(self.mxid)
+    def mxid_localpart(self) -> str:
+        match = re.compile("@(.+):(.+)").match(self.mxid)  # type: Match
         return match.group(1)
 
+    # TODO replace with proper displayname getting everywhere
     @property
-    def db_contacts(self):
+    def displayname(self) -> str:
+        return self.mxid_localpart
+
+    @property
+    def db_contacts(self) -> List[DBContact]:
         return [self.db.merge(DBContact(user=self.tgid, contact=puppet.id))
                 for puppet in self.contacts]
 
     @db_contacts.setter
-    def db_contacts(self, contacts):
-        if contacts:
-            self.contacts = [pu.Puppet.get(entry.contact) for entry in contacts]
-        else:
-            self.contacts = []
+    def db_contacts(self, contacts: List[DBContact]):
+        self.contacts = [pu.Puppet.get(entry.contact) for entry in contacts] if contacts else []
 
     @property
-    def db_portals(self):
-        return [portal.db_instance for portal in self.portals.values()]
+    def db_portals(self) -> List[DBPortal]:
+        return [portal.db_instance for portal in self.portals.values() if not portal.deleted]
 
     @db_portals.setter
-    def db_portals(self, portals):
-        if portals:
-            self.portals = {(portal.tgid, portal.tg_receiver):
-                                po.Portal.get_by_tgid(portal.tgid, portal.tg_receiver)
-                            for portal in portals}
-        else:
-            self.portals = {}
+    def db_portals(self, portals: List[DBPortal]):
+        self.portals = {(portal.tgid, portal.tg_receiver):
+                            po.Portal.get_by_tgid(portal.tgid, portal.tg_receiver)
+                        for portal in portals} if portals else {}
 
     # region Database conversion
 
     @property
-    def db_instance(self):
+    def db_instance(self) -> DBUser:
         if not self._db_instance:
             self._db_instance = self.new_db_instance()
         return self._db_instance
 
-    def new_db_instance(self):
+    def new_db_instance(self) -> DBUser:
         return DBUser(mxid=self.mxid, tgid=self.tgid, tg_username=self.username,
-                      contacts=self.db_contacts, saved_contacts=self.saved_contacts,
+                      contacts=self.db_contacts, saved_contacts=self.saved_contacts or 0,
                       portals=self.db_portals)
 
     def save(self):
         self.db_instance.tgid = self.tgid
         self.db_instance.username = self.username
         self.db_instance.contacts = self.db_contacts
-        self.db_instance.saved_contacts = self.saved_contacts
+        self.db_instance.saved_contacts = self.saved_contacts or 0
         self.db_instance.portals = self.db_portals
         self.db.commit()
 
@@ -125,42 +138,72 @@ class User(AbstractUser):
             self.db.commit()
 
     @classmethod
-    def from_db(cls, db_user):
+    def from_db(cls, db_user: DBUser) -> "User":
         return User(db_user.mxid, db_user.tgid, db_user.tg_username, db_user.contacts,
-                    db_user.saved_contacts, db_user.portals, db_instance=db_user)
+                    False, db_user.saved_contacts, db_user.portals, db_instance=db_user)
 
     # endregion
     # region Telegram connection management
 
-    async def start(self, delete_unless_authenticated=False):
+    async def start(self, delete_unless_authenticated: bool = False) -> "User":
         await super().start()
-        if self.logged_in:
+        if await self.is_logged_in():
             self.log.debug(f"Ensuring post_login() for {self.name}")
             asyncio.ensure_future(self.post_login(), loop=self.loop)
         elif delete_unless_authenticated:
-            self.log.debug(f"Unauthenticated user {self.name} start()ed, deleting...")
-            # User not logged in -> forget user
-            self.client.disconnect()
-            # self.client.session.delete()
-            self.delete()
+            self.log.debug(f"Unauthenticated user {self.name} start()ed, deleting session...")
+            await self.client.disconnect()
+            self.client.session.delete()
         return self
 
-    async def post_login(self, info=None):
+    async def post_login(self, info: TLUser = None):
         try:
             await self.update_info(info)
-            await self.sync_dialogs()
-            await self.sync_contacts()
+            if not self.is_bot:
+                await self.sync_dialogs()
+                await self.sync_contacts()
             if config["bridge.catch_up"]:
                 await self.client.catch_up()
         except Exception:
-            self.log.exception("Failed to run post-login functions")
+            self.log.exception("Failed to run post-login functions for %s", self.mxid)
+
+    async def update(self, update: TypeUpdate):
+        if not self.is_bot:
+            return
+
+        if isinstance(update, (UpdateNewMessage, UpdateNewChannelMessage)):
+            message = update.message
+            if isinstance(message.to_id, PeerUser) and not message.out:
+                portal = po.Portal.get_by_tgid(message.from_id, peer_type="user",
+                                               tg_receiver=self.tgid)
+            else:
+                portal = po.Portal.get_by_entity(message.to_id, receiver_id=self.tgid)
+        elif isinstance(update, UpdateShortChatMessage):
+            portal = po.Portal.get_by_tgid(update.chat_id, peer_type="chat")
+        elif isinstance(update, UpdateShortMessage):
+            portal = po.Portal.get_by_tgid(update.user_id, self.tgid, "user")
+        else:
+            return
+
+        self.register_portal(portal)
 
     # endregion
     # region Telegram actions that need custom methods
 
-    async def update_info(self, info=None):
+    def ensure_started(self, even_if_no_session: bool = False) -> "Awaitable[User]":
+        return super().ensure_started(even_if_no_session)
+
+    def set_presence(self, online: bool = True):
+        if self.is_bot:
+            return
+        return self.client(UpdateStatusRequest(offline=not online))
+
+    async def update_info(self, info: TLUser = None):
         info = info or await self.client.get_me()
         changed = False
+        if self.is_bot != info.bot:
+            self.is_bot = info.bot
+            changed = True
         if self.username != info.username:
             self.username = info.username
             changed = True
@@ -171,8 +214,11 @@ class User(AbstractUser):
             self.save()
 
     async def log_out(self):
+        puppet = pu.Puppet.get(self.tgid)
+        if puppet.is_real_user:
+            await puppet.switch_mxid(None, None)
         for _, portal in self.portals.items():
-            if portal.has_bot:
+            if not portal.mxid or portal.has_bot:
                 continue
             try:
                 await portal.main_intent.kick(portal.mxid, self.mxid, "Logged out of Telegram.")
@@ -194,8 +240,9 @@ class User(AbstractUser):
         self.delete()
         return True
 
-    def _search_local(self, query, max_results=5, min_similarity=45):
-        results = []
+    def _search_local(self, query: str, max_results: int = 5, min_similarity: int = 45
+                      ) -> SearchResults:
+        results = []  # type: SearchResults
         for contact in self.contacts:
             similarity = contact.similarity(query)
             if similarity >= min_similarity:
@@ -203,11 +250,11 @@ class User(AbstractUser):
         results.sort(key=lambda tup: tup[1], reverse=True)
         return results[0:max_results]
 
-    async def _search_remote(self, query, max_results=5):
+    async def _search_remote(self, query: str, max_results: int = 5) -> SearchResults:
         if len(query) < 5:
             return []
         server_results = await self.client(SearchRequest(q=query, limit=max_results))
-        results = []
+        results = []  # type: SearchResults
         for user in server_results.users:
             puppet = pu.Puppet.get(user.id)
             await puppet.update_info(self, user)
@@ -215,7 +262,7 @@ class User(AbstractUser):
         results.sort(key=lambda tup: tup[1], reverse=True)
         return results[0:max_results]
 
-    async def search(self, query, force_remote=False):
+    async def search(self, query: str, force_remote: bool = False) -> Tuple[SearchResults, bool]:
         if force_remote:
             return await self._search_remote(query), True
 
@@ -225,9 +272,9 @@ class User(AbstractUser):
 
         return await self._search_remote(query), True
 
-    async def sync_dialogs(self, synchronous_create=False):
+    async def sync_dialogs(self, synchronous_create: bool = False):
         creators = []
-        for entity in await self._get_dialogs(limit=30):
+        for entity in await self.get_dialogs(limit=30):
             portal = po.Portal.get_by_entity(entity)
             self.portals[portal.tgid_full] = portal
             creators.append(
@@ -236,7 +283,7 @@ class User(AbstractUser):
         self.save()
         await asyncio.gather(*creators, loop=self.loop)
 
-    def register_portal(self, portal):
+    def register_portal(self, portal: po.Portal):
         try:
             if self.portals[portal.tgid_full] == portal:
                 return
@@ -245,14 +292,18 @@ class User(AbstractUser):
         self.portals[portal.tgid_full] = portal
         self.save()
 
-    def unregister_portal(self, portal):
+    def unregister_portal(self, portal: po.Portal):
         try:
             del self.portals[portal.tgid_full]
             self.save()
         except KeyError:
             pass
 
-    def _hash_contacts(self):
+    async def needs_relaybot(self, portal: po.Portal) -> bool:
+        return not await self.is_logged_in() or (
+            self.is_bot and portal.tgid_full not in self.portals)
+
+    def _hash_contacts(self) -> int:
         acc = 0
         for id in sorted([self.saved_contacts] + [contact.id for contact in self.contacts]):
             acc = (acc * 20261 + id) & 0xffffffff
@@ -275,7 +326,10 @@ class User(AbstractUser):
     # region Class instance lookup
 
     @classmethod
-    def get_by_mxid(cls, mxid, create=True):
+    def get_by_mxid(cls, mxid: str, create: bool=True) -> "Optional[User]":
+        if not mxid:
+            raise ValueError("Matrix ID can't be empty")
+
         try:
             return cls.by_mxid[mxid]
         except KeyError:
@@ -295,7 +349,7 @@ class User(AbstractUser):
         return None
 
     @classmethod
-    def get_by_tgid(cls, tgid):
+    def get_by_tgid(cls, tgid: int) -> "Optional[User]":
         try:
             return cls.by_tgid[tgid]
         except KeyError:
@@ -309,7 +363,7 @@ class User(AbstractUser):
         return None
 
     @classmethod
-    def find_by_username(cls, username):
+    def find_by_username(cls, username: str) -> "Optional[User]":
         if not username:
             return None
 
@@ -325,9 +379,9 @@ class User(AbstractUser):
     # endregion
 
 
-def init(context):
+def init(context: "Context") -> List[Awaitable[User]]:
     global config
     config = context.config
 
     users = [User.from_db(user) for user in DBUser.query.all()]
-    return [user.start(delete_unless_authenticated=True) for user in users]
+    return [user.ensure_started() for user in users]

mautrix_telegram/util/__init__.py

@@ -1,2 +1,3 @@
 from .file_transfer import transfer_file_to_matrix, convert_image
 from .format_duration import format_duration
+from .signed_token import sign_token, verify_token

mautrix_telegram/util/file_transfer.py

@@ -14,15 +14,25 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Optional, Tuple, Union, Dict
 from io import BytesIO
 import time
 import logging
 import asyncio
 
 import magic
+from sqlalchemy import orm
 from sqlalchemy.exc import IntegrityError, InvalidRequestError
 from sqlalchemy.orm.exc import FlushError
 
+from telethon.tl.types import (Document, FileLocation, InputFileLocation,
+                               InputDocumentFileLocation, PhotoSize, PhotoCachedSize)
+from telethon.errors import *
+from mautrix_appservice import IntentAPI
+
+from ..tgclient import MautrixTelegramClient
+from ..db import TelegramFile as DBTelegramFile
+
 try:
     from PIL import Image
 except ImportError:
@@ -36,20 +46,18 @@ try:
 except ImportError:
     VideoFileClip = random = string = os = mimetypes = None
 
-from telethon.tl.types import (Document, FileLocation, InputFileLocation,
-                               InputDocumentFileLocation, PhotoSize, PhotoCachedSize)
-from telethon.errors import LocationInvalidError
+log = logging.getLogger("mau.util")  # type: logging.Logger
 
-from ..db import TelegramFile as DBTelegramFile
-
-log = logging.getLogger("mau.util")
+TypeLocation = Union[Document, InputDocumentFileLocation, FileLocation, InputFileLocation]
 
 
-def convert_image(file, source_mime="image/webp", target_type="png", thumbnail_to=None):
+def convert_image(file: bytes, source_mime: str = "image/webp", target_type: str = "png",
+                  thumbnail_to: Optional[Tuple[int, int]] = None
+                  ) -> Tuple[str, bytes, Optional[int], Optional[int]]:
     if not Image:
         return source_mime, file, None, None
     try:
-        image = Image.open(BytesIO(file)).convert("RGBA")
+        image = Image.open(BytesIO(file)).convert("RGBA")  # type: Image.Image
         if thumbnail_to:
             image.thumbnail(thumbnail_to, Image.ANTIALIAS)
         new_file = BytesIO()
@@ -61,13 +69,14 @@ def convert_image(file, source_mime="image/webp", target_type="png", thumbnail_t
         return source_mime, file, None, None
 
 
-def _temp_file_name(ext):
+def _temp_file_name(ext: str) -> str:
     return ("/tmp/mxtg-video-"
             + "".join(random.choice(string.ascii_uppercase + string.digits) for _ in range(10))
             + ext)
 
 
-def _read_video_thumbnail(data, video_ext="mp4", frame_ext="png", max_size=(1024, 720)):
+def _read_video_thumbnail(data: bytes, video_ext: str = "mp4", frame_ext: str = "png",
+                          max_size: Tuple[int, int] = (1024, 720)) -> Tuple[bytes, int, int]:
     # We don't have any way to read the video from memory, so save it to disk.
     temp_file = _temp_file_name(video_ext)
     with open(temp_file, "wb") as file:
@@ -90,21 +99,21 @@ def _read_video_thumbnail(data, video_ext="mp4", frame_ext="png", max_size=(1024
     return thumbnail_file.getvalue(), w, h
 
 
-def _location_to_id(location):
+def _location_to_id(location: TypeLocation) -> str:
     if isinstance(location, (Document, InputDocumentFileLocation)):
         return f"{location.id}-{location.version}"
     elif isinstance(location, (FileLocation, InputFileLocation)):
         return f"{location.volume_id}-{location.local_id}"
-    else:
-        return None
 
 
-async def transfer_thumbnail_to_matrix(client, intent, thumbnail_loc, video, mime):
+async def transfer_thumbnail_to_matrix(client: MautrixTelegramClient, intent: IntentAPI,
+                                       thumbnail_loc: TypeLocation, video: bytes,
+                                       mime: str) -> Optional[DBTelegramFile]:
     if not Image or not VideoFileClip:
         return None
 
-    id = _location_to_id(thumbnail_loc)
-    if not id:
+    loc_id = _location_to_id(thumbnail_loc)
+    if not loc_id:
         return None
 
     video_ext = mimetypes.guess_extension(mime)
@@ -115,63 +124,72 @@ async def transfer_thumbnail_to_matrix(client, intent, thumbnail_loc, video, mim
             return None
         mime_type = "image/png"
     else:
-        file = await client.download_file_bytes(thumbnail_loc)
+        file = await client.download_file(thumbnail_loc)
         width, height = None, None
         mime_type = magic.from_buffer(file, mime=True)
 
-    uploaded = await intent.upload_file(file, mime_type)
+    content_uri = await intent.upload_file(file, mime_type)
 
-    return DBTelegramFile(id=id, mxc=uploaded["content_uri"], mime_type=mime_type,
+    return DBTelegramFile(id=loc_id, mxc=content_uri, mime_type=mime_type,
                           was_converted=False, timestamp=int(time.time()), size=len(file),
                           width=width, height=height)
 
 
-transfer_locks = {}
-transfer_locks_lock = asyncio.Lock()
+transfer_locks = {}  # type: Dict[str, asyncio.Lock]
 
 
-async def transfer_file_to_matrix(db, client, intent, location, thumbnail=None, is_sticker=False):
-    id = _location_to_id(location)
-    if not id:
+async def transfer_file_to_matrix(db: orm.Session, client: MautrixTelegramClient, intent: IntentAPI,
+                                  location: TypeLocation, thumbnail: Optional[TypeLocation] = None,
+                                  is_sticker: bool = False) -> Optional[DBTelegramFile]:
+    location_id = _location_to_id(location)
+    if not location_id:
         return None
 
-    db_file = DBTelegramFile.query.get(id)
-    if db_file:
-        return db_file
-
-    async with transfer_locks_lock:
-        try:
-            lock = transfer_locks[id]
-        except KeyError:
-            lock = asyncio.Lock()
-            transfer_locks[id] = lock
-    async with lock:
-        return await _unlocked_transfer_file_to_matrix(db, client, intent, id, location, thumbnail, is_sticker)
-
-
-async def _unlocked_transfer_file_to_matrix(db, client, intent, id, location, thumbnail, is_sticker):
-    db_file = DBTelegramFile.query.get(id)
+    db_file = DBTelegramFile.query.get(location_id)
     if db_file:
         return db_file
 
     try:
-        file = await client.download_file_bytes(location)
+        lock = transfer_locks[location_id]
+    except KeyError:
+        lock = asyncio.Lock()
+        transfer_locks[location_id] = lock
+    async with lock:
+        return await _unlocked_transfer_file_to_matrix(db, client, intent, location_id, location,
+                                                       thumbnail, is_sticker)
+
+
+async def _unlocked_transfer_file_to_matrix(db: orm.Session, client: MautrixTelegramClient,
+                                            intent: IntentAPI, loc_id: str, location: TypeLocation,
+                                            thumbnail: Optional[TypeLocation],
+                                            is_sticker: bool) -> Optional[DBTelegramFile]:
+    db_file = DBTelegramFile.query.get(loc_id)
+    if db_file:
+        return db_file
+
+    try:
+        file = await client.download_file(location)
     except LocationInvalidError:
         return None
+    except (AuthBytesInvalidError, AuthKeyInvalidError, SecurityError) as e:
+        log.exception(f"{e.__class__.__name__} while downloading a file.")
+        return None
+
     width, height = None, None
     mime_type = magic.from_buffer(file, mime=True)
 
     image_converted = False
     if mime_type == "image/webp":
-        new_mime_type, file, width, height = convert_image(file, source_mime="image/webp", target_type="png", thumbnail_to=(
-            256, 256) if is_sticker else None)
+        new_mime_type, file, width, height = convert_image(
+            file, source_mime="image/webp", target_type="png",
+            thumbnail_to=(256, 256) if is_sticker else None)
         image_converted = new_mime_type != mime_type
         mime_type = new_mime_type
         thumbnail = None
 
-    uploaded = await intent.upload_file(file, mime_type)
+    content_uri = await intent.upload_file(file, mime_type)
 
-    db_file = DBTelegramFile(id=id, mxc=uploaded["content_uri"],
+    db_file = DBTelegramFile(id=loc_id, mxc=content_uri,
                              mime_type=mime_type, was_converted=image_converted,
                              timestamp=int(time.time()), size=len(file),
                              width=width, height=height)

mautrix_telegram/util/format_duration.py

@@ -16,10 +16,12 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
 
-def format_duration(seconds):
-    def pluralize(count, singular): return singular if count == 1 else singular + "s"
+def format_duration(seconds: int) -> str:
+    def pluralize(count, singular):
+        return singular if count == 1 else singular + "s"
 
-    def include(count, word): return f"{count} {pluralize(count, word)}" if count > 0 else ""
+    def include(count, word):
+        return f"{count} {pluralize(count, word)}" if count > 0 else ""
 
     minutes, seconds = divmod(seconds, 60)
     hours, minutes = divmod(minutes, 60)

mautrix_telegram/util/signed_token.py

@@ -0,0 +1,53 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2018 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Optional
+import json
+import base64
+import hashlib
+
+
+def _get_checksum(key: str, payload: bytes) -> str:
+    hasher = hashlib.sha256()
+    hasher.update(payload)
+    hasher.update(key.encode("utf-8"))
+    checksum = hasher.hexdigest()
+    return checksum
+
+
+def sign_token(key: str, payload: dict) -> str:
+    payload = base64.urlsafe_b64encode(json.dumps(payload).encode("utf-8"))
+    checksum = _get_checksum(key, payload)
+    return f"{checksum}:{payload.decode('utf-8')}"
+
+
+def verify_token(key: str, data: str) -> Optional[dict]:
+    if not data:
+        return None
+
+    try:
+        checksum, payload = data.split(":", 1)
+    except ValueError:
+        return None
+
+    if checksum != _get_checksum(key, payload.encode("utf-8")):
+        return None
+
+    payload = base64.urlsafe_b64decode(payload).decode("utf-8")
+    try:
+        return json.loads(payload)
+    except json.JSONDecodeError:
+        return None

mautrix_telegram/web/__init__.py

@@ -0,0 +1,2 @@
+from .provisioning import ProvisioningAPI
+from .public import PublicBridgeWebsite

mautrix_telegram/web/common/__init__.py

@@ -0,0 +1 @@
+from .auth_api import AuthAPI

mautrix_telegram/web/common/auth_api.py

@@ -0,0 +1,178 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2018 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from abc import abstractmethod
+import abc
+import asyncio
+import logging
+
+from telethon.errors import *
+
+from ...commands.auth import enter_password
+from ...util import format_duration
+from ...puppet import Puppet
+from ...user import User
+
+
+class AuthAPI(abc.ABC):
+    log = logging.getLogger("mau.web.auth")
+
+    def __init__(self, loop):
+        self.loop = loop  # type: asyncio.AbstractEventLoop
+
+    @abstractmethod
+    def get_login_response(self, status=200, state="", username="", mxid="", message="", error="",
+                           errcode=""):
+        raise NotImplementedError()
+
+    @abstractmethod
+    def get_mx_login_response(self, status=200, state="", username="", mxid="", message="",
+                              error="", errcode=""):
+        raise NotImplementedError()
+
+    async def post_matrix_token(self, user: User, token):
+        puppet = Puppet.get(user.tgid)
+        if puppet.is_real_user:
+            return self.get_mx_login_response(state="already-logged-in", status=409,
+                                              error="You have already logged in with your Matrix "
+                                                    "account.", errcode="already-logged-in")
+
+        resp = await puppet.switch_mxid(token, user.mxid)
+        if resp == 2:
+            return self.get_mx_login_response(status=403, errcode="only-login-self",
+                                              error="You can only log in as your own Matrix user.")
+        elif resp == 1:
+            return self.get_mx_login_response(status=401, errcode="invalid-access-token",
+                                              error="Failed to verify access token.")
+
+        return self.get_mx_login_response(mxid=user.mxid, status=200, state="logged-in")
+
+    async def post_matrix_password(self, user, password):
+        return self.get_mx_login_response(mxid=user.mxid, status=501, error="Not yet implemented",
+                                          errcode="not-yet-implemented")
+
+    async def post_login_phone(self, user, phone):
+        try:
+            await user.client.sign_in(phone or "+123")
+            return self.get_login_response(mxid=user.mxid, state="code", status=200,
+                                           message="Code requested successfully.")
+        except PhoneNumberInvalidError:
+            return self.get_login_response(mxid=user.mxid, state="request", status=400,
+                                           errcode="phone_number_invalid",
+                                           error="Invalid phone number.")
+        except PhoneNumberBannedError:
+            return self.get_login_response(mxid=user.mxid, state="request", status=403,
+                                           errcode="phone_number_banned",
+                                           error="Your phone number is banned from Telegram.")
+        except PhoneNumberAppSignupForbiddenError:
+            return self.get_login_response(mxid=user.mxid, state="request", status=403,
+                                           errcode="phone_number_app_signup_forbidden",
+                                           error="You have disabled 3rd party apps on your account.")
+        except PhoneNumberUnoccupiedError:
+            return self.get_login_response(mxid=user.mxid, state="request", status=404,
+                                           errcode="phone_number_unoccupied",
+                                           error="That phone number has not been registered.")
+        except PhoneNumberFloodError:
+            return self.get_login_response(
+                mxid=user.mxid, state="request", status=429, errcode="phone_number_flood",
+                error="Your phone number has been temporarily blocked for flooding. "
+                      "The ban is usually applied for around a day.")
+        except FloodWaitError as e:
+            return self.get_login_response(
+                mxid=user.mxid, state="request", status=429, errcode="flood_wait",
+                error="Your phone number has been temporarily blocked for flooding. "
+                      f"Please wait for {format_duration(e.seconds)} before trying again.")
+        except Exception:
+            self.log.exception("Error requesting phone code")
+            return self.get_login_response(mxid=user.mxid, state="request", status=500,
+                                           errcode="unknown_error",
+                                           error="Internal server error while requesting code.")
+
+    async def post_login_token(self, user, token):
+        try:
+            user_info = await user.client.sign_in(bot_token=token)
+            asyncio.ensure_future(user.post_login(user_info), loop=self.loop)
+            if user.command_status and user.command_status["action"] == "Login":
+                user.command_status = None
+            return self.get_login_response(mxid=user.mxid, state="logged-in", status=200,
+                                           username=user_info.username)
+        except AccessTokenInvalidError:
+            return self.get_login_response(mxid=user.mxid, state="token", status=401,
+                                           errcode="bot_token_invalid",
+                                           error="Bot token invalid.")
+        except AccessTokenExpiredError:
+            return self.get_login_response(mxid=user.mxid, state="token", status=403,
+                                           errcode="bot_token_expired",
+                                           error="Bot token expired.")
+        except Exception:
+            self.log.exception("Error sending bot token")
+            return self.get_login_response(mxid=user.mxid, state="token", status=500,
+                                           error="Internal server error while sending token.")
+
+    async def post_login_code(self, user, code, password_in_data):
+        try:
+            user_info = await user.client.sign_in(code=code)
+            asyncio.ensure_future(user.post_login(user_info), loop=self.loop)
+            if user.command_status and user.command_status["action"] == "Login":
+                user.command_status = None
+            return self.get_login_response(mxid=user.mxid, state="logged-in", status=200,
+                                           username=user_info.username)
+        except PhoneCodeInvalidError:
+            return self.get_login_response(mxid=user.mxid, state="code", status=401,
+                                           errcode="phone_code_invalid",
+                                           error="Incorrect phone code.")
+        except PhoneCodeExpiredError:
+            return self.get_login_response(mxid=user.mxid, state="code", status=403,
+                                           errcode="phone_code_expired",
+                                           error="Phone code expired.")
+        except SessionPasswordNeededError:
+            if not password_in_data:
+                if user.command_status and user.command_status["action"] == "Login":
+                    user.command_status = {
+                        "next": enter_password,
+                        "action": "Login (password entry)",
+                    }
+                return self.get_login_response(
+                    mxid=user.mxid, state="password", status=202,
+                    message="Code accepted, but you have 2-factor authentication is enabled.")
+            return None
+        except Exception:
+            self.log.exception("Error sending phone code")
+            return self.get_login_response(mxid=user.mxid, state="code", status=500,
+                                           errcode="unknown_error",
+                                           error="Internal server error while sending code.")
+
+    async def post_login_password(self, user, password):
+        try:
+            user_info = await user.client.sign_in(password=password)
+            asyncio.ensure_future(user.post_login(user_info), loop=self.loop)
+            if user.command_status and user.command_status["action"] == "Login (password entry)":
+                user.command_status = None
+            return self.get_login_response(mxid=user.mxid, state="logged-in", status=200,
+                                           username=user_info.username)
+        except PasswordEmptyError:
+            return self.get_login_response(mxid=user.mxid, state="password", status=400,
+                                           errcode="password_empty",
+                                           error="Empty password.")
+        except PasswordHashInvalidError:
+            return self.get_login_response(mxid=user.mxid, state="password", status=401,
+                                           errcode="password_invalid",
+                                           error="Incorrect password.")
+        except Exception:
+            self.log.exception("Error sending password")
+            return self.get_login_response(mxid=user.mxid, state="password", status=500,
+                                           errcode="unknown_error",
+                                           error="Internal server error while sending password.")

mautrix_telegram/web/provisioning/__init__.py

@@ -0,0 +1,459 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2018 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from aiohttp import web
+from typing import Tuple, Optional, Callable, Awaitable, TYPE_CHECKING
+import asyncio
+import logging
+import json
+
+from telethon.utils import get_peer_id, resolve_id
+from telethon.tl.types import ChatForbidden, ChannelForbidden, TypeChat
+from mautrix_appservice import AppService, MatrixRequestError, IntentError
+
+from ...user import User
+from ...portal import Portal
+from ...commands.portal import user_has_power_level, get_initial_state
+from ..common import AuthAPI
+
+if TYPE_CHECKING:
+    from ...context import Context
+
+
+class ProvisioningAPI(AuthAPI):
+    log = logging.getLogger("mau.web.provisioning")
+
+    def __init__(self, context: "Context"):
+        super().__init__(context.loop)
+        self.secret = context.config["appservice.provisioning.shared_secret"]
+        self.az = context.az  # type: AppService
+        self.context = context  # type: Context
+
+        self.app = web.Application(loop=context.loop, middlewares=[self.error_middleware])
+
+        portal_prefix = "/portal/{mxid:![^/]+}"
+        self.app.router.add_route("GET", f"{portal_prefix}", self.get_portal_by_mxid)
+        self.app.router.add_route("GET", "/portal/{tgid:-[0-9]+}", self.get_portal_by_tgid)
+        self.app.router.add_route("POST", portal_prefix + "/connect/{chat_id:[0-9]+}",
+                                  self.connect_chat)
+        self.app.router.add_route("POST", f"{portal_prefix}/create", self.create_chat)
+        self.app.router.add_route("POST", f"{portal_prefix}/disconnect", self.disconnect_chat)
+
+        user_prefix = "/user/{mxid:@[^:]*:[^/]+}"
+        self.app.router.add_route("GET", f"{user_prefix}", self.get_user_info)
+        self.app.router.add_route("GET", f"{user_prefix}/chats", self.get_chats)
+
+        self.app.router.add_route("POST", f"{user_prefix}/logout", self.logout)
+        self.app.router.add_route("POST", f"{user_prefix}/login/bot_token", self.send_bot_token)
+        self.app.router.add_route("POST", f"{user_prefix}/login/request_code", self.request_code)
+        self.app.router.add_route("POST", f"{user_prefix}/login/send_code", self.send_code)
+        self.app.router.add_route("POST", f"{user_prefix}/login/send_password", self.send_password)
+
+    async def get_portal_by_mxid(self, request: web.Request) -> web.Response:
+        err = self.check_authorization(request)
+        if err is not None:
+            return err
+
+        mxid = request.match_info["mxid"]
+        portal = Portal.get_by_mxid(mxid)
+        if not portal:
+            return self.get_error_response(404, "portal_not_found",
+                                           "Portal with given Matrix ID not found.")
+        return web.json_response({
+            "mxid": portal.mxid,
+            "chat_id": get_peer_id(portal.peer),
+            "peer_type": portal.peer_type,
+            "title": portal.title,
+            "about": portal.about,
+            "username": portal.username,
+            "megagroup": portal.megagroup,
+        })
+
+    async def get_portal_by_tgid(self, request: web.Request) -> web.Response:
+        err = self.check_authorization(request)
+        if err is not None:
+            return err
+
+        try:
+            tgid, _ = resolve_id(int(request.match_info["tgid"]))
+        except ValueError:
+            return self.get_error_response(400, "tgid_invalid",
+                                           "Given chat ID is not valid.")
+        portal = Portal.get_by_tgid(tgid)
+        if not portal:
+            return self.get_error_response(404, "portal_not_found",
+                                           "Portal to given Telegram chat not found.")
+        return web.json_response({
+            "mxid": portal.mxid,
+            "chat_id": get_peer_id(portal.peer),
+            "peer_type": portal.peer_type,
+            "title": portal.title,
+            "about": portal.about,
+            "username": portal.username,
+            "megagroup": portal.megagroup,
+        })
+
+    async def connect_chat(self, request: web.Request) -> web.Response:
+        err = self.check_authorization(request)
+        if err is not None:
+            return err
+
+        room_id = request.match_info["mxid"]
+        if Portal.get_by_mxid(room_id):
+            return self.get_error_response(409, "room_already_bridged",
+                                           "Room is already bridged to another Telegram chat.")
+
+        chat_id = request.match_info["chat_id"]
+        if chat_id.startswith("-100"):
+            tgid = int(chat_id[4:])
+            peer_type = "channel"
+        elif chat_id.startswith("-"):
+            tgid = -int(chat_id)
+            peer_type = "chat"
+        else:
+            return self.get_error_response(400, "tgid_invalid", "Invalid Telegram chat ID.")
+
+        user, err = await self.get_user(request.query.get("user_id", None), expect_logged_in=None,
+                                        require_puppeting=False)
+        if err is not None:
+            return err
+        elif user and not await user_has_power_level(room_id, self.az.intent, user, "bridge"):
+            return self.get_error_response(403, "not_enough_permissions",
+                                           "You do not have the permissions to bridge that room.")
+
+        portal = Portal.get_by_tgid(tgid, peer_type=peer_type)
+        if portal.mxid == room_id:
+            return self.get_error_response(200, "bridge_exists",
+                                           "Telegram chat is already bridged to that Matrix room.")
+        elif portal.mxid:
+            force = request.query.get("force", None)
+            if force in ("delete", "unbridge"):
+                delete = force == "delete"
+                await portal.cleanup_room(portal.main_intent, portal.mxid, puppets_only=not delete,
+                                          message=("Portal deleted (moving to another room)"
+                                                   if delete
+                                                   else "Room unbridged (portal moving to another "
+                                                        "room)"))
+            else:
+                return self.get_error_response(409, "chat_already_bridged",
+                                               "Telegram chat is already bridged to another "
+                                               "Matrix room.")
+
+        is_logged_in = user is not None and await user.is_logged_in()
+        user = user if is_logged_in else self.context.bot
+        if not user:
+            return self.get_login_response(status=403, errcode="not_logged_in",
+                                           error="You are not logged in and there is no relay bot.")
+
+        entity = None  # type: Optional[TypeChat]
+        try:
+            entity = await user.client.get_entity(portal.peer)
+        except Exception:
+            self.log.exception("Failed to get_entity(%s) for manual bridging.", portal.peer)
+
+        if not entity or isinstance(entity, (ChatForbidden, ChannelForbidden)):
+            if is_logged_in:
+                return self.get_error_response(403, "user_not_in_chat",
+                                               "Failed to get info of Telegram chat. "
+                                               "Are you in the chat?")
+            return self.get_error_response(403, "bot_not_in_chat",
+                                           "Failed to get info of Telegram chat. "
+                                           "Is the relay bot in the chat?")
+
+        direct = False
+
+        portal.mxid = room_id
+        portal.title, portal.about, levels = await get_initial_state(self.az.intent, room_id)
+        portal.photo_id = ""
+        portal.save()
+
+        asyncio.ensure_future(portal.update_matrix_room(user, entity, direct, levels=levels),
+                              loop=self.loop)
+
+        return web.Response(status=202, body="{}")
+
+    async def create_chat(self, request: web.Request) -> web.Response:
+        err = self.check_authorization(request)
+        if err is not None:
+            return err
+
+        data = await self.get_data(request)
+        if not data:
+            return self.get_error_response(400, "json_invalid", "Invalid JSON.")
+
+        room_id = request.match_info["mxid"]
+        if Portal.get_by_mxid(room_id):
+            return self.get_error_response(409, "room_already_bridged",
+                                           "Room is already bridged to another Telegram chat.")
+
+        user, err = await self.get_user(request.query.get("user_id", None), expect_logged_in=None,
+                                        require_puppeting=False)
+        if err is not None:
+            return err
+        elif not await user.is_logged_in() or user.is_bot:
+            return self.get_error_response(403, "not_logged_in_real_account",
+                                           "You are not logged in with a real account.")
+        elif not await user_has_power_level(room_id, self.az.intent, user, "bridge"):
+            return self.get_error_response(403, "not_enough_permissions",
+                                           "You do not have the permissions to bridge that room.")
+
+        try:
+            title, about, _ = await get_initial_state(self.az.intent, room_id)
+        except (MatrixRequestError, IntentError):
+            return self.get_error_response(403, "bot_not_in_room",
+                                           "The bridge bot is not in the given room.")
+
+        about = data.get("about", about)
+
+        title = data.get("title", title)
+        if len(title) == 0:
+            return self.get_error_response(400, "body_value_invalid", "Title can not be empty.")
+
+        type = data.get("type", "")
+        if type not in ("group", "chat", "supergroup", "channel"):
+            return self.get_error_response(400, "body_value_invalid",
+                                           "Given chat type is not valid.")
+
+        supergroup = type == "supergroup"
+        type = {
+            "supergroup": "channel",
+            "channel": "channel",
+            "chat": "chat",
+            "group": "chat",
+        }[type]
+
+        portal = Portal(tgid=None, mxid=room_id, title=title, about=about, peer_type=type)
+        try:
+            await portal.create_telegram_chat(user, supergroup=supergroup)
+        except ValueError as e:
+            portal.delete()
+            return self.get_error_response(500, "unknown_error", e.args[0])
+
+        return web.json_response({
+            "chat_id": portal.tgid,
+        }, status=201)
+
+    async def disconnect_chat(self, request: web.Request) -> web.Response:
+        err = self.check_authorization(request)
+        if err is not None:
+            return err
+
+        portal = Portal.get_by_mxid(request.match_info["mxid"])
+        if not portal or not portal.tgid:
+            return self.get_error_response(404, "portal_not_found",
+                                           "Room is not a portal.")
+
+        user, err = await self.get_user(request.query.get("user_id", None), expect_logged_in=None,
+                                        require_puppeting=False, require_user=False)
+        if err is not None:
+            return err
+        elif user and not await user_has_power_level(portal.mxid, self.az.intent, user, "unbridge"):
+            return self.get_error_response(403, "not_enough_permissions",
+                                           "You do not have the permissions to unbridge that room.")
+
+        delete = request.query.get("delete", "").lower() in ("true", "t", "1", "yes", "y")
+        sync = request.query.get("delete", "").lower() in ("true", "t", "1", "yes", "y")
+
+        coro = portal.cleanup_and_delete() if delete else portal.unbridge()
+        if sync:
+            try:
+                await coro
+            except Exception:
+                self.log.exception("Failed to disconnect chat")
+                return self.get_error_response(500, "exception", "Failed to disconnect chat")
+        else:
+            asyncio.ensure_future(coro, loop=self.loop)
+        return web.json_response({}, status=200 if sync else 202)
+
+    async def get_user_info(self, request: web.Request) -> web.Response:
+        data, user, err = await self.get_user_request_info(request, expect_logged_in=None,
+                                                           require_puppeting=False)
+        if err is not None:
+            return err
+
+        user_data = None
+        if await user.is_logged_in():
+            me = await user.client.get_me()
+            await user.update_info(me)
+            user_data = {
+                "id": user.tgid,
+                "username": user.username,
+                "first_name": me.first_name,
+                "last_name": me.last_name,
+                "phone": me.phone,
+                "is_bot": user.is_bot,
+            }
+        return web.json_response({
+            "telegram": user_data,
+            "mxid": user.mxid,
+            "permissions": user.permissions,
+        })
+
+    async def get_chats(self, request: web.Request) -> web.Response:
+        data, user, err = await self.get_user_request_info(request, expect_logged_in=True)
+        if err is not None:
+            return err
+
+        if not user.is_bot:
+            chats = await user.get_dialogs()
+            return web.json_response([{
+                "id": get_peer_id(chat),
+                "title": chat.title,
+            } for chat in chats])
+        else:
+            return web.json_response([{
+                "id": get_peer_id(chat.peer),
+                "title": chat.title,
+            } for chat in user.portals.values() if chat.tgid])
+
+    async def send_bot_token(self, request: web.Request) -> web.Response:
+        data, user, err = await self.get_user_request_info(request)
+        if err is not None:
+            return err
+        return await self.post_login_token(user, data.get("token", ""))
+
+    async def request_code(self, request: web.Request) -> web.Response:
+        data, user, err = await self.get_user_request_info(request)
+        if err is not None:
+            return err
+        return await self.post_login_phone(user, data.get("phone", ""))
+
+    async def send_code(self, request: web.Request) -> web.Response:
+        data, user, err = await self.get_user_request_info(request)
+        if err is not None:
+            return err
+        return await self.post_login_code(user, data.get("code", 0), password_in_data=False)
+
+    async def send_password(self, request: web.Request) -> web.Response:
+        data, user, err = await self.get_user_request_info(request)
+        if err is not None:
+            return err
+        return await self.post_login_password(user, data.get("password", ""))
+
+    async def logout(self, request: web.Request) -> web.Response:
+        _, user, err = await self.get_user_request_info(request, expect_logged_in=True,
+                                                        require_puppeting=False,
+                                                        want_data=False)
+        if err is not None:
+            return err
+        await user.log_out()
+
+    @staticmethod
+    async def error_middleware(_, handler) -> Callable[[web.Request], Awaitable[web.Response]]:
+        async def middleware_handler(request: web.Request) -> web.Response:
+            try:
+                return await handler(request)
+            except web.HTTPException as ex:
+                return web.json_response({
+                    "error": f"Unhandled HTTP {ex.status}",
+                    "errcode": f"unhandled_http_{ex.status}",
+                }, status=ex.status)
+
+        return middleware_handler
+
+    @staticmethod
+    def get_error_response(status=200, errcode="", error="") -> web.Response:
+        return web.json_response({
+            "error": error,
+            "errcode": errcode,
+        }, status=status)
+
+    def get_mx_login_response(self, status=200, state="", username="", mxid="", message="",
+                              error="", errcode=""):
+        raise NotImplementedError()
+
+    def get_login_response(self, status=200, state="", username="", mxid="", message="", error="",
+                           errcode="") -> web.Response:
+        if username:
+            resp = {
+                "state": "logged-in",
+                "username": username,
+            }
+        elif message:
+            resp = {
+                "state": state,
+                "message": message,
+            }
+        else:
+            resp = {
+                "error": error,
+                "errcode": errcode,
+            }
+            if state:
+                resp["state"] = state
+        return web.json_response(resp, status=status)
+
+    def check_authorization(self, request: web.Request) -> Optional[web.Response]:
+        auth = request.headers.get("Authorization", "")
+        if auth != f"Bearer {self.secret}":
+            return self.get_error_response(error="Shared secret is not valid.",
+                                           errcode="shared_secret_invalid",
+                                           status=401)
+        return None
+
+    @staticmethod
+    async def get_data(request: web.Request) -> Optional[dict]:
+        try:
+            return await request.json()
+        except json.JSONDecodeError:
+            return None
+
+    async def get_user(self, mxid: str, expect_logged_in: Optional[bool] = False,
+                       require_puppeting: bool = True, require_user: bool = True
+                       ) -> Tuple[Optional[User], Optional[web.Response]]:
+        if not mxid:
+            if not require_user:
+                return None, None
+            return None, self.get_login_response(error="User ID not given.",
+                                                 errcode="mxid_empty", status=400)
+
+        user = await User.get_by_mxid(mxid).ensure_started(even_if_no_session=True)
+        if require_puppeting and not user.puppet_whitelisted:
+            return user, self.get_login_response(error="You are not whitelisted.",
+                                                 errcode="mxid_not_whitelisted", status=403)
+        if expect_logged_in is not None:
+            logged_in = await user.is_logged_in()
+            if not expect_logged_in and logged_in:
+                return user, self.get_login_response(username=user.username, status=409,
+                                                     error="You are already logged in.",
+                                                     errcode="already_logged_in")
+            elif expect_logged_in and not logged_in:
+                return user, self.get_login_response(status=403, error="You are not logged in.",
+                                                     errcode="not_logged_in")
+        return user, None
+
+    async def get_user_request_info(self, request: web.Request,
+                                    expect_logged_in: Optional[bool] = False,
+                                    require_puppeting: bool = False,
+                                    want_data: bool = True,
+                                    ) -> (Tuple[Optional[dict],
+                                                Optional[User],
+                                                Optional[web.Response]]):
+        err = self.check_authorization(request)
+        if err is not None:
+            return err
+
+        data = None
+        if want_data and (request.method == "POST" or request.method == "PUT"):
+            data = await self.get_data(request)
+            if not data:
+                return None, None, self.get_login_response(error="Invalid JSON.",
+                                                           errcode="json_invalid", status=400)
+
+        mxid = request.match_info["mxid"]
+        user, err = await self.get_user(mxid, expect_logged_in, require_puppeting)
+
+        return data, user, err

mautrix_telegram/web/provisioning/spec.yaml

@@ -0,0 +1,861 @@
+swagger: "2.0"
+
+info:
+  title: Mautrix-Telegram provisioning
+  version: 0.3.0
+  description: The provisioning API for Mautrix-Telegram, the Matrix-Telegram puppeting/relaybot bridge.
+  license:
+    name: AGPLv3
+    url: https://github.com/tulir/mautrix-telegram/blob/master/LICENSE
+
+externalDocs:
+  description: Provisioning API wiki page on GitHub
+  url: https://github.com/tulir/mautrix-telegram/wiki/Provisioning-API
+
+basePath: /_matrix/provision/v1
+
+schemes: [https]
+consumes: [application/json]
+produces: [application/json]
+
+tags:
+- name: User info
+- name: Authentication
+- name: Bridging
+
+paths:
+  /portal/{room_id}:
+    get:
+      operationId: get_portal
+      summary: Get the bridging status and info of the connected Telegram chat
+      tags: [Bridging]
+      responses:
+        200:
+          description: Room is bridged
+          schema:
+            $ref: "#/definitions/PortalInfo"
+        400:
+          $ref: "#/responses/BadRequest"
+        404:
+          description: Unknown portal
+          schema:
+            type: object
+            title: Error
+            properties:
+              errcode:
+                type: string
+                title: Error code
+                description: A machine-readable error code
+                enum:
+                - portal_not_found
+              error:
+                $ref: "#/definitions/HumanReadableError"
+      parameters:
+        - name: room_id
+          in: path
+          description: The Matrix ID of the room whose bridging status to get
+          required: true
+          type: string
+          pattern: "![^/]+"
+  /portal/{chat_id}:
+    get:
+      operationId: get_portal_by_tgid
+      summary: Get the bridging status and info of the connected Telegram chat
+      tags: [Bridging]
+      responses:
+        200:
+          description: Chat is bridged
+          schema:
+            $ref: "#/definitions/PortalInfo"
+        400:
+          description: Invalid Telegram chat ID
+          schema:
+            type: object
+            title: Error
+            properties:
+              errcode:
+                type: string
+                title: Error code
+                description: A machine-readable error code
+                enum:
+                - tgid_invalid
+              error:
+                $ref: "#/definitions/HumanReadableError"
+        404:
+          description: Unknown portal
+          schema:
+            type: object
+            title: Error
+            properties:
+              errcode:
+                type: string
+                title: Error code
+                description: A machine-readable error code
+                enum:
+                - portal_not_found
+              error:
+                $ref: "#/definitions/HumanReadableError"
+      parameters:
+        - name: chat_id
+          in: path
+          description: The Matrix ID of the room whose bridging status to get
+          required: true
+          type: integer
+          pattern: "-[0-9]+"
+  /portal/{room_id}/connect/{chat_id}:
+    post:
+      operationId: connect_portal
+      summary: Connect an existing Telegram chat to the given room
+      tags: [Bridging]
+      parameters:
+        - name: room_id
+          in: path
+          description: The Matrix ID of the room to which the Telegram chat should be connected
+          required: true
+          type: string
+        - name: chat_id
+          in: path
+          description: The ID of the Telegram chat to connect
+          required: true
+          type: integer
+          pattern: "-[0-9]+"
+        - name: force
+          in: query
+          description: Set to force bridging by unbridging or deleting existing portal rooms.
+          required: false
+          type: string
+          enum:
+          - delete
+          - unbridge
+        - name: user_id
+          in: query
+          description: Optional Matrix user ID to check if the user has permissions to do the bridging.
+          required: false
+          type: string
+      responses:
+        200:
+          description: Telegram chat was already bridged to given room.
+        202:
+          description: Room bridging initiated
+        400:
+          $ref: "#/responses/BadRequest"
+        403:
+          description: "Given user doesn't have permission to bridge the room, or the bridge bot is not in the room"
+          schema:
+            type: object
+            title: Error
+            properties:
+              errcode:
+                type: string
+                title: Error code
+                description: A machine-readable error code
+                enum:
+                - not_enough_permissions
+                - bot_not_in_room
+                - bot_not_in_chat
+                - not_logged_in
+              error:
+                $ref: "#/definitions/HumanReadableError"
+        409:
+          description: Matrix room or Telegram chat is already bridged to another chat/room
+          schema:
+            type: object
+            title: Error
+            properties:
+              errcode:
+                type: string
+                title: Error code
+                description: A machine-readable error code
+                example: <room|chat>_already_bridged
+                enum:
+                - room_already_bridged
+                - chat_already_bridged
+              error:
+                $ref: "#/definitions/HumanReadableError"
+  /portal/{room_id}/create:
+    post:
+      operationId: create_portal
+      summary: Create a new Telegram chat for the given room
+      tags: [Bridging]
+      responses:
+        201:
+          description: Telegram chat created
+          schema:
+            type: object
+            properties:
+              chat_id:
+                type: integer
+        400:
+          $ref: "#/responses/BadRequest"
+        403:
+          description: "Given user isn't logged in with a real account or doesn't have permission to bridge the room, or the bridge bot is not in the room"
+          schema:
+            type: object
+            title: Error
+            properties:
+              errcode:
+                type: string
+                title: Error code
+                description: A machine-readable error code
+                enum:
+                - not_logged_in_real_account
+                - not_enough_permissions
+                - bot_not_in_room
+              error:
+                $ref: "#/definitions/HumanReadableError"
+        409:
+          description: Room is already bridged
+          schema:
+            type: object
+            title: Error
+            properties:
+              errcode:
+                type: string
+                title: Error code
+                description: A machine-readable error code
+                enum:
+                - room_already_bridged
+              error:
+                $ref: "#/definitions/HumanReadableError"
+      parameters:
+        - name: room_id
+          in: path
+          description: The Matrix ID of the room whose bridging status to get
+          required: true
+          type: string
+        - name: body
+          in: body
+          required: true
+          schema:
+            type: object
+            required: [type]
+            properties:
+              type:
+                description: The type of chat to create
+                type: string
+                example: supergroup
+                enum:
+                - chat
+                - supergroup
+                - channel
+              title:
+                description: Title for the new chat
+                type: string
+                example: Mautrix-Telegram Bridge
+              about:
+                description: About text for the new chat
+                type: string
+                example: Discussion about mautrix-telegram
+        - name: user_id
+          in: query
+          description: Matrix user to create the chat as.
+          required: true
+          type: string
+  /portal/{room_id}/disconnect:
+    post:
+      operationId: disconnect_portal
+      summary: Disconnect the Telegram chat from the room
+      tags: [Bridging]
+      responses:
+        202:
+          description: Room unbridging initiated
+        400:
+          $ref: "#/responses/BadRequest"
+        403:
+          $ref: "#/responses/PermissionError"
+        404:
+          description: Unknown portal
+          schema:
+            type: object
+            title: Error
+            properties:
+              errcode:
+                type: string
+                title: Error code
+                description: A machine-readable error code
+                enum:
+                - portal_not_found
+              error:
+                $ref: "#/definitions/HumanReadableError"
+      parameters:
+        - name: room_id
+          in: path
+          description: The Matrix ID of the room whose bridging status to get
+          required: true
+          type: string
+        - name: user_id
+          in: query
+          description: Optional Matrix user ID to check if the user has permissions to do the bridging.
+          required: false
+          type: string
+        - name: delete
+          in: query
+          description: Whether or not to delete the room completely (kick all users instead of just Telegram puppets)
+          required: false
+          type: boolean
+          default: false
+        - name: sync
+          in: query
+          description: Whether or not to wait for the unbridging to be completed before responding. **Could cause timeouts in large rooms**
+          required: false
+          type: boolean
+          default: false
+
+  /user/{user_id}:
+    get:
+      operationId: get_me
+      summary: Get the info of the Telegram user the given Matrix user is logged in as
+      tags: [User info]
+      responses:
+        200:
+          description: User found
+          schema:
+            $ref: "#/definitions/UserInfo"
+        400:
+          $ref: "#/responses/BadRequest"
+        403:
+          $ref: "#/responses/NotWhitelistedError"
+        500:
+          $ref: "#/responses/UnknownError"
+      parameters:
+        - name: user_id
+          in: path
+          description: The Matrix ID of the user who to log in as
+          required: true
+          type: string
+  /user/{user_id}/chats:
+    get:
+      operationId: get_chats
+      summary: Get the list of Telegram chats the given Matrix user has access to
+      tags: [User info]
+      responses:
+        200:
+          description: User is logged in
+          schema:
+            $ref: "#/definitions/UserChats"
+        400:
+          $ref: "#/responses/BadRequest"
+        403:
+          description: User is not logged in or not whitelisted
+          schema:
+            type: object
+            title: Error
+            properties:
+              errcode:
+                type: string
+                title: Error code
+                description: A machine-readable error code
+                enum:
+                - not_logged_in
+                - mxid_not_whitelisted
+              error:
+                $ref: "#/definitions/HumanReadableError"
+        500:
+          $ref: "#/responses/UnknownError"
+      parameters:
+        - name: user_id
+          in: path
+          description: The Matrix ID of the user who to log in as
+          required: true
+          type: string
+
+  /user/{user_id}/login/bot_token:
+    post:
+      operationId: post_bot_token
+      summary: Log in with a bot token
+      tags: [Authentication]
+      responses:
+        200:
+          description: Login successful
+          schema:
+            $ref: "#/definitions/AuthSuccess"
+        400:
+          $ref: "#/responses/BadRequest"
+        401:
+          description: Invalid or expired bot token or invalid shared secret
+          schema:
+            type: object
+            title: Error
+            properties:
+              errcode:
+                type: string
+                title: Error code
+                description: A machine-readable error code
+                example: bot_token_<error>
+                enum:
+                - bot_token_invalid
+                - bot_token_expired
+                - shared_secret_invalid
+              error:
+                $ref: "#/definitions/HumanReadableError"
+        403:
+          $ref: "#/responses/NotWhitelistedError"
+        409:
+          $ref: "#/responses/AlreadyLoggedInError"
+        500:
+          $ref: "#/responses/UnknownError"
+      parameters:
+        - name: user_id
+          in: path
+          description: The Matrix ID of the user who to log in as
+          required: true
+          type: string
+        - name: body
+          in: body
+          required: true
+          schema:
+            type: object
+            properties:
+              token:
+                type: string
+                description: The access token of the bot to log in as
+                example: "297900271:IXjeGEcAN61zHnjPgkWnYWyvVp9K4ulHBEv"
+  /user/{user_id}/login/request_code:
+    post:
+      operationId: post_login_phone
+      summary: Request a phone code from Telegram
+      tags: [Authentication]
+      responses:
+        200:
+          description: Code requested successfully
+          schema:
+            $ref: "#/definitions/AuthSuccess"
+        400:
+          description: Invalid phone number or JSON
+          schema:
+            type: object
+            title: Error
+            properties:
+              errcode:
+                type: string
+                title: Error code
+                description: A machine-readable error code
+                example: machine_readable_error
+                enum:
+                - phone_number_invalid
+                - json_invalid
+              error:
+                $ref: "#/definitions/HumanReadableError"
+        401:
+          description: Invalid shared secret
+          schema:
+            type: object
+            title: Error
+            properties:
+              errcode:
+                type: string
+                title: Error code
+                description: A machine-readable error code
+                enum:
+                - shared_secret_invalid
+              error:
+                $ref: "#/definitions/HumanReadableError"
+        403:
+          description: Matrix ID is not whitelisted or phone number is banned or has forbidden 3rd party apps
+          schema:
+            type: object
+            title: Error
+            properties:
+              errcode:
+                type: string
+                title: Error code
+                description: A machine-readable error code
+                example: machine_readable_error
+                enum:
+                - mxid_not_whitelisted
+                - phone_number_banned
+                - phone_number_app_signup_forbidden
+              error:
+                $ref: "#/definitions/HumanReadableError"
+        404:
+          description: Unregistered phone number
+          schema:
+            type: object
+            title: Error
+            properties:
+              errcode:
+                type: string
+                title: Error code
+                description: A machine-readable error code
+                enum:
+                - phone_number_unoccupied
+              error:
+                $ref: "#/definitions/HumanReadableError"
+        409:
+          $ref: "#/responses/AlreadyLoggedInError"
+        429:
+          description: Phone number has been temporarily blocked for flooding
+          schema:
+            type: object
+            title: Error
+            properties:
+              errcode:
+                type: string
+                title: Error code
+                description: A machine-readable error code
+                enum:
+                - flood_wait
+                - phone_number_flood
+              error:
+                $ref: "#/definitions/HumanReadableError"
+        500:
+          $ref: "#/responses/UnknownError"
+      parameters:
+        - name: user_id
+          in: path
+          description: The Matrix ID of the user who to log in as
+          required: true
+          type: string
+        - name: body
+          in: body
+          required: true
+          schema:
+            type: object
+            properties:
+              phone:
+                type: string
+                description: The phone number to log in as.
+                example: "+123456789"
+  /user/{user_id}/login/send_code:
+    post:
+      operationId: post_login_code
+      summary: Send the login code
+      tags: [Authentication]
+      responses:
+        200:
+          description: Login successful
+          schema:
+            $ref: "#/definitions/AuthSuccess"
+        202:
+          description: Correct code, but two-factor authentication is enabled
+          schema:
+            $ref: "#/definitions/AuthSuccess"
+        400:
+          $ref: "#/responses/BadRequest"
+        401:
+          description: Invalid phone code or shared secret
+          schema:
+            type: object
+            title: Error
+            properties:
+              errcode:
+                type: string
+                title: Error code
+                description: A machine-readable error code
+                enum:
+                - phone_code_invalid
+                - shared_secret_invalid
+              error:
+                $ref: "#/definitions/HumanReadableError"
+        403:
+          description: Matrix ID not whitelisted or phone code expired
+          schema:
+            type: object
+            title: Error
+            properties:
+              errcode:
+                type: string
+                title: Error code
+                description: A machine-readable error code
+                example: machine_readable_error
+                enum:
+                - mxid_not_whitelisted
+                - phone_code_expired
+              error:
+                $ref: "#/definitions/HumanReadableError"
+        409:
+          $ref: "#/responses/AlreadyLoggedInError"
+        500:
+          $ref: "#/responses/UnknownError"
+      parameters:
+        - name: user_id
+          in: path
+          description: The Matrix ID of the user who to log in as
+          required: true
+          type: string
+        - name: body
+          in: body
+          required: true
+          schema:
+            type: object
+            properties:
+              code:
+                type: integer
+                description: The phone code from Telegram.
+                format: int32
+                example: 123456
+  /user/{user_id}/login/send_password:
+    post:
+      operationId: post_login_password
+      summary: Send the two-factor auth password
+      tags: [Authentication]
+      responses:
+        200:
+          description: Login successful
+          schema:
+            $ref: "#/definitions/AuthSuccess"
+        400:
+          description: Missing password or invalid JSON
+          schema:
+            type: object
+            title: Error
+            properties:
+              errcode:
+                type: string
+                title: Error code
+                description: A machine-readable error code
+                example: <field>_empty
+                enum:
+                - password_empty
+                - json_invalid
+              error:
+                $ref: "#/definitions/HumanReadableError"
+        401:
+          description: Incorrect password or invalid shared secret
+          schema:
+            type: object
+            title: Error
+            properties:
+              errcode:
+                type: string
+                title: Error code
+                description: A machine-readable error code
+                enum:
+                - password_invalid
+                - shared_secret_invalid
+              error:
+                $ref: "#/definitions/HumanReadableError"
+        403:
+          $ref: "#/responses/NotWhitelistedError"
+        409:
+          $ref: "#/responses/AlreadyLoggedInError"
+        500:
+          $ref: "#/responses/UnknownError"
+      parameters:
+        - name: user_id
+          in: path
+          description: The Matrix ID of the user who to log in as
+          required: true
+          type: string
+        - name: body
+          in: body
+          required: true
+          schema:
+            type: object
+            properties:
+              password:
+                type: string
+                description: The two-factor auth password
+                format: password
+                example: hunter2
+  /user/{user_id}/logout:
+    post:
+      operationId: logout
+      summary: Log out
+      tags: [Authentication]
+      responses:
+        200:
+          description: Logout successful
+        403:
+          description: User was not logged in
+          schema:
+            type: object
+            title: Error
+            properties:
+              errcode:
+                type: string
+                title: Error code
+                description: A machine-readable error code
+                enum:
+                - not_logged_in
+              error:
+                $ref: "#/definitions/HumanReadableError"
+        500:
+          $ref: "#/responses/UnknownError"
+      parameters:
+        - name: user_id
+          in: path
+          description: The Matrix ID of the user who to log out as
+          required: true
+          type: string
+
+responses:
+  NotWhitelistedError:
+    description: Matrix ID not whitelisted for puppeting
+    schema:
+      type: object
+      title: Error
+      properties:
+        errcode:
+          type: string
+          title: Error code
+          description: A machine-readable error code
+          enum:
+          - mxid_not_whitelisted
+        error:
+          $ref: "#/definitions/HumanReadableError"
+  AlreadyLoggedInError:
+    description: The Matrix user is already logged in
+    schema:
+      type: object
+      properties:
+        state:
+          type: string
+          enum:
+          - logged-in
+        username:
+          type: string
+          description: The Telegram username the user is logged in as.
+  BadRequest:
+    description: Invalid JSON.
+    schema:
+      type: object
+      title: Error
+      properties:
+        errcode:
+          type: string
+          title: Error code
+          description: A machine-readable error code
+          enum:
+          - json_invalid
+          - mxid_empty
+          - body_value_missing
+          - body_value_invalid
+        error:
+          $ref: "#/definitions/HumanReadableError"
+  UnknownError:
+    description: Unknown error
+    schema:
+      type: object
+      title: UnknownError
+      properties:
+        errcode:
+          type: string
+          title: Error code
+          description: A machine-readable error code
+          enum:
+          - unknown_error
+          - unhandled_error
+        error:
+          type: string
+          title: Error
+          description: A human-readable description of the error
+          example: Internal server error while <action>.
+  PermissionError:
+    description: The given Matrix user doesn't have the permissions to do that.
+    schema:
+      type: object
+      title: Error
+      properties:
+        errcode:
+          type: string
+          title: Error code
+          description: A machine-readable error code
+          example: not_enough_permissions
+          enum:
+          - not_enough_permissions
+        error:
+          $ref: "#/definitions/HumanReadableError"
+
+definitions:
+  UserInfo:
+    type: object
+    properties:
+      mxid:
+        type: string
+        example: "@usern:example.com"
+      permissions:
+        type: string
+        example: user
+        enum:
+        - none
+        - relaybot
+        - user
+        - full
+        - admin
+      telegram:
+        type: object
+        properties:
+          id:
+            type: integer
+            example: 123456789
+          username:
+            type: string
+            example: username
+          first_name:
+            type: string
+            example: Usern
+          last_name:
+            type: string
+            example: A.
+          phone:
+            type: string
+            example: +123456789
+          is_bot:
+            type: boolean
+            example: false
+  UserChats:
+    type: array
+    items:
+      type: object
+      properties:
+        id:
+          type: integer
+          example: -123456789
+          description: A bot API style chat ID.
+        title:
+          type: string
+
+  PortalInfo:
+    type: object
+    properties:
+      mxid:
+        type: string
+        example: "!foo:example.com"
+      chat_id:
+        type: integer
+        example: -100123456789
+      peer_type:
+        type: string
+        enum:
+        - user
+        - chat
+        - channel
+      megagroup:
+        type: boolean
+      username:
+        type: string
+      title:
+        type: string
+      about:
+        type: string
+
+  AuthSuccess:
+    type: object
+    properties:
+      state:
+        type: string
+        description: The state/next step after the successful operation.
+        enum:
+        - code
+        - request
+        - password
+        - token
+        - logged-in
+      username:
+        type: string
+        description: The Telegram username the user is logged in as. Only applicable if state=logged-in
+
+  HumanReadableError:
+    type: string
+    description: A human-readable description of the error
+    example: A human-readable description of the error
+
+security:
+  - Bearer: []
+securityDefinitions:
+  Bearer:
+    description: Required authentication for all endpoints
+    name: Authorization
+    in: header
+    type: apiKey

mautrix_telegram/web/public/__init__.py

@@ -0,0 +1,173 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2018 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from aiohttp import web
+from mako.template import Template
+import pkg_resources
+import logging
+import random
+import string
+import time
+
+from ...util import sign_token, verify_token
+from ...user import User
+from ...puppet import Puppet
+from ..common import AuthAPI
+
+
+class PublicBridgeWebsite(AuthAPI):
+    log = logging.getLogger("mau.web.public")
+
+    def __init__(self, loop):
+        super().__init__(loop)
+        self.secret_key = "".join(
+            random.choice(string.ascii_lowercase + string.digits) for _ in range(64))
+
+        self.login = Template(
+            pkg_resources.resource_string("mautrix_telegram", "web/public/login.html.mako"))
+
+        self.mx_login = Template(
+            pkg_resources.resource_string("mautrix_telegram", "web/public/matrix-login.html.mako"))
+
+        self.app = web.Application(loop=loop)
+        self.app.router.add_route("GET", "/login", self.get_login)
+        self.app.router.add_route("POST", "/login", self.post_login)
+        self.app.router.add_route("GET", "/matrix-login", self.get_matrix_login)
+        self.app.router.add_route("POST", "/matrix-login", self.post_matrix_login)
+        self.app.router.add_static("/", pkg_resources.resource_filename("mautrix_telegram",
+                                                                        "web/public/"))
+
+    def make_token(self, mxid, endpoint="/login", expires_in=900):
+        return sign_token(self.secret_key, {
+            "mxid": mxid,
+            "endpoint": endpoint,
+            "expiry": int(time.time()) + expires_in,
+        })
+
+    def verify_token(self, token, endpoint="/login"):
+        token = verify_token(self.secret_key, token)
+        if token and (token.get("expiry", 0) > int(time.time()) and
+                      token.get("endpoint", None) == endpoint):
+            return token.get("mxid", None)
+        return None
+
+    async def get_login(self, request):
+        state = "bot_token" if request.rel_url.query.get("mode", "") == "bot" else "request"
+
+        mxid = self.verify_token(request.rel_url.query.get("token", None), endpoint="/login")
+        if not mxid:
+            return self.get_login_response(status=401, state="invalid-token")
+        user = User.get_by_mxid(mxid, create=False) if mxid else None
+
+        if not user:
+            return self.get_login_response(mxid=mxid, state=state)
+        elif not user.puppet_whitelisted:
+            return self.get_login_response(mxid=user.mxid, error="You are not whitelisted.",
+                                           status=403)
+        await user.ensure_started()
+        if not await user.is_logged_in():
+            return self.get_login_response(mxid=user.mxid, state=state)
+
+        return self.get_login_response(mxid=user.mxid, username=user.username)
+
+    async def get_matrix_login(self, request):
+        mxid = self.verify_token(request.rel_url.query.get("token", None), endpoint="/matrix-login")
+        if not mxid:
+            return self.get_mx_login_response(status=401, state="invalid-token")
+        user = User.get_by_mxid(mxid, create=False) if mxid else None
+
+        if not user:
+            return self.get_mx_login_response(mxid=mxid)
+        elif not user.puppet_whitelisted:
+            return self.get_mx_login_response(mxid=user.mxid, error="You are not whitelisted.",
+                                              status=403)
+        await user.ensure_started()
+        if not await user.is_logged_in():
+            return self.get_mx_login_response(mxid=user.mxid, status=403,
+                                              error="You are not logged in to Telegram.")
+
+        puppet = Puppet.get(user.tgid)
+        if puppet.is_real_user:
+            return self.get_mx_login_response(state="already-logged-in", status=409)
+
+        return self.get_mx_login_response(mxid=user.mxid)
+
+    def get_login_response(self, status=200, state="", username="", mxid="", message="", error="",
+                           errcode=""):
+        return web.Response(status=status, content_type="text/html",
+                            text=self.login.render(username=username, state=state, error=error,
+                                                   message=message, mxid=mxid))
+
+    def get_mx_login_response(self, status=200, state="", username="", mxid="", message="",
+                              error="", errcode=""):
+        return web.Response(status=status, content_type="text/html",
+                            text=self.mx_login.render(username=username, state=state, error=error,
+                                                      message=message, mxid=mxid))
+
+    async def post_matrix_login(self, request):
+        mxid = self.verify_token(request.rel_url.query.get("token", None), endpoint="/matrix-login")
+        if not mxid:
+            return self.get_mx_login_response(status=401, state="invalid-token")
+
+        data = await request.post()
+
+        user = await User.get_by_mxid(mxid).ensure_started()
+        if not user.puppet_whitelisted:
+            return self.get_mx_login_response(mxid=user.mxid, error="You are not whitelisted.",
+                                              status=403)
+        elif not await user.is_logged_in():
+            return self.get_mx_login_response(mxid=user.mxid, status=403,
+                                              error="You are not logged in to Telegram.")
+        mode = data.get("mode", "access_token")
+        if mode == "password":
+            return await self.post_matrix_password(user, data["value"])
+        elif mode == "access_token":
+            return await self.post_matrix_token(user, data["value"])
+        return self.get_mx_login_response(mxid=user.mxid, status=400,
+                                          error="You must provide an access token or "
+                                                "password.")
+
+    async def post_login(self, request):
+        mxid = self.verify_token(request.rel_url.query.get("token", None), endpoint="/login")
+        if not mxid:
+            return self.get_login_response(status=401, state="invalid-token")
+
+        data = await request.post()
+
+        user = await User.get_by_mxid(mxid).ensure_started(even_if_no_session=True)
+        if not user.puppet_whitelisted:
+            return self.get_login_response(mxid=user.mxid, error="You are not whitelisted.",
+                                           status=403)
+        elif await user.is_logged_in():
+            return self.get_login_response(mxid=user.mxid, username=user.username)
+
+        await user.ensure_started(even_if_no_session=True)
+
+        if "phone" in data:
+            return await self.post_login_phone(user, data["phone"])
+        elif "bot_token" in data:
+            return await self.post_login_token(user, data["bot_token"])
+        elif "code" in data:
+            resp = await self.post_login_code(user, data["code"],
+                                              password_in_data="password" in data)
+            if resp or "password" not in data:
+                return resp
+        elif "password" not in data:
+            return self.get_login_response(error="No data given.", status=400)
+
+        if "password" in data:
+            return await self.post_login_password(user, data["password"])
+        return self.get_login_response(error="This should never happen.", status=500)

mautrix_telegram/web/public/login.css

@@ -19,8 +19,8 @@ form > div {
 	display: none;
 }
 
-form[data-status="request"]  > div.status-request,
-form[data-status="code"]     > div.status-code,
+form[data-status="request"] > div.status-request,
+form[data-status="code"] > div.status-code,
 form[data-status="password"] > div.status-password {
 	display: initial;
 }
@@ -48,3 +48,52 @@ form[data-status="password"] > div.status-password {
 	background-color: #d4edda;
 	color: #155724;
 }
+
+[type="checkbox"], [type="radio"] {
+	position: absolute;
+	opacity: 0;
+}
+
+[type="checkbox"] + label, [type="radio"] + label {
+	position: relative;
+	padding-left: 2.5rem;
+	cursor: pointer;
+	display: inline-block;
+}
+
+[type="checkbox"] + label:before, [type="radio"] + label:before {
+	content: '';
+	position: absolute;
+	left: 0;
+	top: 0.4rem;
+	width: 1.8rem;
+	height: 1.8rem;
+	border: 0.1rem solid #d1d1d1;
+}
+
+[type="radio"] + label:before, [type="radio"] + label:after {
+	border-radius: 50%;
+}
+
+[type="checkbox"]:checked + label:after,
+[type="radio"]:checked + label:after {
+	content: '';
+	width: 0.8rem;
+	height: 0.8rem;
+	background: #9b4dca;
+	position: absolute;
+	top: 0.9rem;
+	left: 0.5rem;
+}
+
+[type="radio"]:disabled + label:before, [type="checkbox"]:disabled + label:before {
+	background-color: #d1d1d1;
+}
+
+[type="radio"]:disabled + label, [type="checkbox"]:disabled + label {
+	color: #d1d1d1;
+}
+
+[type="radio"]:disabled:checked + label:after, [type="checkbox"]:disabled:checked + label:after {
+	background: #606c76;
+}

mautrix_telegram/web/public/login.html.mako

@@ -18,9 +18,9 @@ along with this program.  If not, see <https://www.gnu.org/licenses/>.
 <!DOCTYPE html>
 <html>
 <head>
-	<title>Mautrix-Telegram bridge</title>
+	<title>Login - Mautrix-Telegram bridge</title>
 	<link rel="icon" type="image/png" href="favicon.png"/>
-	<meta property="og:title" content="Mautrix-Telegram bridge">
+	<meta property="og:title" content="Login - Mautrix-Telegram bridge">
 	<meta property="og:description" content="A hybrid puppeting/relaybot Matrix-Telegram bridge">
 	<meta property="og:image" content="favicon.png">
 	<meta charset="utf-8">
@@ -29,6 +29,25 @@ along with this program.  If not, see <https://www.gnu.org/licenses/>.
 	<link rel="stylesheet"
 		  href="//cdn.rawgit.com/milligram/milligram/master/dist/milligram.min.css">
 	<link rel="stylesheet" href="login.css"/>
+
+	<script>
+		function switchToBotLogin() {
+			const params = new URLSearchParams(location.search.slice(1))
+			params.set("mode", "bot")
+			location.search = "?" + params.toString()
+			console.log(location.search)
+		}
+
+		function goBack() {
+			let params = new URLSearchParams(location.search.slice(1))
+			const token = params.get("token")
+			params = new URLSearchParams()
+			if (token) {
+				params.set("token", token)
+			}
+			location.replace(location.href.split("?")[0] + "?" + params.toString())
+		}
+	</script>
 </head>
 <body>
 <main class="container">
@@ -40,6 +59,13 @@ along with this program.  If not, see <https://www.gnu.org/licenses/>.
 				You can now close this page.
 				You should be invited to Telegram portals on Matrix momentarily.
 			</p>
+		% elif state == "bot-logged-in":
+			<h1>Logged in successfully!</h1>
+			<p>
+				Logged in as @${username}.
+				You can now close this page.
+				You should be invited to Telegram portals on Matrix momentarily.
+			</p>
 		% else:
 			<h1>You're already logged in!</h1>
 			<p>
@@ -50,6 +76,9 @@ along with this program.  If not, see <https://www.gnu.org/licenses/>.
 				management command first.
 			</p>
 		% endif
+	% elif state == "invalid-token":
+		<h1>Invalid or expired token</h1>
+		<div class="error">Please ask the bridge bot for a new login link.</div>
 	% else:
 		<h1>Log in to Telegram</h1>
 	% if error:
@@ -61,30 +90,31 @@ along with this program.  If not, see <https://www.gnu.org/licenses/>.
 		<form method="post">
 			<fieldset>
 				<label for="mxid">Matrix ID</label>
-				<input type="text" id="mxid" name="mxid" placeholder="Enter Matrix ID"
-					   value="${mxid}"/>
+				<input type="text" id="mxid" name="mxid" disabled value="${mxid}"/>
 				% if state == "request":
 					<label for="value">Phone number</label>
 					<input type="tel" id="value" name="phone" placeholder="Enter phone number"/>
 					<button type="submit">Request code</button>
+					<button class="button-clear" type="button" onclick="switchToBotLogin()">
+						Use bot token
+					</button>
+				% elif state == "bot_token":
+					<label for="value">Bot token</label>
+					<input type="text" id="value" name="bot_token" placeholder="Enter bot API token"/>
+					<button type="submit">Sign in</button>
 				% elif state == "code":
 					<label for="value">Phone code</label>
 					<input type="number" id="value" name="code" placeholder="Enter phone code"/>
 					<button type="submit">Sign in</button>
-					<div class="float-right">
-						<button class="button-clear" type="button"
-								onclick="location.replace(location.href)">
-							Go back
-						</button>
-					</div>
 				% elif state == "password":
 					<label for="value">Password</label>
 					<input type="password" id="value" name="password"
 						   placeholder="Enter password"/>
 					<button type="submit">Sign in</button>
+				% endif
+				% if state != "request":
 					<div class="float-right">
-						<button class="button-clear" type="button"
-								onclick="location.replace(location.href)">
+						<button class="button-clear" type="button" onclick="goBack()">
 							Go back
 						</button>
 					</div>

mautrix_telegram/web/public/matrix-login.html.mako

@@ -0,0 +1,78 @@
+<!--
+mautrix-telegram - A Matrix-Telegram puppeting bridge
+Copyright (C) 2018 Tulir Asokan
+
+This program is free software: you can redistribute it and/or modify
+it under the terms of the GNU Affero General Public License as published by
+the Free Software Foundation, either version 3 of the License, or
+(at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU Affero General Public License for more details.
+
+You should have received a copy of the GNU Affero General Public License
+along with this program.  If not, see <https://www.gnu.org/licenses/>.
+-->
+<!DOCTYPE html>
+<html>
+<head>
+	<title>Matrix login - Mautrix-Telegram bridge</title>
+	<link rel="icon" type="image/png" href="favicon.png"/>
+	<meta property="og:title" content="Matrix login - Mautrix-Telegram bridge">
+	<meta property="og:description" content="A hybrid puppeting/relaybot Matrix-Telegram bridge">
+	<meta property="og:image" content="favicon.png">
+	<meta charset="utf-8">
+	<link rel="stylesheet" href="//fonts.googleapis.com/css?family=Roboto:300,700">
+	<link rel="stylesheet" href="//cdn.rawgit.com/necolas/normalize.css/master/normalize.css">
+	<link rel="stylesheet"
+		  href="//cdn.rawgit.com/milligram/milligram/master/dist/milligram.min.css">
+	<link rel="stylesheet" href="login.css"/>
+</head>
+<body>
+<main class="container">
+	% if state == "logged-in":
+		<h1>Logged in successfully!</h1>
+		<p>
+			Logged in as ${mxid}.
+			You can now close this page.
+		</p>
+	% elif state == "already-logged-in":
+		<h1>You're already logged in!</h1>
+		<p>
+			If you want to log in with another account, log out using the
+			<code>logout-matrix</code> management command first.
+		</p>
+	% elif state == "invalid-token":
+		<h1>Invalid or expired token</h1>
+		<div class="error">Please ask the bridge bot for a new login link.</div>
+	% else:
+		<h1>Log in to Matrix</h1>
+	% if error:
+		<div class="error">${error}</div>
+	% endif
+	% if message:
+		<div class="message">${message}</div>
+	% endif
+		<form method="post">
+			<fieldset>
+				<label for="mxid">Matrix ID</label>
+				<input type="text" id="mxid" name="mxid" disabled value="${mxid}"/>
+
+				<input id="access_token" type="radio" name="mode" value="access_token" checked>
+				<label for="access_token">Access token</label><br>
+				<input id="password" type="radio" name="mode" value="password" disabled>
+				<label for="password">Password</label><br>
+
+				<label for="value">Value</label>
+				<input type="text" id="value" name="value"
+					   placeholder="Enter Matrix access token or password"/>
+
+				<button type="submit">Sign in</button>
+			</fieldset>
+		</form>
+	% endif
+</main>
+</body>
+</html>

requirements.txt

@@ -6,5 +6,5 @@ SQLAlchemy
 alembic
 Markdown
 future-fstrings
-telethon-aio
+telethon
 telethon-session-sqlalchemy

setup.py

@@ -1,15 +1,15 @@
 import setuptools
-import sys
 import glob
 import mautrix_telegram
 
 extras = {
     "highlight_edits": ["lxml>=4.1.1,<5"],
+    "better_formatter": ["lxml>=4.1.1,<5"],
     "fast_crypto": ["cryptg>=0.1,<0.2"],
     "webp_convert": ["Pillow>=5.0.0,<6"],
     "hq_thumbnails": ["moviepy>=0.2,<0.3"],
 }
-extras["all"] = [deps[0] for deps in extras.values()]
+extras["all"] = list(set(deps[0] for deps in extras.values()))
 
 setuptools.setup(
     name="mautrix-telegram",
@@ -21,19 +21,20 @@ setuptools.setup(
 
     description="A Matrix-Telegram hybrid puppeting/relaybot bridge.",
     long_description=open("README.md").read(),
+    long_description_content_type="text/markdown",
 
     packages=setuptools.find_packages(),
 
     install_requires=[
         "aiohttp>=3.0.1,<4",
-        "mautrix-appservice>=0.1.4,<0.2.0",
+        "mautrix-appservice>=0.3.6,<0.4.0",
         "SQLAlchemy>=1.2.3,<2",
-        "alembic>=0.9.8,<0.10",
+        "alembic>=1.0.0,<2",
         "Markdown>=2.6.11,<3",
         "ruamel.yaml>=0.15.35,<0.16",
         "future-fstrings>=0.4.2",
         "python-magic>=0.4.15,<0.5",
-        "telethon-aio>=0.19.0,<0.19.1",
+        "telethon>=1.0,<1.2",
         "telethon-session-sqlalchemy>=0.2.3,<0.3",
     ],
     extras_require=extras,
@@ -52,7 +53,7 @@ setuptools.setup(
         mautrix-telegram=mautrix_telegram.__main__:main
     """,
     package_data={"mautrix_telegram": [
-        "public/*.mako", "public/*.png", "public/*.css",
+        "web/public/*.mako", "web/public/*.png", "web/public/*.css",
     ]},
     data_files=[
         (".", ["example-config.yaml", "alembic.ini"]),