Bump version to 0.5.0

Add pytest unit testing framework

.codeclimate.yml

@@ -0,0 +1,8 @@
+engines:
+  sonar-python:
+    enabled: true
+    checks:
+      python:S107:
+        enabled: false
+exclude_patterns:
+- "alembic/"

.dockerignore

@@ -0,0 +1,4 @@
+.editorconfig
+.codeclimate.yml
+*.png
+*.md

.gitignore

@@ -1,12 +1,12 @@
 .idea/
 
 .venv
+env/
 pip-selfcheck.json
 *.pyc
 __pycache__
 
 config.yaml
 registration.yaml
+*.log*
 *.db
-*.session
-*.json

Dockerfile

@@ -1,30 +1,45 @@
-FROM docker.io/alpine:3.7
+FROM docker.io/alpine:3.9
 
 ENV UID=1337 \
-    GID=1337
+    GID=1337 \
+    FFMPEG_BINARY=/usr/bin/ffmpeg
 
-COPY . /opt/mautrixtelegram
+COPY . /opt/mautrix-telegram
+WORKDIR /opt/mautrix-telegram
 RUN apk add --no-cache \
-      python3-dev \
       py3-virtualenv \
       py3-pillow \
       py3-aiohttp \
       py3-lxml \
       py3-magic \
-      py3-numpy \
-      py3-asn1crypto \
       py3-sqlalchemy \
+      py3-markdown \
+      py3-psycopg2 \
+      # Not yet in stable repos:
+      #py3-ruamel \
+      # Indirect dependencies
+      #commonmark
+        py3-future \
+      #alembic
+        py3-mako \
+        py3-dateutil \
+        py3-markupsafe \
+      #moviepy
+        py3-decorator \
+        #py3-tqdm \
+        py3-requests \
+        #imageio
+          py3-numpy \
+      #telethon
+        py3-rsa \
+      # Other dependencies
+      python3-dev \
       build-base \
       ffmpeg \
-      bash \
       ca-certificates \
       su-exec \
-      s6 \
+ && pip3 install .[all]
- && cd /opt/mautrixtelegram \
- && cp -r docker/root/* / \
- && rm docker -rf \
- && pip3 install -r requirements.txt -r optional-requirements.txt
 
 VOLUME /data
 
-CMD ["/bin/s6-svscan", "/etc/s6.d"]
+CMD ["/opt/mautrix-telegram/docker-run.sh"]

ROADMAP.md

@@ -4,9 +4,9 @@
   * [x] Message content (text, formatting, files, etc..)
   * [x] Message redactions
   * [ ] ‡ Message history
-  * [ ] † Presence
+  * [x] Presence
-  * [ ] † Typing notifications
+  * [x] Typing notifications
-  * [ ] † Read receipts
+  * [x] Read receipts
   * [x] Pinning messages
   * [x] Power level
     * [x] Normal chats
@@ -21,6 +21,10 @@
     * [ ] ‡ Changes to displayname/avatar
 * Telegram → Matrix
   * [x] Message content (text, formatting, files, etc..)
+  * [ ] Advanced message content/media
+    * [x] Polls
+	* [x] Games
+	* [ ] Buttons
   * [x] Message deletions
   * [x] Message edits
   * [ ] Message history
@@ -46,8 +50,10 @@
     * [x] When receiving invite or message
   * [x] Private chat creation by inviting Matrix puppet of Telegram user to new room
   * [x] Option to use bot to relay messages for unauthenticated Matrix users
-  * [ ] Option to use own Matrix account for messages sent from other Telegram clients
+  * [x] Option to use own Matrix account for messages sent from other Telegram clients
   * [ ] ‡ Calls (hard, not yet supported by Telethon)
+  * [ ] ‡ Secret chats (not yet supported by Telethon)
+  * [ ] ‡ E2EE in Matrix rooms (not yet supported 
 
 † Information not automatically sent from source, i.e. implementation may not be possible  
 ‡ Maybe, i.e. this feature may or may not be implemented at some point

alembic/env.py

@@ -4,11 +4,12 @@ from logging.config import fileConfig
 
 import sys
 from os.path import abspath, dirname
+
 sys.path.insert(0, dirname(dirname(abspath(__file__))))
 
-from mautrix_telegram.base import Base
+from mautrix_telegram.db import Base
 from mautrix_telegram.config import Config
-import mautrix_telegram.db
+from alchemysession import AlchemySessionContainer
 
 # this is the Alembic Config object, which provides
 # access to the values within the .ini file in use.
@@ -20,6 +21,15 @@ mxtg_config.load()
 config.set_main_option("sqlalchemy.url",
                        mxtg_config.get("appservice.database", "sqlite:///mautrix-telegram.db"))
 
+
+class FakeDB:
+    @staticmethod
+    def query_property():
+        return None
+
+
+AlchemySessionContainer.create_table_classes(FakeDB(), "telethon_", Base)
+
 # Interpret the config file for Python logging.
 # This line sets up loggers basically.
 fileConfig(config.config_file_name)
@@ -30,6 +40,7 @@ fileConfig(config.config_file_name)
 # target_metadata = mymodel.Base.metadata
 target_metadata = Base.metadata
 
+
 # other values from the config, defined by the needs of env.py,
 # can be acquired:
 # my_important_option = config.get_main_option("my_important_option")
@@ -77,6 +88,7 @@ def run_migrations_online():
         with context.begin_transaction():
             context.run_migrations()
 
+
 if context.is_offline_mode():
     run_migrations_offline()
 else:

alembic/versions/1fa46383a9d3_add_is_bot_field_to_puppets.py

@@ -21,4 +21,5 @@ def upgrade():
 
 
 def downgrade():
-    op.drop_column('puppet', 'is_bot')
+    with op.batch_alter_table("puppet") as batch_op:
+        batch_op.drop_column('is_bot')

alembic/versions/30eca60587f1_add_megagroup_field_to_portals.py

@@ -20,4 +20,5 @@ def upgrade():
 
 
 def downgrade():
-    op.drop_column('portal', 'megagroup')
+    with op.batch_alter_table("portal") as batch_op:
+        batch_op.drop_column('megagroup')

alembic/versions/6ca3d74d51e4_move_state_store_to_main_database.py

@@ -0,0 +1,136 @@
+"""Move state store to main database
+
+Revision ID: 6ca3d74d51e4
+Revises: 2228d49c383f
+Create Date: 2018-06-26 21:31:26.911307
+
+"""
+from alembic import context, op
+import sqlalchemy.orm as orm
+import sqlalchemy as sa
+import json
+import re
+
+from mautrix_telegram.config import Config
+from mautrix_telegram.db import Base
+
+# revision identifiers, used by Alembic.
+revision = "6ca3d74d51e4"
+down_revision = "2228d49c383f"
+branch_labels = None
+depends_on = None
+
+
+class RoomState(Base):
+    query = None
+    __tablename__ = "mx_room_state"
+    __table_args__ = {"extend_existing": True}
+
+    room_id = sa.Column(sa.String, primary_key=True)
+    power_levels = sa.Column("power_levels", sa.Text, nullable=True)
+
+
+class UserProfile(Base):
+    query = None
+    __tablename__ = "mx_user_profile"
+    __table_args__ = {"extend_existing": True}
+
+    room_id = sa.Column(sa.String, primary_key=True)
+    user_id = sa.Column(sa.String, primary_key=True)
+    membership = sa.Column(sa.String, nullable=False, default="leave")
+    displayname = sa.Column(sa.String, nullable=True)
+    avatar_url = sa.Column(sa.String, nullable=True)
+
+
+class Puppet(Base):
+    query = None
+    __tablename__ = "puppet"
+    __table_args__ = {"extend_existing": True}
+
+    id = sa.Column(sa.Integer, primary_key=True)
+    displayname = sa.Column(sa.String, nullable=True)
+    displayname_source = sa.Column(sa.Integer, nullable=True)
+    username = sa.Column(sa.String, nullable=True)
+    photo_id = sa.Column(sa.String, nullable=True)
+    is_bot = sa.Column(sa.Boolean, nullable=True)
+    matrix_registered = sa.Column(sa.Boolean, nullable=False, default=False)
+
+
+def upgrade():
+    op.add_column("puppet", sa.Column("matrix_registered", sa.Boolean(), nullable=False,
+                                      server_default=sa.sql.expression.false()))
+    op.create_table("mx_room_state",
+                    sa.Column("room_id", sa.String(), nullable=False),
+                    sa.Column("power_levels", sa.Text(), nullable=True),
+                    sa.PrimaryKeyConstraint("room_id"))
+    op.create_table("mx_user_profile",
+                    sa.Column("room_id", sa.String(), nullable=False),
+                    sa.Column("user_id", sa.String(), nullable=False),
+                    sa.Column("membership", sa.String(), nullable=False,
+                              default="leave"),
+                    sa.Column("displayname", sa.String(), nullable=True),
+                    sa.Column("avatar_url", sa.String(), nullable=True),
+                    sa.PrimaryKeyConstraint("room_id", "user_id"))
+
+    try:
+        migrate_state_store()
+    except Exception as e:
+        print("Failed to migrate state store:", e)
+        print("Migrating the state store isn't required, but you can retry by alembic downgrading "
+              "to revision 2228d49c383f and upgrading again.")
+
+
+def migrate_state_store():
+    conn = op.get_bind()
+    session = orm.sessionmaker(bind=conn)()  # type: orm.Session
+
+    try:
+        with open("mx-state.json") as file:
+            data = json.load(file)
+    except FileNotFoundError:
+        return
+    if not data:
+        return
+    registrations = data.get("registrations", [])
+
+    mxtg_config_path = context.get_x_argument(as_dictionary=True).get("config", "config.yaml")
+    mxtg_config = Config(mxtg_config_path, None, None)
+    mxtg_config.load()
+
+    username_template = mxtg_config.get("bridge.username_template", "telegram_{userid}")
+    hs_domain = mxtg_config["homeserver.domain"]
+    localpart = username_template.format(userid="(.+)")
+    mxid_regex = re.compile("@{}:{}".format(localpart, hs_domain))
+    for user in registrations:
+        match = mxid_regex.match(user)
+        if not match:
+            continue
+
+        puppet = session.query(Puppet).get(match.group(1))
+        if not puppet:
+            continue
+
+        puppet.matrix_registered = True
+        session.merge(puppet)
+    session.commit()
+
+    user_profiles = [UserProfile(room_id=room, user_id=user,
+                                 membership=member.get("membership", "leave"),
+                                 displayname=member.get("displayname", None),
+                                 avatar_url=member.get("avatar_url", None))
+                     for room, members in data.get("members", {}).items()
+                     for user, member in members.items()]
+    session.add_all(user_profiles)
+    session.commit()
+
+    room_state = [RoomState(room_id=room, power_levels=json.dumps(levels))
+                  for room, levels in data.get("power_levels", {}).items()]
+    session.add_all(room_state)
+    session.commit()
+
+
+def downgrade():
+    op.drop_table("mx_user_profile")
+    op.drop_table("mx_room_state")
+    with op.batch_alter_table("puppet") as batch_op:
+        batch_op.drop_column("matrix_registered")

alembic/versions/7d47d84380b6_add_timestamp_to_telegramfile.py

@@ -17,9 +17,10 @@ depends_on = None
 
 def upgrade():
     op.add_column('telegram_file',
-                  sa.Column('timestamp', sa.BigInteger(), nullable=False, default=0,
+                  sa.Column('timestamp', sa.BigInteger(), nullable=True, default=0,
                             server_default="0"))
 
 
 def downgrade():
-    op.drop_column('telegram_file', 'timestamp')
+    with op.batch_alter_table("telegram_file") as batch_op:
+        batch_op.drop_column('timestamp')

alembic/versions/a9119be92164_add_phone_number_field_to_users.py

@@ -0,0 +1,25 @@
+"""Add phone number field to users
+
+Revision ID: a9119be92164
+Revises: b54929c22c86
+Create Date: 2018-09-28 02:38:40.626282
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = "a9119be92164"
+down_revision = "b54929c22c86"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.add_column("user", sa.Column("tg_phone", sa.String(), nullable=True))
+
+
+def downgrade():
+    with op.batch_alter_table("user") as batch_op:
+        batch_op.drop_column("tg_phone")

alembic/versions/b54929c22c86_add_portal_specific_config.py

@@ -0,0 +1,25 @@
+"""Add portal-specific config
+
+Revision ID: b54929c22c86
+Revises: d5f7b8b4b456
+Create Date: 2018-09-24 23:40:33.528710
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = "b54929c22c86"
+down_revision = "d5f7b8b4b456"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.add_column("portal", sa.Column("config", sa.Text(), nullable=True))
+
+
+def downgrade():
+    with op.batch_alter_table("portal") as batch_op:
+        batch_op.drop_column("config")

alembic/versions/bcfefa1f1299_add_displayname_source_fields_for_.py

@@ -20,4 +20,5 @@ def upgrade():
 
 
 def downgrade():
-    op.drop_column('puppet', 'displayname_source')
+    with op.batch_alter_table("puppet") as batch_op:
+        batch_op.drop_column('displayname_source')

alembic/versions/d5f7b8b4b456_add_access_token_and_custom_mxid_fields_.py

@@ -0,0 +1,26 @@
+"""Add access_token and custom_mxid fields for puppets
+
+Revision ID: d5f7b8b4b456
+Revises: 6ca3d74d51e4
+Create Date: 2018-07-20 12:09:30.277960
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = "d5f7b8b4b456"
+down_revision = "6ca3d74d51e4"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.add_column("puppet", sa.Column("access_token", sa.String(), nullable=True))
+    op.add_column("puppet", sa.Column("custom_mxid", sa.String(), nullable=True))
+
+
+def downgrade():
+    with op.batch_alter_table("puppet") as batch_op:
+        batch_op.drop_column("custom_mxid")
+        batch_op.drop_column("access_token")

docker-run.sh

@@ -1,22 +1,22 @@
-#!/bin/bash
+#!/bin/sh
 
-# Define functions
+# Define functions.
 function fixperms {
-	chown -R ${UID}:${GID} /data /opt/mautrixtelegram
+	chown -R $UID:$GID /data /opt/mautrix-telegram
 }
 
-
+cd /opt/mautrix-telegram
-# Go into env
-cd /opt/mautrixtelegram
-export FFMPEG_BINARY=/usr/bin/ffmpeg
 
 # Replace database path in config.
 sed -i "s#sqlite:///mautrix-telegram.db#sqlite:////data/mautrix-telegram.db#" /data/config.yaml
 
+if [ -f /data/mx-state.json ]; then
+	ln -s /data/mx-state.json
+fi
 # Check that database is in the right state
 alembic -x config=/data/config.yaml upgrade head
 
-if [[ ! -f /data/config.yaml ]]; then
+if [ ! -f /data/config.yaml ]; then
 	cp example-config.yaml /data/config.yaml
 	echo "Didn't find a config file."
 	echo "Copied default config file to /data/config.yaml"
@@ -26,14 +26,14 @@ if [[ ! -f /data/config.yaml ]]; then
 	exit
 fi
 
-if [[ ! -f /data/registration.yaml ]]; then
+if [ ! -f /data/registration.yaml ]; then
 	python3 -m mautrix_telegram -g -c /data/config.yaml -r /data/registration.yaml
 	echo "Didn't find a registration file."
-	echo "Generated ode for you."
+	echo "Generated one for you."
 	echo "Copy that over to synapses app service directory."
 	fixperms
 	exit
 fi
 
 fixperms
-exec su-exec ${UID}:${GID} python3 -m mautrix_telegram -c /data/config.yaml
+exec su-exec $UID:$GID python3 -m mautrix_telegram -c /data/config.yaml

docker/root/etc/s6.d/.s6-svscan/finish

@@ -1 +0,0 @@
-#!/bin/sh

docker/root/etc/s6.d/mautrix-telegram/finish

@@ -1,2 +0,0 @@
-#!/bin/bash
-s6-svscanctl -t /etc/s6.d

example-config.yaml

@@ -11,15 +11,21 @@ homeserver:
 # Application service host/registration related details
 # Changing these values requires regeneration of the registration.
 appservice:
-    # The protocol the homeserver should use when connecting to this appservice.
+    # The address that the homeserver can use to connect to this appservice.
-    # Usually "http" or "https".
+    address: http://localhost:8080
-    protocol: http
 
-    # The hostname and port where the homeserver can find this appservice.
+    # The hostname and port where this appservice should listen.
-    hostname: localhost
+    hostname: 0.0.0.0
     port: 8080
+    # The maximum body size of appservice API requests (from the homeserver) in mebibytes
+    # Usually 1 is enough, but on high-traffic bridges you might need to increase this to avoid 413s
+    max_body_size: 1
 
-    # The full URI to the database.
+    # The full URI to the database. SQLite and Postgres are fully supported.
+    # Other DBMSes supported by SQLAlchemy may or may not work.
+    # Format examples:
+    #   SQLite:   sqlite:///filename.db
+    #   Postgres: postgres://username:password@hostname/dbname
     database: sqlite:///mautrix-telegram.db
 
     # Public part of web server for out-of-Matrix interaction with the bridge.
@@ -34,14 +40,25 @@ appservice:
         # implicitly.
         external: https://example.com/public
 
-    # Whether or not to enable debug messages in the console.
+    # Provisioning API part of the web server for automated portal creation and fetching information.
-    debug: true
+    # Used by things like Dimension (https://dimension.t2bot.io/).
+    provisioning:
+        # Whether or not the provisioning API should be enabled.
+        enabled: true
+        # The prefix to use in the provisioning API endpoints.
+        prefix: /_matrix/provision/v1
+        # The shared secret to authorize users of the API.
+        # Set to "generate" to generate and save a new token.
+        shared_secret: generate
 
     # The unique ID of this appservice.
     id: telegram
     # Username of the appservice bot.
     bot_username: telegrambot
+    # Display name and avatar for bot. Set to "remove" to remove display name/avatar, leave empty
+    # to leave display name/avatar as-is.
     bot_displayname: Telegram bridge bot
+    bot_avatar: mxc://maunium.net/tJCRmUyJDsgRNgqhOgoiHWbX
 
     # Authentication tokens for AS <-> HS communication. Autogenerated; do not modify.
     as_token: "This value is generated when generating the registration"
@@ -78,48 +95,118 @@ bridge:
     - username
     - phone number
 
-    # Show message editing as a reply to the original message.
-    # If this is false, message edits are not shown at all, as Matrix does not support editing yet.
-    edits_as_replies: false
-    # Highlight changed/added parts in edits. Requires lxml.
-    highlight_edits: false
-    # Whether or not Matrix bot messages (type m.notice) should be bridged.
-    bridge_notices: true
-    # Whether to bridge Telegram bot messages as m.notices or m.texts.
-    bot_messages_as_notices: true
     # Maximum number of members to sync per portal when starting up. Other members will be
     # synced when they send messages. The maximum is 10000, after which the Telegram server
     # will not send any more members.
     # Defaults to no local limit (-> limited to 10000 by server)
     max_initial_member_sync: -1
+    # Whether or not to sync the member list in channels.
+    # If no channel admins have logged into the bridge, the bridge won't be able to sync the member
+    # list regardless of this setting.
+    sync_channel_members: true
+    # Whether or not to skip deleted members when syncing members.
+    skip_deleted_members: true
+    # Whether or not to automatically synchronize contacts and chats of Matrix users logged into
+    # their Telegram account at startup.
+    startup_sync: true
+    # Number of most recently active dialogs to check when syncing chats.
+    # Dialogs include groups and private chats, but only groups are synced.
+    # Set to 0 to remove limit.
+    sync_dialog_limit: 30
     # The maximum number of simultaneous Telegram deletions to handle.
     # A large number of simultaneous redactions could put strain on your homeserver.
     max_telegram_delete: 10
+    # Whether or not to automatically sync the Matrix room state (mostly unpuppeted displaynames)
+    # at startup and when creating a bridge.
+    sync_matrix_state: true
     # Allow logging in within Matrix. If false, the only way to log in is using the out-of-Matrix
     # login website (see appservice.public config section)
     allow_matrix_login: true
-    # Use inline images instead of m.image to make rich captions possible.
-    # N.B. Inline images are not supported on all clients (e.g. Riot iOS).
-    inline_images: false
     # Whether or not to bridge plaintext highlights.
     # Only enable this if your displayname_template has some static part that the bridge can use to
     # reliably identify what is a plaintext highlight.
     plaintext_highlights: false
+    # Show message editing as a reply to the original message.
+    # If this is false, message edits are not shown at all, as Matrix does not support editing yet.
+    edits_as_replies: true
+    # Highlight changed/added parts in edits. Requires lxml.
+    highlight_edits: false
     # Whether or not to make portals of publicly joinable channels/supergroups publicly joinable on Matrix.
     public_portals: true
-    # Whether to send stickers as the new native m.sticker type or normal m.images.
-    # Old versions of Riot don't support the new type at all.
-    # Remember that proper sticker support always requires Pillow to convert webp into png.
-    native_stickers: true
     # Whether or not to fetch and handle Telegram updates at startup from the time the bridge was down.
-    # WARNING: Probably buggy, might get stuck in infinite loop.
+    # Currently only works for private chats and normal groups.
     catch_up: false
+    # Whether or not to use /sync to get presence, read receipts and typing notifications when using
+    # your own Matrix account as the Matrix puppet for your Telegram account.
+    sync_with_custom_puppets: true
+    # Set to false to disable link previews in messages sent to Telegram.
+    telegram_link_preview: true
+    # Use inline images instead of a separate message for the caption.
+    # N.B. Inline images are not supported on all clients (e.g. Riot iOS).
+    inline_images: false
+    # Maximum size of image in megabytes before sending to Telegram as a document.
+    image_as_file_size: 10
 
+    # Whether to bridge Telegram bot messages as m.notices or m.texts.
+    bot_messages_as_notices: true
+    bridge_notices:
+        # Whether or not Matrix bot messages (type m.notice) should be bridged.
+        default: false
+        # List of user IDs for whom the previous flag is flipped.
+        # e.g. if bridge_notices.default is false, notices from other users will not be bridged, but
+        #      notices from users listed here will be bridged.
+        exceptions:
+        - "@importantbot:example.com"
+
+    # Some config options related to Telegram message deduplication.
+    # The default values are usually fine, but some debug messages/warnings might recommend you
+    # change these.
+    deduplication:
+        # Whether or not to check the database if the message about to be sent is a duplicate.
+        pre_db_check: false
+        # The number of latest events to keep when checking for duplicates.
+        # You might need to increase this on high-traffic bridge instances.
+        cache_queue_length: 20
+
+
+    # The formats to use when sending messages to Telegram via the relay bot.
+    #
+    # Telegram doesn't have built-in emotes, so the m.emote format is also used for non-relaybot users.
+    #
+    # Available variables:
+    #   $sender_displayname    - The display name of the sender (e.g. Example User)
+    #   $sender_username       - The username (Matrix ID localpart) of the sender (e.g. exampleuser)
+    #   $sender_mxid           - The Matrix ID of the sender (e.g. @exampleuser:example.com)
+    #   $message               - The message content as HTML
+    message_formats:
+        m.text: "<b>$sender_displayname</b>: $message"
+        m.emote: "* <b>$sender_displayname</b> $message"
+        m.file: "<b>$sender_displayname</b> sent a file: $message"
+        m.image: "<b>$sender_displayname</b> sent an image: $message"
+        m.audio: "<b>$sender_displayname</b> sent an audio file: $message"
+        m.video: "<b>$sender_displayname</b> sent a video: $message"
+        m.location: "<b>$sender_displayname</b> sent a location: $message"
+
+    # The formats to use when sending state events to Telegram via the relay bot.
+    #
+    # Variables from `message_formats` that have the `sender_` prefix are available without the prefix.
+    # In name_change events, `$prev_displayname` is the previous displayname.
+    #
+    # Set format to an empty string to disable the messages for that event.
+    state_event_formats:
+        join: "<b>$displayname</b> joined the room."
+        leave: "<b>$displayname</b> left the room."
+        name_change: "<b>$prev_displayname</b> changed their name to <b>$displayname</b>"
+
+    # Filter rooms that can/can't be bridged. Can also be managed using the `filter` and
+    # `filter-mode` management commands.
+    #
+    # Filters do not affect direct chats.
+    # An empty blacklist will essentially disable the filter.
     filter:
         # Filter mode to use. Either "blacklist" or "whitelist".
-        # If the mode is "blacklist", the listed chats will never be bridged. An empty blacklist disables the filter.
+        # If the mode is "blacklist", the listed chats will never be bridged.
         # If the mode is "whitelist", only the listed chats can be bridged.
-        # Direct chats are not affected.
         mode: blacklist
         # The list of group/channel IDs to filter.
         list: []
@@ -130,7 +217,9 @@ bridge:
     # Permissions for using the bridge.
     # Permitted values:
     #   relaybot - Only use the bridge via the relaybot, no access to commands.
-    #       full - Full access to use the bridge via relaybot or logging in with Telegram account.
+    #       user - Relaybot level + access to commands to create bridges.
+    #  puppeting - User level + logging in with a Telegram account.
+    #       full - Full access to use the bridge, i.e. previous levels + Matrix login.
     #      admin - Full access to use the bridge and some extra administration commands.
     # Permitted keys:
     #        * - All Matrix users
@@ -138,8 +227,8 @@ bridge:
     #     mxid - Specific user
     permissions:
         "*": "relaybot"
+        "public.example.com": "user"
         "example.com": "full"
-        "public.example.com": "full"
         "@admin:example.com": "admin"
 
     # Options related to the message relay Telegram bot.
@@ -148,6 +237,8 @@ bridge:
         authless_portals: true
         # Whether or not to allow Telegram group admins to use the bot commands.
         whitelist_group_admins: true
+        # Whether or not to ignore incoming events sent by the relay bot.
+        ignore_own_incoming_events: true
         # List of usernames/user IDs who are also allowed to use the bot commands.
         whitelist:
         - myusername
@@ -160,3 +251,57 @@ telegram:
     api_hash: tjyd5yge35lbodk1xwzw2jstp90k55qz
     # (Optional) Create your own bot at https://t.me/BotFather
     bot_token: disabled
+    # Custom server to connect to.
+    server:
+        # Set to true to use these server settings. If false, will automatically
+        # use production server assigned by Telegram. Set to false in production.
+        enabled: false
+        # The DC ID to connect to.
+        dc: 2
+        # The IP to connect to.
+        ip: 149.154.167.40
+        # The port to connect to. 443 may not work, 80 is better and both are equally secure.
+        port: 80
+    # Telethon proxy configuration.
+    # You must install PySocks from pip for proxies to work.
+    proxy:
+        # Allowed types: disabled, socks4, socks5, http
+        type: disabled
+        # Proxy IP address and port.
+        address: 127.0.0.1
+        port: 1080
+        # Whether or not to perform DNS resolving remotely.
+        rdns: true
+        # Proxy authentication (optional).
+        username: ""
+        password: ""
+
+# Python logging configuration.
+#
+# See section 16.7.2 of the Python documentation for more info:
+# https://docs.python.org/3.6/library/logging.config.html#configuration-dictionary-schema
+logging:
+    version: 1
+    formatters:
+        precise:
+            format: "[%(asctime)s] [%(levelname)s@%(name)s] %(message)s"
+    handlers:
+        file:
+            class: logging.handlers.RotatingFileHandler
+            formatter: precise
+            filename: ./mautrix-telegram.log
+            maxBytes: 10485760
+            backupCount: 10
+        console:
+            class: logging.StreamHandler
+            formatter: precise
+    loggers:
+        mau:
+            level: DEBUG
+        telethon:
+            level: DEBUG
+        aiohttp:
+            level: INFO
+    root:
+        level: DEBUG
+        handlers: [file, console]

mautrix_telegram/__init__.py

@@ -1,2 +1,2 @@
-__version__ = "0.2.0"
+__version__ = "0.5.0"
 __author__ = "Tulir Asokan <tulir@maunium.net>"

mautrix_telegram/__main__.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -14,36 +14,34 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Awaitable, List, Any
+from time import time
 import argparse
-import sys
-import logging
 import asyncio
+import logging.config
+import sys
+import copy
+import signal
 
 import sqlalchemy as sql
-from sqlalchemy import orm
 
-from alchemysession import AlchemySessionContainer
 from mautrix_appservice import AppService
+from alchemysession import AlchemySessionContainer
 
-from .base import Base
+from .web.provisioning import ProvisioningAPI
-from .config import Config
+from .web.public import PublicBridgeWebsite
-from .matrix import MatrixHandler
-
-from .db import init as init_db
 from .abstract_user import init as init_abstract_user
-from .user import init as init_user, User
 from .bot import init as init_bot
+from .config import Config
+from .context import Context
+from .db import Base, init as init_db
+from .formatter import init as init_formatter
+from .matrix import MatrixHandler
 from .portal import init as init_portal
 from .puppet import init as init_puppet
-from .formatter import init as init_formatter
+from .sqlstatestore import SQLStateStore
-from .public import PublicBridgeWebsite
+from .user import User, init as init_user
-from .context import Context
+from . import __version__
-
-log = logging.getLogger("mau")
-time_formatter = logging.Formatter("[%(asctime)s] [%(levelname)s@%(name)s] %(message)s")
-handler = logging.StreamHandler()
-handler.setFormatter(time_formatter)
-log.addHandler(handler)
 
 parser = argparse.ArgumentParser(
     description="A Matrix-Telegram puppeting bridge.",
@@ -69,52 +67,85 @@ if args.generate_registration:
     print(f"Registration generated and saved to {config.registration_path}")
     sys.exit(0)
 
-if config["appservice.debug"]:
+logging.config.dictConfig(copy.deepcopy(config["logging"]))
-    telethon_log = logging.getLogger("telethon")
+log = logging.getLogger("mau.init")  # type: logging.Logger
-    telethon_log.addHandler(handler)
+log.debug(f"Initializing mautrix-telegram {__version__}")
-    telethon_log.setLevel(logging.DEBUG)
-    log.setLevel(logging.DEBUG)
-    log.debug("Debug messages enabled.")
 
-db_engine = sql.create_engine(config.get("appservice.database", "sqlite:///mautrix-telegram.db"))
+db_engine = sql.create_engine(config["appservice.database"] or "sqlite:///mautrix-telegram.db")
-db_factory = orm.sessionmaker(bind=db_engine)
-db_session = orm.scoping.scoped_session(db_factory)
 Base.metadata.bind = db_engine
 
-telethon_session_container = AlchemySessionContainer(engine=db_engine, session=db_session,
+session_container = AlchemySessionContainer(engine=db_engine, table_base=Base, session=False,
-                                                     table_base=Base, table_prefix="telethon_",
+                                            table_prefix="telethon_", manage_tables=False)
-                                                     manage_tables=False)
+session_container.core_mode = True
 
-loop = asyncio.get_event_loop()
+try:
+    import uvloop
 
+    asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
+    log.debug("Using uvloop for asyncio")
+except ImportError:
+    pass
+
+loop = asyncio.get_event_loop()  # type: asyncio.AbstractEventLoop
+
+state_store = SQLStateStore()
+mebibyte = 1024 ** 2
 appserv = AppService(config["homeserver.address"], config["homeserver.domain"],
                      config["appservice.as_token"], config["appservice.hs_token"],
                      config["appservice.bot_username"], log="mau.as", loop=loop,
-                     verify_ssl=config["homeserver.verify_ssl"])
+                     verify_ssl=config["homeserver.verify_ssl"], state_store=state_store,
-
+                     real_user_content_key="net.maunium.telegram.puppet",
-context = Context(appserv, db_session, config, loop, None, None, telethon_session_container)
+                     aiohttp_params={
+                         "client_max_size": config["appservice.max_body_size"] * mebibyte
+                     })
+bot = init_bot(config)
+context = Context(appserv, config, loop, session_container, bot)
 
 if config["appservice.public.enabled"]:
-    public = PublicBridgeWebsite(loop)
+    public_website = PublicBridgeWebsite(loop)
-    appserv.app.add_subapp(config.get("appservice.public.prefix", "/public"), public.app)
+    appserv.app.add_subapp(config["appservice.public.prefix"] or "/public", public_website.app)
+    context.public_website = public_website
+
+if config["appservice.provisioning.enabled"]:
+    provisioning_api = ProvisioningAPI(context)
+    appserv.app.add_subapp(config["appservice.provisioning.prefix"] or "/_matrix/provisioning",
+                           provisioning_api.app)
+    context.provisioning_api = provisioning_api
+
+context.mx = MatrixHandler(context)
 
 with appserv.run(config["appservice.hostname"], config["appservice.port"]) as start:
-    init_db(db_session)
+    start_ts = time()
+    init_db(db_engine)
     init_abstract_user(context)
-    context.bot = init_bot(context)
-    context.mx = MatrixHandler(context)
     init_formatter(context)
     init_portal(context)
-    init_puppet(context)
+    startup_actions = (init_puppet(context) +
-    startup_actions = init_user(context) + [start, context.mx.init_as_bot()]
+                       init_user(context) +
+                       [start, context.mx.init_as_bot()])  # type: List[Awaitable[Any]]
 
     if context.bot:
         startup_actions.append(context.bot.start())
 
+    signal.signal(signal.SIGINT, signal.default_int_handler)
+    signal.signal(signal.SIGTERM, signal.default_int_handler)
+
+    end_ts = time()
     try:
+        log.debug(f"Initialization complete in {round(end_ts - start_ts, 2)} seconds,"
+                  " running startup actions")
+        start_ts = time()
         loop.run_until_complete(asyncio.gather(*startup_actions, loop=loop))
+        end_ts = time()
+        log.debug(f"Startup actions complete in {round(end_ts - start_ts, 2)} seconds,"
+                  " now running forever")
         loop.run_forever()
     except KeyboardInterrupt:
-        for user in User.by_tgid.values():
+        log.debug("Interrupt received, stopping clients")
-            user.stop()
+        loop.run_until_complete(
+            asyncio.gather(*[user.stop() for user in User.by_tgid.values()], loop=loop))
+        log.debug("Clients stopped, shutting down")
         sys.exit(0)
+    except Exception as e:
+        log.exception("Unexpected error")
+        sys.exit(1)

mautrix_telegram/abstract_user.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -14,41 +14,98 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Tuple, Optional, List, Union, TYPE_CHECKING
+from abc import ABC, abstractmethod
+import asyncio
+import logging
 import platform
-import os
 
-from telethon.tl.types import *
+from telethon.tl.patched import MessageService, Message
-from mautrix_appservice import MatrixRequestError
+from telethon.tl.types import (
+    Channel, ChannelForbidden, Chat, ChatForbidden, MessageActionChannelMigrateFrom, PeerUser,
+    TypeUpdate, UpdateChannelPinnedMessage, UpdateChatPinnedMessage, UpdateChatParticipantAdmin,
+    UpdateChatParticipants, UpdateChatUserTyping, UpdateDeleteChannelMessages, UpdateDeleteMessages,
+    UpdateEditChannelMessage, UpdateEditMessage, UpdateNewChannelMessage, UpdateNewMessage,
+    UpdateReadHistoryOutbox, UpdateShortChatMessage, UpdateShortMessage, UpdateUserName,
+    UpdateUserPhoto, UpdateUserStatus, UpdateUserTyping, User, UserStatusOffline, UserStatusOnline)
+
+from mautrix_appservice import MatrixRequestError, AppService
+from alchemysession import AlchemySessionContainer
 
-from .tgclient import MautrixTelegramClient
-from .db import Message as DBMessage
 from . import portal as po, puppet as pu, __version__
+from .db import Message as DBMessage
+from .types import TelegramID, MatrixUserID
+from .tgclient import MautrixTelegramClient
 
-config = None
+if TYPE_CHECKING:
+    from .context import Context
+    from .config import Config
+    from .bot import Bot
+
+config = None  # type: Config
 # Value updated from config in init()
-MAX_DELETIONS = 10
+MAX_DELETIONS = 10  # type: int
+
+UpdateMessage = Union[UpdateShortChatMessage, UpdateShortMessage, UpdateNewChannelMessage,
+                      UpdateNewMessage, UpdateEditMessage, UpdateEditChannelMessage]
+UpdateMessageContent = Union[UpdateShortMessage, UpdateShortChatMessage, Message, MessageService]
 
 
-class AbstractUser:
+class AbstractUser(ABC):
-    session_container = None
+    session_container = None  # type: AlchemySessionContainer
-    loop = None
+    loop = None  # type: asyncio.AbstractEventLoop
-    log = None
+    log = None  # type: logging.Logger
-    db = None
+    az = None  # type: AppService
-    az = None
+    bot = None  # type: Bot
+    ignore_incoming_bot_events = True  # type: bool
 
-    def __init__(self):
+    def __init__(self) -> None:
-        self.connected = False
+        self.is_admin = False  # type: bool
-        self.whitelisted = False
+        self.matrix_puppet_whitelisted = False  # type: bool
-        self.client = None
+        self.puppet_whitelisted = False  # type: bool
-        self.tgid = None
+        self.whitelisted = False  # type: bool
-        self.mxid = None
+        self.relaybot_whitelisted = False  # type: bool
-        self.is_relaybot = False
+        self.client = None  # type: MautrixTelegramClient
+        self.tgid = None  # type: TelegramID
+        self.mxid = None  # type: MatrixUserID
+        self.is_relaybot = False  # type: bool
+        self.is_bot = False  # type: bool
+        self.relaybot = None  # type: Optional[Bot]
 
-    async def _init_client(self):
+    @property
+    def connected(self) -> bool:
+        return self.client and self.client.is_connected()
+
+    @property
+    def _proxy_settings(self) -> Optional[Tuple[int, str, str, str, str, str]]:
+        proxy_type = config["telegram.proxy.type"].lower()
+        if proxy_type == "disabled":
+            return None
+        elif proxy_type == "socks4":
+            proxy_type = 1
+        elif proxy_type == "socks5":
+            proxy_type = 2
+        elif proxy_type == "http":
+            proxy_type = 3
+
+        return (proxy_type,
+                config["telegram.proxy.address"], config["telegram.proxy.port"],
+                config["telegram.proxy.rdns"],
+                config["telegram.proxy.username"], config["telegram.proxy.password"])
+
+    def _init_client(self) -> None:
         self.log.debug(f"Initializing client for {self.name}")
         device = f"{platform.system()} {platform.release()}"
         sysversion = MautrixTelegramClient.__version__
         self.session = self.session_container.new_session(self.name)
+        if config["telegram.server.enabled"]:
+            self.session.set_dc(config["telegram.server.dc"],
+                                config["telegram.server.ip"],
+                                config["telegram.server.port"])
+        if self.is_relaybot:
+            base_logger = logging.getLogger("telethon.relaybot")
+        else:
+            base_logger = logging.getLogger(f"telethon.{self.tgid or -hash(self.mxid)}")
         self.client = MautrixTelegramClient(session=self.session,
                                             api_id=config["telegram.api_id"],
                                             api_hash=config["telegram.api_hash"],
@@ -56,23 +113,37 @@ class AbstractUser:
                                             app_version=__version__,
                                             system_version=sysversion,
                                             device_model=device,
-                                            report_errors=False)
+                                            timeout=120,
-        await self.client.add_event_handler(self._update_catch)
+                                            base_logger=base_logger,
+                                            proxy=self._proxy_settings)
+        self.client.add_event_handler(self._update_catch)
 
-    async def update(self, update):
+    @abstractmethod
+    async def update(self, update: TypeUpdate) -> bool:
         return False
 
-    async def post_login(self):
+    @abstractmethod
+    async def post_login(self) -> None:
         raise NotImplementedError()
 
-    async def _update_catch(self, update):
+    @abstractmethod
+    def register_portal(self, portal: po.Portal) -> None:
+        raise NotImplementedError()
+
+    @abstractmethod
+    def unregister_portal(self, portal: po.Portal) -> None:
+        raise NotImplementedError()
+
+    async def _update_catch(self, update: TypeUpdate) -> None:
         try:
             if not await self.update(update):
                 await self._update(update)
         except Exception:
             self.log.exception("Failed to handle Telegram update")
 
-    async def _get_dialogs(self, limit=None):
+    async def get_dialogs(self, limit: int = None) -> List[Union[Chat, Channel]]:
+        if self.is_bot:
+            return []
         dialogs = await self.client.get_dialogs(limit=limit)
         return [dialog.entity for dialog in dialogs if (
             not isinstance(dialog.entity, (User, ChatForbidden, ChannelForbidden))
@@ -80,37 +151,40 @@ class AbstractUser:
                      and (dialog.entity.deactivated or dialog.entity.left)))]
 
     @property
-    def name(self):
+    @abstractmethod
+    def name(self) -> str:
         raise NotImplementedError()
 
-    @property
+    async def is_logged_in(self) -> bool:
-    def logged_in(self):
+        return self.client and self.client.is_connected() and await self.client.is_user_authorized()
-        return self.client and self.client.is_user_authorized()
 
-    @property
+    async def has_full_access(self, allow_bot: bool = False) -> bool:
-    def has_full_access(self):
+        return (self.puppet_whitelisted
-        return self.logged_in and self.whitelisted
+                and (not self.is_bot or allow_bot)
+                and await self.is_logged_in())
 
-    async def start(self):
+    async def start(self, delete_unless_authenticated: bool = False) -> 'AbstractUser':
         if not self.client:
-            await self._init_client()
+            self._init_client()
-        self.connected = await self.client.connect()
+        await self.client.connect()
-
+        self.log.debug("%s connected: %s", self.mxid, self.connected)
-    async def ensure_started(self, even_if_no_session=False):
-        if not self.whitelisted:
-            return self
-        elif not self.connected and (even_if_no_session or os.path.exists(f"{self.name}.session")):
-            return await self.start()
         return self
 
-    def stop(self):
+    async def ensure_started(self, even_if_no_session=False) -> 'AbstractUser':
-        self.client.disconnect()
+        if not self.puppet_whitelisted or self.connected:
+            return self
+        self.log.debug("ensure_started(%s, even_if_no_session=%s)", self.mxid, even_if_no_session)
+        if even_if_no_session or self.session_container.has_session(self.mxid):
+            await self.start(delete_unless_authenticated=not even_if_no_session)
+        return self
+
+    async def stop(self) -> None:
+        await self.client.disconnect()
         self.client = None
-        self.connected = False
 
     # region Telegram update handling
 
-    async def _update(self, update):
+    async def _update(self, update: TypeUpdate) -> None:
         if isinstance(update, (UpdateShortChatMessage, UpdateShortMessage, UpdateNewChannelMessage,
                                UpdateNewMessage, UpdateEditMessage, UpdateEditChannelMessage)):
             await self.update_message(update)
@@ -122,11 +196,11 @@ class AbstractUser:
             await self.update_typing(update)
         elif isinstance(update, UpdateUserStatus):
             await self.update_status(update)
-        elif isinstance(update, (UpdateChatAdmins, UpdateChatParticipantAdmin)):
+        elif isinstance(update, UpdateChatParticipantAdmin):
             await self.update_admin(update)
         elif isinstance(update, UpdateChatParticipants):
             await self.update_participants(update)
-        elif isinstance(update, UpdateChannelPinnedMessage):
+        elif isinstance(update, (UpdateChannelPinnedMessage, UpdateChatPinnedMessage)):
             await self.update_pinned_messages(update)
         elif isinstance(update, (UpdateUserName, UpdateUserPhoto)):
             await self.update_others_info(update)
@@ -135,55 +209,63 @@ class AbstractUser:
         else:
             self.log.debug("Unhandled update: %s", update)
 
-    async def update_pinned_messages(self, update):
+    async def update_pinned_messages(self, update: Union[UpdateChannelPinnedMessage,
-        portal = po.Portal.get_by_tgid(update.channel_id)
+                                                         UpdateChatPinnedMessage]) -> None:
+        if isinstance(update, UpdateChatPinnedMessage):
+            portal = po.Portal.get_by_tgid(TelegramID(update.chat_id))
+        else:
+            portal = po.Portal.get_by_tgid(TelegramID(update.channel_id))
         if portal and portal.mxid:
-            await portal.receive_telegram_pin_id(update.id)
+            await portal.receive_telegram_pin_id(update.id, self.tgid)
 
-    async def update_participants(self, update):
+    @staticmethod
-        portal = po.Portal.get_by_tgid(update.participants.chat_id)
+    async def update_participants(update: UpdateChatParticipants) -> None:
+        portal = po.Portal.get_by_tgid(TelegramID(update.participants.chat_id))
         if portal and portal.mxid:
             await portal.update_telegram_participants(update.participants.participants)
 
-    async def update_read_receipt(self, update):
+    async def update_read_receipt(self, update: UpdateReadHistoryOutbox) -> None:
         if not isinstance(update.peer, PeerUser):
             self.log.debug("Unexpected read receipt peer: %s", update.peer)
             return
 
-        portal = po.Portal.get_by_tgid(update.peer.user_id, self.tgid)
+        portal = po.Portal.get_by_tgid(TelegramID(update.peer.user_id), self.tgid)
         if not portal or not portal.mxid:
             return
 
         # We check that these are user read receipts, so tg_space is always the user ID.
-        message = DBMessage.query.get((update.max_id, self.tgid))
+        message = DBMessage.get_by_tgid(TelegramID(update.max_id), self.tgid)
         if not message:
             return
 
-        puppet = pu.Puppet.get(update.peer.user_id)
+        puppet = pu.Puppet.get(TelegramID(update.peer.user_id))
         await puppet.intent.mark_read(portal.mxid, message.mxid)
 
-    async def update_admin(self, update):
+    async def update_admin(self, update: UpdateChatParticipantAdmin) -> None:
         # TODO duplication not checked
-        portal = po.Portal.get_by_tgid(update.chat_id, peer_type="chat")
+        portal = po.Portal.get_by_tgid(TelegramID(update.chat_id), peer_type="chat")
-        if isinstance(update, UpdateChatAdmins):
+        if not portal or not portal.mxid:
-            await portal.set_telegram_admins_enabled(update.enabled)
+            return
-        elif isinstance(update, UpdateChatParticipantAdmin):
-            await portal.set_telegram_admin(update.user_id)
-        else:
-            self.log.warning("Unexpected admin status update: %s", update)
 
-    async def update_typing(self, update):
+        await portal.set_telegram_admin(TelegramID(update.user_id))
+
+    async def update_typing(self, update: Union[UpdateUserTyping, UpdateChatUserTyping]) -> None:
         if isinstance(update, UpdateUserTyping):
-            portal = po.Portal.get_by_tgid(update.user_id, self.tgid, "user")
+            portal = po.Portal.get_by_tgid(TelegramID(update.user_id), self.tgid, "user")
         else:
-            portal = po.Portal.get_by_tgid(update.chat_id, peer_type="chat")
+            portal = po.Portal.get_by_tgid(TelegramID(update.chat_id), peer_type="chat")
-        sender = pu.Puppet.get(update.user_id)
+
+        if not portal or not portal.mxid:
+            return
+
+        sender = pu.Puppet.get(TelegramID(update.user_id))
         await portal.handle_telegram_typing(sender, update)
 
-    async def update_others_info(self, update):
+    async def update_others_info(self, update: Union[UpdateUserName, UpdateUserPhoto]) -> None:
         # TODO duplication not checked
-        puppet = pu.Puppet.get(update.user_id)
+        puppet = pu.Puppet.get(TelegramID(update.user_id))
         if isinstance(update, UpdateUserName):
+            puppet.username = update.username
             if await puppet.update_displayname(self, update):
                 puppet.save()
         elif isinstance(update, UpdateUserPhoto):
@@ -192,22 +274,24 @@ class AbstractUser:
         else:
             self.log.warning("Unexpected other user info update: %s", update)
 
-    async def update_status(self, update):
+    async def update_status(self, update: UpdateUserStatus) -> None:
-        puppet = pu.Puppet.get(update.user_id)
+        puppet = pu.Puppet.get(TelegramID(update.user_id))
         if isinstance(update.status, UserStatusOnline):
-            await puppet.intent.set_presence("online")
+            await puppet.default_mxid_intent.set_presence("online")
         elif isinstance(update.status, UserStatusOffline):
-            await puppet.intent.set_presence("offline")
+            await puppet.default_mxid_intent.set_presence("offline")
         else:
             self.log.warning("Unexpected user status update: %s", update)
         return
 
-    def get_message_details(self, update):
+    def get_message_details(self, update: UpdateMessage) -> Tuple[UpdateMessageContent,
+                                                                  Optional[pu.Puppet],
+                                                                  Optional[po.Portal]]:
         if isinstance(update, UpdateShortChatMessage):
-            portal = po.Portal.get_by_tgid(update.chat_id, peer_type="chat")
+            portal = po.Portal.get_by_tgid(TelegramID(update.chat_id), peer_type="chat")
-            sender = pu.Puppet.get(update.from_id)
+            sender = pu.Puppet.get(TelegramID(update.from_id))
         elif isinstance(update, UpdateShortMessage):
-            portal = po.Portal.get_by_tgid(update.user_id, self.tgid, "user")
+            portal = po.Portal.get_by_tgid(TelegramID(update.user_id), self.tgid, "user")
             sender = pu.Puppet.get(self.tgid if update.out else update.user_id)
         elif isinstance(update, (UpdateNewMessage, UpdateNewChannelMessage,
                                  UpdateEditMessage, UpdateEditChannelMessage)):
@@ -225,7 +309,7 @@ class AbstractUser:
         return update, sender, portal
 
     @staticmethod
-    async def _try_redact(portal, message):
+    async def _try_redact(portal: po.Portal, message: DBMessage) -> None:
         if not portal:
             return
         try:
@@ -233,41 +317,47 @@ class AbstractUser:
         except MatrixRequestError:
             pass
 
-    async def delete_message(self, update):
+    async def delete_message(self, update: UpdateDeleteMessages) -> None:
         if len(update.messages) > MAX_DELETIONS:
             return
 
         for message in update.messages:
-            message = DBMessage.query.get((message, self.tgid))
+            message = DBMessage.get_by_tgid(TelegramID(message), self.tgid)
             if not message:
                 continue
-            self.db.delete(message)
+            message.delete()
-            number_left = DBMessage.query.filter(DBMessage.mxid == message.mxid,
+            number_left = DBMessage.count_spaces_by_mxid(message.mxid, message.mx_room)
-                                                 DBMessage.mx_room == message.mx_room).count()
             if number_left == 0:
                 portal = po.Portal.get_by_mxid(message.mx_room)
                 await self._try_redact(portal, message)
-        self.db.commit()
 
-    async def delete_channel_message(self, update):
+    async def delete_channel_message(self, update: UpdateDeleteChannelMessages) -> None:
         if len(update.messages) > MAX_DELETIONS:
             return
 
-        portal = po.Portal.get_by_tgid(update.channel_id)
+        portal = po.Portal.get_by_tgid(TelegramID(update.channel_id))
         if not portal:
             return
 
         for message in update.messages:
-            message = DBMessage.query.get((message, portal.tgid))
+            message = DBMessage.get_by_tgid(TelegramID(message), portal.tgid)
             if not message:
                 continue
-            self.db.delete(message)
+            message.delete()
             await self._try_redact(portal, message)
-        self.db.commit()
 
-    async def update_message(self, original_update):
+    async def update_message(self, original_update: UpdateMessage) -> None:
         update, sender, portal = self.get_message_details(original_update)
 
+        if self.is_bot and not portal.mxid:
+            self.log.debug(f"Ignoring message received by bot in unbridged chat %s",
+                           portal.tgid_log)
+            return
+
+        if self.ignore_incoming_bot_events and self.bot and sender.id == self.bot.tgid:
+            self.log.debug(f"Ignoring relaybot-sent message %s to %s", update, portal.tgid_log)
+            return
+
         if isinstance(update, MessageService):
             if isinstance(update.action, MessageActionChannelMigrateFrom):
                 self.log.debug(f"Ignoring action %s to %s by %d", update.action,
@@ -291,8 +381,9 @@ class AbstractUser:
     # endregion
 
 
-def init(context):
+def init(context: "Context") -> None:
     global config, MAX_DELETIONS
-    AbstractUser.az, AbstractUser.db, config, AbstractUser.loop, _ = context
+    AbstractUser.az, config, AbstractUser.loop, AbstractUser.relaybot = context.core
-    AbstractUser.session_container = context.telethon_session_container
+    AbstractUser.ignore_incoming_bot_events = config["bridge.relaybot.ignore_own_incoming_events"]
+    AbstractUser.session_container = context.session_container
     MAX_DELETIONS = config.get("bridge.max_telegram_delete", 10)

mautrix_telegram/base.py

@@ -1,2 +0,0 @@
-from sqlalchemy.ext.declarative import declarative_base
-Base = declarative_base()

mautrix_telegram/bot.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -14,78 +14,94 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Awaitable, Callable
+from typing import Awaitable, Callable, Dict, List, Optional, Pattern, TYPE_CHECKING
 import logging
 import re
 
-from telethon.tl.types import *
+from telethon.tl.patched import Message, MessageService
+from telethon.tl.types import (
+    ChannelParticipantAdmin, ChannelParticipantCreator, ChatForbidden, ChatParticipantAdmin,
+    ChatParticipantCreator, InputChannel, InputUser, MessageActionChatAddUser,
+    MessageActionChatDeleteUser, MessageEntityBotCommand, PeerChannel, PeerChat, TypePeer,
+    UpdateNewChannelMessage, UpdateNewMessage, MessageActionChatMigrateTo)
 from telethon.tl.functions.messages import GetChatsRequest, GetFullChatRequest
 from telethon.tl.functions.channels import GetChannelsRequest, GetParticipantRequest
 from telethon.errors import ChannelInvalidError, ChannelPrivateError
 
+from .types import MatrixUserID
 from .abstract_user import AbstractUser
 from .db import BotChat
+from .types import TelegramID
 from . import puppet as pu, portal as po, user as u
 
-config = None
+if TYPE_CHECKING:
+    from .config import Config
+    from .context import Context
+
+config = None  # type: Config
 
 ReplyFunc = Callable[[str], Awaitable[Message]]
 
 
 class Bot(AbstractUser):
-    log = logging.getLogger("mau.bot")
+    log = logging.getLogger("mau.bot")  # type: logging.Logger
-    mxid_regex = re.compile("@.+:.+")
+    mxid_regex = re.compile("@.+:.+")  # type: Pattern
 
-    def __init__(self, token: str):
+    def __init__(self, token: str) -> None:
         super().__init__()
-        self.token = token
+        self.token = token  # type: str
-        self.whitelisted = True
+        self.puppet_whitelisted = True  # type: bool
-        self.username = None
+        self.whitelisted = True  # type: bool
-        self.is_relaybot = True
+        self.relaybot_whitelisted = True  # type: bool
-        self.chats = {chat.id: chat.type for chat in BotChat.query.all()}
+        self.username = None  # type: str
-        self.tg_whitelist = []
+        self.is_relaybot = True  # type: bool
-        self.whitelist_group_admins = config["bridge.relaybot.whitelist_group_admins"] or False
+        self.is_bot = True  # type: bool
+        self.chats = {}  # type: Dict[int, str]
+        self.tg_whitelist = []  # type: List[int]
+        self.whitelist_group_admins = (config["bridge.relaybot.whitelist_group_admins"]
+                                       or False)  # type: bool
 
-    async def init_permissions(self):
+    async def init_permissions(self) -> None:
         whitelist = config["bridge.relaybot.whitelist"] or []
-        for id in whitelist:
+        for user_id in whitelist:
-            if isinstance(id, str):
+            if isinstance(user_id, str):
-                entity = await self.client.get_input_entity(id)
+                entity = await self.client.get_input_entity(user_id)
                 if isinstance(entity, InputUser):
-                    id = entity.user_id
+                    user_id = entity.user_id
                 else:
-                    id = None
+                    user_id = None
-            if isinstance(id, int):
+            if isinstance(user_id, int):
-                self.tg_whitelist.append(id)
+                self.tg_whitelist.append(user_id)
 
-    async def start(self):
+    async def start(self, delete_unless_authenticated: bool = False) -> 'Bot':
-        await super().start()
+        self.chats = {chat.id: chat.type for chat in BotChat.all()}
-        if not self.logged_in:
+        await super().start(delete_unless_authenticated)
+        if not await self.is_logged_in():
             await self.client.sign_in(bot_token=self.token)
         await self.post_login()
         return self
 
-    async def post_login(self):
+    async def post_login(self) -> None:
         await self.init_permissions()
         info = await self.client.get_me()
         self.tgid = info.id
         self.username = info.username
         self.mxid = pu.Puppet.get_mxid_from_id(self.tgid)
 
-        chat_ids = [id for id, type in self.chats.items() if type == "chat"]
+        chat_ids = [chat_id for chat_id, chat_type in self.chats.items() if chat_type == "chat"]
         response = await self.client(GetChatsRequest(chat_ids))
         for chat in response.chats:
             if isinstance(chat, ChatForbidden) or chat.left or chat.deactivated:
-                self.remove_chat(chat.id)
+                self.remove_chat(TelegramID(chat.id))
 
-        channel_ids = [InputChannel(id, 0)
+        channel_ids = [InputChannel(chat_id, 0)
-                       for id, type in self.chats.items()
+                       for chat_id, chat_type in self.chats.items()
-                       if type == "channel"]
+                       if chat_type == "channel"]
-        for id in channel_ids:
+        for channel_id in channel_ids:
             try:
-                await self.client(GetChannelsRequest([id]))
+                await self.client(GetChannelsRequest([channel_id]))
             except (ChannelPrivateError, ChannelInvalidError):
-                self.remove_chat(id.channel_id)
+                self.remove_chat(TelegramID(channel_id.channel_id))
 
         if config["bridge.catch_up"]:
             try:
@@ -93,29 +109,25 @@ class Bot(AbstractUser):
             except Exception:
                 self.log.exception("Failed to run catch_up() for bot")
 
-    def register_portal(self, portal: po.Portal):
+    def register_portal(self, portal: po.Portal) -> None:
         self.add_chat(portal.tgid, portal.peer_type)
 
-    def unregister_portal(self, portal: po.Portal):
+    def unregister_portal(self, portal: po.Portal) -> None:
         self.remove_chat(portal.tgid)
 
-    def add_chat(self, id: int, type: str):
+    def add_chat(self, chat_id: TelegramID, chat_type: str) -> None:
-        if id not in self.chats:
+        if chat_id not in self.chats:
-            self.chats[id] = type
+            self.chats[chat_id] = chat_type
-            self.db.add(BotChat(id=id, type=type))
+            BotChat(id=TelegramID(chat_id), type=chat_type).insert()
-            self.db.commit()
 
-    def remove_chat(self, id: int):
+    def remove_chat(self, chat_id: TelegramID) -> None:
         try:
-            del self.chats[id]
+            del self.chats[chat_id]
         except KeyError:
             pass
-        existing_chat = BotChat.query.get(id)
+        BotChat.delete(chat_id)
-        if existing_chat:
-            self.db.delete(existing_chat)
-            self.db.commit()
 
-    async def _can_use_commands(self, chat, tgid):
+    async def _can_use_commands(self, chat: TypePeer, tgid: TelegramID) -> bool:
         if tgid in self.tg_whitelist:
             return True
 
@@ -134,14 +146,15 @@ class Bot(AbstractUser):
                 for p in participants:
                     if p.user_id == tgid:
                         return isinstance(p, (ChatParticipantCreator, ChatParticipantAdmin))
+        return False
 
-    async def check_can_use_commands(self, event: Message, reply: ReplyFunc):
+    async def check_can_use_commands(self, event: Message, reply: ReplyFunc) -> bool:
         if not await self._can_use_commands(event.to_id, event.from_id):
             await reply("You do not have the permission to use that command.")
             return False
         return True
 
-    async def handle_command_portal(self, portal: po.Portal, reply: ReplyFunc):
+    async def handle_command_portal(self, portal: po.Portal, reply: ReplyFunc) -> Message:
         if not config["bridge.relaybot.authless_portals"]:
             return await reply("This bridge doesn't allow portal creation from Telegram.")
 
@@ -157,18 +170,19 @@ class Bot(AbstractUser):
                 return await reply(
                     "Portal is not public. Use `/invite <mxid>` to get an invite.")
 
-    async def handle_command_invite(self, portal: po.Portal, reply: ReplyFunc, mxid: str):
+    async def handle_command_invite(self, portal: po.Portal, reply: ReplyFunc,
-        if len(mxid) == 0:
+                                    mxid_input: MatrixUserID) -> Message:
+        if len(mxid_input) == 0:
             return await reply("Usage: `/invite <mxid>`")
         elif not portal.mxid:
             return await reply("Portal does not have Matrix room. "
                                "Create one with /portal first.")
-        if not self.mxid_regex.match(mxid):
+        if not self.mxid_regex.match(mxid_input):
             return await reply("That doesn't look like a Matrix ID.")
-        user = await u.User.get_by_mxid(mxid).ensure_started()
+        user = await u.User.get_by_mxid(MatrixUserID(mxid_input)).ensure_started()
         if not user.relaybot_whitelisted:
             return await reply("That user is not whitelisted to use the bridge.")
-        elif user.logged_in:
+        elif await user.is_logged_in():
             displayname = f"@{user.username}" if user.username else user.displayname
             return await reply("That user seems to be logged in. "
                                f"Just invite [{displayname}](tg://user?id={user.tgid})")
@@ -176,7 +190,8 @@ class Bot(AbstractUser):
             await portal.main_intent.invite(portal.mxid, user.mxid)
             return await reply(f"Invited `{user.mxid}` to the portal.")
 
-    def handle_command_id(self, message: Message, reply: ReplyFunc):
+    @staticmethod
+    def handle_command_id(message: Message, reply: ReplyFunc) -> Awaitable[Message]:
         # Provide the prefixed ID to the user so that the user wouldn't need to specify whether the
         # chat is a normal group or a supergroup/channel when using the ID.
         if isinstance(message.to_id, PeerChannel):
@@ -198,15 +213,15 @@ class Bot(AbstractUser):
 
         return False
 
-    async def handle_command(self, message: Message):
+    async def handle_command(self, message: Message) -> None:
-        def reply(reply_text):
+        def reply(reply_text: str) -> Awaitable[Message]:
-            return self.client.send_message(message.to_id, reply_text, markdown=True,
+            return self.client.send_message(message.to_id, reply_text, reply_to=message.id)
-                                            reply_to=message.id)
 
         text = message.message
 
         if self.match_command(text, "id"):
-            return await self.handle_command_id(message, reply)
+            await self.handle_command_id(message, reply)
+            return
 
         portal = po.Portal.get_by_entity(message.to_id)
 
@@ -221,36 +236,42 @@ class Bot(AbstractUser):
                 mxid = text[text.index(" ") + 1:]
             except ValueError:
                 mxid = ""
-            await self.handle_command_invite(portal, reply, mxid=mxid)
+            await self.handle_command_invite(portal, reply, mxid_input=mxid)
 
-    def handle_service_message(self, message: MessageService):
+    def handle_service_message(self, message: MessageService) -> None:
-        to_id = message.to_id
+        to_id = message.to_id  # type: TelegramID
         if isinstance(to_id, PeerChannel):
             to_id = to_id.channel_id
-            type = "channel"
+            chat_type = "channel"
         elif isinstance(to_id, PeerChat):
             to_id = to_id.chat_id
-            type = "chat"
+            chat_type = "chat"
         else:
             return
 
         action = message.action
         if isinstance(action, MessageActionChatAddUser) and self.tgid in action.users:
-            self.add_chat(to_id, type)
+            self.add_chat(to_id, chat_type)
         elif isinstance(action, MessageActionChatDeleteUser) and action.user_id == self.tgid:
             self.remove_chat(to_id)
+        elif isinstance(action, MessageActionChatMigrateTo):
+            self.remove_chat(to_id)
+            self.add_chat(TelegramID(action.channel_id), "channel")
 
-    async def update(self, update):
+    async def update(self, update) -> bool:
         if not isinstance(update, (UpdateNewMessage, UpdateNewChannelMessage)):
-            return
+            return False
         if isinstance(update.message, MessageService):
-            return self.handle_service_message(update.message)
+            self.handle_service_message(update.message)
+            return False
 
         is_command = (isinstance(update.message, Message)
                       and update.message.entities and len(update.message.entities) > 0
                       and isinstance(update.message.entities[0], MessageEntityBotCommand))
         if is_command:
-            return await self.handle_command(update.message)
+            await self.handle_command(update.message)
+            return True
+        return False
 
     def is_in_chat(self, peer_id) -> bool:
         return peer_id in self.chats
@@ -260,9 +281,9 @@ class Bot(AbstractUser):
         return "bot"
 
 
-def init(context):
+def init(cfg: 'Config') -> Optional[Bot]:
     global config
-    config = context.config
+    config = cfg
     token = config["telegram.bot_token"]
     if token and not token.lower().startswith("disable"):
         return Bot(token)

mautrix_telegram/commands/__init__.py

@@ -1,2 +1,5 @@
-from .handler import command_handler, CommandHandler, CommandEvent
+from .handler import (command_handler, command_handlers as _command_handlers,
-from . import clean_rooms, auth, meta, telegram, portal
+                      CommandHandler, CommandProcessor, CommandEvent,
+                      SECTION_GENERAL, SECTION_AUTH, SECTION_CREATING_PORTALS,
+                      SECTION_PORTAL_MANAGEMENT, SECTION_MISC, SECTION_ADMIN)
+from . import portal, telegram, clean_rooms, matrix_auth, meta

mautrix_telegram/commands/clean_rooms.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -14,20 +14,27 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from mautrix_appservice import MatrixRequestError
+from typing import Dict, List, NewType, Optional, Tuple, Union
 
-from . import command_handler
+from mautrix_appservice import MatrixRequestError, IntentAPI
+
+from ..types import MatrixRoomID, MatrixUserID
+from . import command_handler, CommandEvent, SECTION_ADMIN
 from .. import puppet as pu, portal as po
 
+ManagementRoom = NewType('ManagementRoom', Tuple[MatrixRoomID, MatrixUserID])
 
-async def _find_rooms(intent):
+
-    management_rooms = []
+async def _find_rooms(intent: IntentAPI) -> Tuple[List[ManagementRoom], List[MatrixRoomID],
-    unidentified_rooms = []
+                                                  List['po.Portal'], List['po.Portal']]:
-    portals = []
+    management_rooms = []  # type: List[ManagementRoom]
-    empty_portals = []
+    unidentified_rooms = []  # type: List[MatrixRoomID]
+    portals = []  # type: List[po.Portal]
+    empty_portals = []  # type: List[po.Portal]
 
     rooms = await intent.get_joined_rooms()
-    for room in rooms:
+    for room_str in rooms:
+        room = MatrixRoomID(room_str)
         portal = po.Portal.get_by_mxid(room)
         if not portal:
             try:
@@ -35,11 +42,11 @@ async def _find_rooms(intent):
             except MatrixRequestError:
                 members = []
             if len(members) == 2:
-                other_member = members[0] if members[0] != intent.mxid else members[1]
+                other_member = MatrixUserID(members[0] if members[0] != intent.mxid else members[1])
                 if pu.Puppet.get_id_from_mxid(other_member):
                     unidentified_rooms.append(room)
                 else:
-                    management_rooms.append((room, other_member))
+                    management_rooms.append(ManagementRoom((room, other_member)))
             else:
                 unidentified_rooms.append(room)
         else:
@@ -52,12 +59,10 @@ async def _find_rooms(intent):
     return management_rooms, unidentified_rooms, portals, empty_portals
 
 
-@command_handler(needs_admin=True, needs_auth=False, name="clean-rooms")
+@command_handler(needs_admin=True, needs_auth=False, management_only=True, name="clean-rooms",
-async def clean_rooms(evt):
+                 help_section=SECTION_ADMIN,
-    if not evt.is_management:
+                 help_text="Clean up unused portal/management rooms.")
-        return await evt.reply("`clean-rooms` is a particularly spammy command. Please don't "
+async def clean_rooms(evt: CommandEvent) -> Optional[Dict]:
-                               "run it in non-management rooms.")
-
     management_rooms, unidentified_rooms, portals, empty_portals = await _find_rooms(evt.az.intent)
 
     reply = ["#### Management rooms (M)"]
@@ -65,7 +70,7 @@ async def clean_rooms(evt):
                for n, (room, other_member) in enumerate(management_rooms)]
               or ["No management rooms found."])
     reply.append("#### Active portal rooms (A)")
-    reply += ([f"{n+1}. [P{n+1}](https://matrix.to/#/{portal.mxid}) "
+    reply += ([f"{n+1}. [A{n+1}](https://matrix.to/#/{portal.mxid}) "
                f"(to Telegram chat \"{portal.title}\")"
                for n, portal in enumerate(portals)]
               or ["No active portal rooms found."])
@@ -74,7 +79,7 @@ async def clean_rooms(evt):
                for n, room in enumerate(unidentified_rooms)]
               or ["No unidentified rooms found."])
     reply.append("#### Inactive portal rooms (I)")
-    reply += ([f"{n}. [E{n}](https://matrix.to/#/{portal.mxid}) "
+    reply += ([f"{n}. [I{n}](https://matrix.to/#/{portal.mxid}) "
                f"(to Telegram chat \"{portal.title}\")"
                for n, portal in enumerate(empty_portals)]
               or ["No inactive portal rooms found."])
@@ -88,9 +93,9 @@ async def clean_rooms(evt):
               "",
               ("To clean specific rooms, type `$cmdprefix+sp clean-range <range>` "
                "where `range` is the range (e.g. `5-21`) prefixed with the first letter of"
-               "the group name."),
+               "the group name. (e.g. `I2-6`)"),
               "",
-              ("Please note that you will have to re-run `$cmdprefix+sp cleanrooms` "
+              ("Please note that you will have to re-run `$cmdprefix+sp clean-rooms` "
                "between each use of the commands above.")]
 
     evt.sender.command_status = {
@@ -102,17 +107,20 @@ async def clean_rooms(evt):
     return await evt.reply("\n".join(reply))
 
 
-async def set_rooms_to_clean(evt, management_rooms, unidentified_rooms, portals, empty_portals):
+async def set_rooms_to_clean(evt, management_rooms: List[ManagementRoom],
+                             unidentified_rooms: List[MatrixRoomID], portals: List["po.Portal"],
+                             empty_portals: List["po.Portal"]) -> None:
     command = evt.args[0]
-    rooms_to_clean = []
+    rooms_to_clean = []  # type: List[Union[po.Portal, MatrixRoomID]]
     if command == "clean-recommended":
-        rooms_to_clean = empty_portals + unidentified_rooms
+        rooms_to_clean += empty_portals
+        rooms_to_clean += unidentified_rooms
     elif command == "clean-groups":
         if len(evt.args) < 2:
             return await evt.reply("**Usage:** `$cmdprefix+sp clean-groups [M][A][U][I]")
-        groups_to_clean = evt.args[1]
+        groups_to_clean = evt.args[1].upper()
         if "M" in groups_to_clean:
-            rooms_to_clean += management_rooms
+            rooms_to_clean += [room_id for (room_id, user_id) in management_rooms]
         if "A" in groups_to_clean:
             rooms_to_clean += portals
         if "U" in groups_to_clean:
@@ -121,12 +129,12 @@ async def set_rooms_to_clean(evt, management_rooms, unidentified_rooms, portals,
             rooms_to_clean += empty_portals
     elif command == "clean-range":
         try:
-            range = evt.args[1]
+            clean_range = evt.args[1]
-            group, range = range[0], range[1:]
+            group, clean_range = clean_range[0], clean_range[1:]
-            start, end = range.split("-")
+            start, end = clean_range.split("-")
             start, end = int(start), int(end)
             if group == "M":
-                group = management_rooms
+                group = [room_id for (room_id, user_id) in management_rooms]
             elif group == "A":
                 group = portals
             elif group == "U":
@@ -152,7 +160,7 @@ async def set_rooms_to_clean(evt, management_rooms, unidentified_rooms, portals,
                     "`$cmdprefix+sp confirm-clean`.")
 
 
-async def execute_room_cleanup(evt, rooms_to_clean):
+async def execute_room_cleanup(evt, rooms_to_clean: List[Union[po.Portal, MatrixRoomID]]) -> None:
     if len(evt.args) > 0 and evt.args[0] == "confirm-clean":
         await evt.reply(f"Cleaning {len(rooms_to_clean)} rooms. "
                         "This might take a while.")
@@ -161,7 +169,7 @@ async def execute_room_cleanup(evt, rooms_to_clean):
             if isinstance(room, po.Portal):
                 await room.cleanup_and_delete()
                 cleaned += 1
-            elif isinstance(room, str):
+            elif isinstance(room, str):  # str is aliased by MatrixRoomID
                 await po.Portal.cleanup_room(evt.az.intent, room, message="Room deleted")
                 cleaned += 1
         evt.sender.command_status = None

mautrix_telegram/commands/handler.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -14,90 +14,357 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-import markdown
+"""This module contains classes handling commands issued by Matrix users."""
+from typing import Awaitable, Callable, Dict, List, NamedTuple, Optional
 import logging
+import traceback
+
+import commonmark
 
 from telethon.errors import FloodWaitError
 
+from ..types import MatrixRoomID, MatrixEventID
 from ..util import format_duration
+from .. import user as u, context as c
 
-command_handlers = {}
+command_handlers = {}  # type: Dict[str, CommandHandler]
+
+HelpSection = NamedTuple('HelpSection', [('name', str), ('order', int), ('description', str)])
+
+SECTION_GENERAL = HelpSection("General", 0, "")
+SECTION_AUTH = HelpSection("Authentication", 10, "")
+SECTION_CREATING_PORTALS = HelpSection("Creating portals", 20, "")
+SECTION_PORTAL_MANAGEMENT = HelpSection("Portal management", 30, "")
+SECTION_MISC = HelpSection("Miscellaneous", 40, "")
+SECTION_ADMIN = HelpSection("Administration", 50, "")
 
 
-def command_handler(needs_auth=True, management_only=False, needs_admin=False, name=None):
+class HtmlEscapingRenderer(commonmark.HtmlRenderer):
-    def decorator(func):
+    def __init__(self, allow_html: bool = False):
-        def wrapper(evt):
+        super().__init__()
-            if management_only and not evt.is_management:
+        self.allow_html = allow_html
-                return evt.reply(f"`{evt.command}` is a restricted command:"
-                                 "you may only run it in management rooms.")
-            elif needs_auth and not evt.sender.logged_in:
-                return evt.reply("This command requires you to be logged in.")
-            elif needs_admin and not evt.sender.is_admin:
-                return evt.reply("This is command requires administrator privileges.")
-            return func(evt)
 
-        command_handlers[name or func.__name__.replace("_", "-")] = wrapper
+    def lit(self, s):
-        return wrapper
+        if self.allow_html:
+            return super().lit(s)
+        return super().lit(s.replace("<", "&lt;").replace(">", "&gt;"))
 
-    return decorator
+    def image(self, node, entering):
+        prev = self.allow_html
+        self.allow_html = True
+        super().image(node, entering)
+        self.allow_html = prev
+
+
+md_parser = commonmark.Parser()
+md_renderer = HtmlEscapingRenderer()
+
+
+def ensure_trailing_newline(s: str) -> str:
+    """Returns the passed string, but with a guaranteed trailing newline."""
+    return s + ("" if s[-1] == "\n" else "\n")
 
 
 class CommandEvent:
-    def __init__(self, handler, room, sender, command, args, is_management, is_portal):
+    """Holds information about a command issued in a Matrix room.
-        self.az = handler.az
+
-        self.log = handler.log
+    When a Matrix command was issued to the bot, CommandEvent will hold
-        self.loop = handler.loop
+    information regarding the event.
-        self.tgbot = handler.tgbot
+
-        self.config = handler.config
+    Attributes:
-        self.command_prefix = handler.command_prefix
+        room_id: The id of the Matrix room in which the command was issued.
+        event_id: The id of the matrix event which contained the command.
+        sender: The user who issued the command.
+        command: The issued command.
+        args: Arguments given with the issued command.
+        is_management: Determines whether the room in which the command wa
+            issued is a management room.
+        is_portal: Determines whether the room in which the command was issued
+            is a portal.
+    """
+
+    def __init__(self, processor: 'CommandProcessor', room: MatrixRoomID, event: MatrixEventID,
+                 sender: u.User, command: str, args: List[str], is_management: bool,
+                 is_portal: bool) -> None:
+        self.az = processor.az
+        self.log = processor.log
+        self.loop = processor.loop
+        self.tgbot = processor.tgbot
+        self.config = processor.config
+        self.public_website = processor.public_website
+        self.command_prefix = processor.command_prefix
         self.room_id = room
+        self.event_id = event
         self.sender = sender
         self.command = command
         self.args = args
         self.is_management = is_management
         self.is_portal = is_portal
 
-    def reply(self, message, allow_html=False, render_markdown=True):
+    def reply(self, message: str, allow_html: bool = False, render_markdown: bool = True
-        message = message.replace("$cmdprefix+sp ",
+              ) -> Awaitable[Dict]:
-                                  "" if self.is_management else f"{self.command_prefix} ")
+        """Write a reply to the room in which the command was issued.
-        message = message.replace("$cmdprefix", self.command_prefix)
+
-        html = None
+        Replaces occurences of "$cmdprefix" in the message with the command
+        prefix and replaces occurences of "$cmdprefix+sp " with the command
+        prefix if the command was not issued in a management room.
+        If allow_html and render_markdown are both False, the message will not
+        be rendered to html and sending of html is disabled.
+
+        Args:
+            message: The message to post in the room.
+            allow_html: Escape html in the message or don't render html at all
+                if markdown is disabled.
+            render_markdown: Use markdown formatting to render the passed
+                message to html.
+
+        Returns:
+            Handler for the message sending function.
+        """
+        message_cmd = self._replace_command_prefix(message)
+        html = self._render_message(message_cmd, allow_html=allow_html,
+                                    render_markdown=render_markdown)
+
+        return self.az.intent.send_notice(self.room_id, message_cmd, html=html)
+
+    def mark_read(self) -> Awaitable[Dict]:
+        """Marks the command as read by the bot."""
+        return self.az.intent.mark_read(self.room_id, self.event_id)
+
+    def _replace_command_prefix(self, message: str) -> str:
+        """Returns the string with the proper command prefix entered."""
+        message = message.replace(
+            "$cmdprefix+sp ", "" if self.is_management else f"{self.command_prefix} "
+        )
+        return message.replace("$cmdprefix", self.command_prefix)
+
+    @staticmethod
+    def _render_message(message: str, allow_html: bool, render_markdown: bool) -> Optional[str]:
+        """Renders the message as HTML.
+
+        Args:
+            allow_html: Flag to allow custom HTML in the message.
+            render_markdown: If true, markdown styling is applied to the message.
+
+        Returns:
+            The message rendered as HTML.
+            None is returned if no styled output is required.
+        """
+        html = ""
         if render_markdown:
-            html = markdown.markdown(message, safe_mode="escape" if allow_html else False)
+            md_renderer.allow_html = allow_html
+            html = md_renderer.render(md_parser.parse(message))
         elif allow_html:
             html = message
-        return self.az.intent.send_notice(self.room_id, message, html=html)
+        return ensure_trailing_newline(html) if html else None
 
 
 class CommandHandler:
+    """A command which can be executed from a Matrix room.
+
+    The command manages its permission and help texts.
+    When called, it will check the permission of the command event and execute
+    the command or, in case of error, report back to the user.
+
+    Attributes:
+        needs_auth: Flag indicating if the sender is required to be logged in.
+        needs_puppeting: Flag indicating if the sender is required to use
+            Telegram puppeteering for this command.
+        needs_matrix_puppeting: Flag indicating if the sender is required to use
+            Matrix pupeteering.
+        needs_admin: Flag for whether only admin users can issue this command.
+        management_only: Whether the command can exclusively be issued in a
+            management room.
+        name: The name of this command.
+        help_section: Section of the help in which this command will appear.
+    """
+
+    def __init__(self, handler: Callable[[CommandEvent], Awaitable[Dict]], needs_auth: bool,
+                 needs_puppeting: bool, needs_matrix_puppeting: bool, needs_admin: bool,
+                 management_only: bool, name: str, help_text: str, help_args: str,
+                 help_section: HelpSection) -> None:
+        """
+        Args:
+            handler: The function handling the execution of this command.
+            needs_auth: Flag indicating if the sender is required to be logged in.
+            needs_puppeting: Flag indicating if the sender is required to use
+                Telegram puppeteering for this command.
+            needs_matrix_puppeting: Flag indicating if the sender is required to
+                use Matrix pupeteering.
+            needs_admin: Flag for whether only admin users can issue this command.
+            management_only: Whether the command can exclusively be issued
+                in a management room.
+            name: The name of this command.
+            help_text: The text displayed in the help for this command.
+            help_args: Help text for the arguments of this command.
+            help_section: Section of the help in which this command will appear.
+        """
+        self._handler = handler
+        self.needs_auth = needs_auth
+        self.needs_puppeting = needs_puppeting
+        self.needs_matrix_puppeting = needs_matrix_puppeting
+        self.needs_admin = needs_admin
+        self.management_only = management_only
+        self.name = name
+        self._help_text = help_text
+        self._help_args = help_args
+        self.help_section = help_section
+
+    async def get_permission_error(self, evt: CommandEvent) -> Optional[str]:
+        """Returns the reason why the command could not be issued.
+
+        Args:
+            evt: The event for which to get the error information.
+
+        Returns:
+            A string describing the error or None if there was no error.
+        """
+        if self.management_only and not evt.is_management:
+            return (f"`{evt.command}` is a restricted command: "
+                    "you may only run it in management rooms.")
+        elif self.needs_puppeting and not evt.sender.puppet_whitelisted:
+            return "This command requires puppeting privileges."
+        elif self.needs_matrix_puppeting and not evt.sender.matrix_puppet_whitelisted:
+            return "This command requires Matrix puppeting privileges."
+        elif self.needs_admin and not evt.sender.is_admin:
+            return "This command requires administrator privileges."
+        elif self.needs_auth and not await evt.sender.is_logged_in():
+            return "This command requires you to be logged in."
+        return None
+
+    def has_permission(self, is_management: bool, puppet_whitelisted: bool,
+                       matrix_puppet_whitelisted: bool, is_admin: bool, is_logged_in: bool) -> bool:
+        """Checks the permission for this command with the given status.
+
+        Args:
+            is_management: If the room in which the command will be issued is a
+                management room.
+            puppet_whitelisted: If the connected Telegram account puppet is
+                allowed to issue the command.
+            matrix_puppet_whitelisted: If the connected Matrix account puppet is
+                allowed to issue the command.
+            is_admin: If the issuing user is an admin.
+            is_logged_in: If the issuing user is logged in.
+
+        Returns:
+            True if a user with the given state is allowed to issue the
+            command.
+        """
+        return ((not self.management_only or is_management) and
+                (not self.needs_puppeting or puppet_whitelisted) and
+                (not self.needs_matrix_puppeting or matrix_puppet_whitelisted) and
+                (not self.needs_admin or is_admin) and
+                (not self.needs_auth or is_logged_in))
+
+    async def __call__(self, evt: CommandEvent) -> Dict:
+        """Executes the command if evt was issued with proper rights.
+
+        Args:
+            evt: The CommandEvent for which to check permissions.
+
+        Returns:
+            The result of the command or the error message function.
+
+        Raises:
+            FloodWaitError
+        """
+        error = await self.get_permission_error(evt)
+        if error is not None:
+            return await evt.reply(error)
+        return await self._handler(evt)
+
+    @property
+    def has_help(self) -> bool:
+        """Returns true if this command has a help text."""
+        return bool(self.help_section) and bool(self._help_text)
+
+    @property
+    def help(self) -> str:
+        """Returns the help text to this command."""
+        return f"**{self.name}** {self._help_args} - {self._help_text}"
+
+
+def command_handler(_func: Optional[Callable[[CommandEvent], Awaitable[Dict]]] = None, *,
+                    needs_auth: bool = True,
+                    needs_puppeting: bool = True,
+                    needs_matrix_puppeting: bool = False,
+                    needs_admin: bool = False,
+                    management_only: bool = False,
+                    name: Optional[str] = None,
+                    help_text: str = "",
+                    help_args: str = "",
+                    help_section: HelpSection = None,
+                    ) -> Callable[[Callable[[CommandEvent], Awaitable[Optional[Dict]]]],
+                                  CommandHandler]:
+    def decorator(func: Callable[[CommandEvent], Awaitable[Optional[Dict]]]) -> CommandHandler:
+        actual_name = name or func.__name__.replace("_", "-")
+        handler = CommandHandler(func, needs_auth, needs_puppeting, needs_matrix_puppeting,
+                                 needs_admin, management_only, actual_name, help_text, help_args,
+                                 help_section)
+        command_handlers[handler.name] = handler
+        return handler
+
+    return decorator if _func is None else decorator(_func)
+
+
+class CommandProcessor:
+    """Handles the raw commands issued by a user to the Matrix bot."""
     log = logging.getLogger("mau.commands")
 
-    def __init__(self, context):
+    def __init__(self, context: c.Context) -> None:
-        self.az, self.db, self.config, self.loop, self.tgbot = context
+        self.az, self.config, self.loop, self.tgbot = context.core
+        self.public_website = context.public_website
         self.command_prefix = self.config["bridge.command_prefix"]
 
-    # region Utility functions for handling commands
+    async def handle(self, room: MatrixRoomID, event_id: MatrixEventID, sender: u.User,
+                     command: str, args: List[str], is_management: bool, is_portal: bool
+                     ) -> Optional[Dict]:
+        """Handles the raw commands issued by a user to the Matrix bot.
 
-    async def handle(self, room, sender, command, args, is_management, is_portal):
+        If the command is not known, it might be a followup command and is
-        evt = CommandEvent(self, room, sender, command, args,
+        delegated to a command handler registered for that purpose in the
-                           is_management, is_portal)
+        senders command_status as "next".
+
+        Args:
+            room: ID of the Matrix room in which the command was issued.
+            event_id: ID of the event by which the command was issued.
+            sender: The sender who issued the command.
+            command: The issued command, case insensitive.
+            args: Arguments given with the command.
+            is_management: Whether the room is a management room.
+            is_portal: Whether the room is a portal.
+
+        Returns:
+            The result of the error message function or None if no error
+            occured. Unknown and delegated commands do not count as errors.
+        """
+        if not command_handlers or "unknown-command" not in command_handlers:
+            raise ValueError("command_handlers are not properly initialized.")
+
+        evt = CommandEvent(self, room, event_id, sender, command, args, is_management, is_portal)
         orig_command = command
         command = command.lower()
         try:
-            command = command_handlers[command]
+            handler = command_handlers[command]
         except KeyError:
             if sender.command_status and "next" in sender.command_status:
                 args.insert(0, orig_command)
                 evt.command = ""
-                command = sender.command_status["next"]
+                handler = sender.command_status["next"]
             else:
-                command = command_handlers["unknown-command"]
+                handler = command_handlers["unknown-command"]
         try:
-            await command(evt)
+            await handler(evt)
         except FloodWaitError as e:
             return await evt.reply(f"Flood error: Please wait {format_duration(e.seconds)}")
         except Exception:
-            self.log.exception("Fatal error handling command "
+            self.log.exception("Unhandled error while handling command "
                                f"{evt.command} {' '.join(args)} from {sender.mxid}")
-            return await evt.reply("Fatal error while handling command. "
+            if evt.sender.is_admin and evt.is_management:
+                return await evt.reply("Unhandled error while handling command:\n\n"
+                                       "```traceback\n"
+                                       f"{traceback.format_exc()}"
+                                       "```")
+            return await evt.reply("Unhandled error while handling command. "
                                    "Check logs for more details.")
+        return None

mautrix_telegram/commands/matrix_auth.py

@@ -0,0 +1,101 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Dict, Optional
+
+from . import command_handler, CommandEvent, SECTION_AUTH
+from .. import puppet as pu
+
+
+@command_handler(needs_auth=True, needs_matrix_puppeting=True,
+                 help_section=SECTION_AUTH,
+                 help_text="Revert your Telegram account's Matrix puppet to use the default Matrix "
+                           "account.")
+async def logout_matrix(evt: CommandEvent) -> Optional[Dict]:
+    puppet = pu.Puppet.get(evt.sender.tgid)
+    if not puppet.is_real_user:
+        return await evt.reply("You are not logged in with your Matrix account.")
+    await puppet.switch_mxid(None, None)
+    return await evt.reply("Reverted your Telegram account's Matrix puppet back to the default.")
+
+
+@command_handler(needs_auth=True, management_only=True, needs_matrix_puppeting=True,
+                 help_section=SECTION_AUTH,
+                 help_text="Replace your Telegram account's Matrix puppet with your own Matrix "
+                           "account.")
+async def login_matrix(evt: CommandEvent) -> Optional[Dict]:
+    puppet = pu.Puppet.get(evt.sender.tgid)
+    if puppet.is_real_user:
+        return await evt.reply("You have already logged in with your Matrix account. "
+                               "Log out with `$cmdprefix+sp logout-matrix` first.")
+    allow_matrix_login = evt.config.get("bridge.allow_matrix_login", True)
+    if allow_matrix_login:
+        evt.sender.command_status = {
+            "next": enter_matrix_token,
+            "action": "Matrix login",
+        }
+    if evt.config["appservice.public.enabled"]:
+        prefix = evt.config["appservice.public.external"]
+        token = evt.public_website.make_token(evt.sender.mxid, "/matrix-login")
+        url = f"{prefix}/matrix-login?token={token}"
+        if allow_matrix_login:
+            return await evt.reply(
+                "This bridge instance allows you to log in inside or outside Matrix.\n\n"
+                "If you would like to log in within Matrix, please send your Matrix access token "
+                "here.\n"
+                f"If you would like to log in outside of Matrix, [click here]({url}).\n\n"
+                "Logging in outside of Matrix is recommended, because in-Matrix login would save "
+                "your access token in the message history.")
+        return await evt.reply("This bridge instance does not allow logging in inside Matrix.\n\n"
+                               f"Please visit [the login page]({url}) to log in.")
+    elif allow_matrix_login:
+        return await evt.reply(
+            "This bridge instance does not allow you to log in outside of Matrix.\n\n"
+            "Please send your Matrix access token here to log in.")
+    return await evt.reply("This bridge instance has been configured to not allow logging in.")
+
+
+@command_handler(needs_auth=True, needs_matrix_puppeting=True,
+                 help_section=SECTION_AUTH,
+                 help_text="Pings the server with the stored matrix authentication.")
+async def ping_matrix(evt: CommandEvent) -> Optional[Dict]:
+    puppet = pu.Puppet.get(evt.sender.tgid)
+    if not puppet.is_real_user:
+        return await evt.reply("You are not logged in with your Matrix account.")
+    resp = await puppet.init_custom_mxid()
+    if resp == pu.PuppetError.InvalidAccessToken:
+        return await evt.reply("Your access token is invalid.")
+    elif resp == pu.PuppetError.Success:
+        return await evt.reply("Your Matrix login is working.")
+    return await evt.reply(f"Unknown response while checking your Matrix login: {resp}.")
+
+
+async def enter_matrix_token(evt: CommandEvent) -> Dict:
+    evt.sender.command_status = None
+
+    puppet = pu.Puppet.get(evt.sender.tgid)
+    if puppet.is_real_user:
+        return await evt.reply("You have already logged in with your Matrix account. "
+                               "Log out with `$cmdprefix+sp logout-matrix` first.")
+
+    resp = await puppet.switch_mxid(" ".join(evt.args), evt.sender.mxid)
+    if resp == pu.PuppetError.OnlyLoginSelf:
+        return await evt.reply("You can only log in as your own Matrix user.")
+    elif resp == pu.PuppetError.InvalidAccessToken:
+        return await evt.reply("Failed to verify access token.")
+    assert resp == pu.PuppetError.Success, "Encountered an unhandled PuppetError."
+    return await evt.reply(
+        f"Replaced your Telegram account's Matrix puppet with {puppet.custom_mxid}.")

mautrix_telegram/commands/meta.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -14,75 +14,59 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from . import command_handler
+from typing import Dict, List, Optional, Tuple
+
+from . import command_handler, CommandEvent, _command_handlers, SECTION_GENERAL
+from .handler import HelpSection
 
 
-@command_handler(needs_auth=False)
+@command_handler(needs_auth=False, needs_puppeting=False,
-def cancel(evt):
+                 help_section=SECTION_GENERAL,
+                 help_text="Cancel an ongoing action (such as login)")
+async def cancel(evt: CommandEvent) -> Optional[Dict]:
     if evt.sender.command_status:
         action = evt.sender.command_status["action"]
         evt.sender.command_status = None
-        return evt.reply(f"{action} cancelled.")
+        return await evt.reply(f"{action} cancelled.")
     else:
-        return evt.reply("No ongoing command.")
+        return await evt.reply("No ongoing command.")
 
 
-@command_handler(needs_auth=False)
+@command_handler(needs_auth=False, needs_puppeting=False)
-def unknown_command(evt):
+async def unknown_command(evt: CommandEvent) -> Optional[Dict]:
-    return evt.reply("Unknown command. Try `$cmdprefix+sp help` for help.")
+    return await evt.reply("Unknown command. Try `$cmdprefix+sp help` for help.")
 
 
-@command_handler(needs_auth=False)
+help_cache = {}  # type: Dict[Tuple[bool, bool, bool, bool, bool], str]
-def help(evt):
+
+
+async def _get_help_text(evt: CommandEvent) -> str:
+    cache_key = (evt.is_management, evt.sender.puppet_whitelisted,
+                 evt.sender.matrix_puppet_whitelisted, evt.sender.is_admin,
+                 await evt.sender.is_logged_in())
+    if cache_key not in help_cache:
+        help_sections = {}  # type: Dict[HelpSection, List[str]]
+        for handler in _command_handlers.values():
+            if handler.has_help and handler.has_permission(*cache_key):
+                help_sections.setdefault(handler.help_section, [])
+                help_sections[handler.help_section].append(handler.help + "  ")
+        help_sorted = sorted(help_sections.items(), key=lambda item: item[0].order)
+        helps = ["#### {}\n{}\n".format(key.name, "\n".join(value)) for key, value in help_sorted]
+        help_cache[cache_key] = "\n".join(helps)
+    return help_cache[cache_key]
+
+
+def _get_management_status(evt: CommandEvent) -> str:
     if evt.is_management:
-        management_status = ("This is a management room: prefixing commands "
+        return "This is a management room: prefixing commands with `$cmdprefix` is not required."
-                             "with `$cmdprefix` is not required.\n")
     elif evt.is_portal:
-        management_status = ("**This is a portal room**: you must always "
+        return ("**This is a portal room**: you must always prefix commands with `$cmdprefix`.\n"
-                             "prefix commands with `$cmdprefix`.\n"
+                "Management commands will not be sent to Telegram.")
-                             "Management commands will not be sent to Telegram.")
+    return "**This is not a management room**: you must prefix commands with `$cmdprefix`."
-    else:
-        management_status = ("**This is not a management room**: you must "
-                             "prefix commands with `$cmdprefix`.\n")
-    help = """\n
-#### Generic bridge commands
-**help**   - Show this help message.  
-**cancel** - Cancel an ongoing action (such as login).
 
-#### Authentication
-**login**  - Request an authentication code.  
-**logout** - Log out from Telegram.  
-**ping**   - Check if you're logged into Telegram.
 
-#### Miscellaneous things
+@command_handler(name="help", needs_auth=False, needs_puppeting=False,
-**search** [_-r|--remote_] <_query_> - Search your contacts or the Telegram servers for users.  
+                 help_section=SECTION_GENERAL,
-**sync** [`chats`|`contacts`|`me`]   - Synchronize your chat portals, contacts and/or own info.  
+                 help_text="Show this help message.")
-**ping-bot**                         - Get info of the message relay Telegram bot.  
+async def help_cmd(evt: CommandEvent) -> Optional[Dict]:
-**set-pl** <_level_> [_mxid_]        - Set a temporary power level without affecting Telegram.
+    return await evt.reply(_get_management_status(evt) + "\n" + await _get_help_text(evt))
-
-#### Initiating chats
-**pm** <_identifier_> - Open a private chat with the given Telegram user. The identifier is either
-                        the internal user ID, the username or the phone number.  
-**join** <_link_>     - Join a chat with an invite link.  
-**create** [_type_]   - Create a Telegram chat of the given type for the current Matrix room. The
-                        type is either `group`, `supergroup` or `channel` (defaults to `group`).
-
-#### Portal management  
-**upgrade**                 - Upgrade a normal Telegram group to a supergroup.  
-**invite-link**             - Get a Telegram invite link to the current chat.  
-**delete-portal**           - Remove all users from the current portal room and forget the portal.
-                              Only works for group chats; to delete a private chat portal, simply
-                              leave the room.  
-**unbridge**                - Remove puppets from the current portal room and forget the portal.  
-**bridge** [_id_]           - Bridge the current Matrix room to the Telegram chat with the given
-                              ID. The ID must be the prefixed version that you get with the `/id`
-                              command of the Telegram-side bot.  
-**group-name** <_name_|`-`> - Change the username of a supergroup/channel. To disable, use a dash
-                             (`-`) as the name.  
-**clean-rooms**             - Clean up unused portal/management rooms.
-
-**filter** <`whitelist`|`blacklist`> <_chat ID_> - Allow or disallow bridging a specific chat.  
-**filter-mode** <`whitelist`|`blacklist`>      - Change whether the bridge will allow or disallow
-                                                 bridging rooms by default.
-"""
-    return evt.reply(management_status + help)

mautrix_telegram/commands/portal.py

@@ -1,448 +0,0 @@
-# -*- coding: future_fstrings -*-
-# mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
-import asyncio
-
-from telethon.errors import *
-from telethon.tl.types import ChatForbidden, ChannelForbidden
-from mautrix_appservice import MatrixRequestError
-
-from .. import portal as po
-from . import command_handler, CommandEvent
-
-
-@command_handler(needs_admin=True, needs_auth=False, name="set-pl")
-async def set_power_level(evt: CommandEvent):
-    try:
-        level = int(evt.args[0])
-    except KeyError:
-        return await evt.reply("**Usage:** `$cmdprefix+sp set-power <level> [mxid]`")
-    except ValueError:
-        return await evt.reply("The level must be an integer.")
-    levels = await evt.az.intent.get_power_levels(evt.room_id)
-    mxid = evt.args[1] if len(evt.args) > 1 else evt.sender.mxid
-    levels["users"][mxid] = level
-    try:
-        await evt.az.intent.set_power_levels(evt.room_id, levels)
-    except MatrixRequestError:
-        evt.log.exception("Failed to set power level.")
-        return await evt.reply("Failed to set power level.")
-
-
-@command_handler()
-async def invite_link(evt: CommandEvent):
-    portal = po.Portal.get_by_mxid(evt.room_id)
-    if not portal:
-        return await evt.reply("This is not a portal room.")
-
-    if portal.peer_type == "user":
-        return await evt.reply("You can't invite users to private chats.")
-
-    try:
-        link = await portal.get_invite_link(evt.sender)
-        return await evt.reply(f"Invite link to {portal.title}: {link}")
-    except ValueError as e:
-        return await evt.reply(e.args[0])
-    except ChatAdminRequiredError:
-        return await evt.reply("You don't have the permission to create an invite link.")
-
-
-async def _has_access_to(room, intent, sender, event, default=50):
-    if sender.is_admin:
-        return True
-    # Make sure the state store contains the power levels.
-    try:
-        await intent.get_power_levels(room)
-    except MatrixRequestError:
-        return False
-    return intent.state_store.has_power_level(room, sender.mxid,
-                                              event=f"net.maunium.telegram.{event}",
-                                              default=default)
-
-
-async def _get_portal_and_check_permission(evt, permission, action=None):
-    room_id = evt.args[0] if len(evt.args) > 0 else evt.room_id
-
-    portal = po.Portal.get_by_mxid(room_id)
-    if not portal:
-        that_this = "This" if room_id == evt.room_id else "That"
-        return await evt.reply(f"{that_this} is not a portal room."), False
-
-    if not await _has_access_to(portal.mxid, evt.az.intent, evt.sender, permission):
-        action = action or f"{permission.replace('_', ' ')}s"
-        return await evt.reply(f"You do not have the permissions to {action} that portal."), False
-    return portal, True
-
-
-def _get_portal_murder_function(action, room_id, function, command, completed_message):
-    async def post_confirm(confirm):
-        confirm.sender.command_status = None
-        if len(confirm.args) > 0 and confirm.args[0] == f"confirm-{command}":
-            await function()
-            if confirm.room_id != room_id:
-                return await confirm.reply(completed_message)
-        else:
-            return await confirm.reply(f"{action} cancelled.")
-
-    return {
-        "next": post_confirm,
-        "action": action,
-    }
-
-
-@command_handler(needs_auth=False)
-async def delete_portal(evt: CommandEvent):
-    portal, ok = await _get_portal_and_check_permission(evt, "delete_portal")
-    if not ok:
-        return
-
-    evt.sender.command_status = _get_portal_murder_function("Portal deletion", portal.mxid,
-                                                            portal.cleanup_and_delete, "delete",
-                                                            "Portal successfully deleted.")
-    return await evt.reply("Please confirm deletion of portal "
-                           f"[{portal.alias or portal.mxid}](https://matrix.to/#/{portal.mxid}) "
-                           f"to Telegram chat \"{portal.title}\" "
-                           "by typing `$cmdprefix+sp confirm-delete`"
-                           "\n\n"
-                           "**WARNING:** If the bridge bot has the power level to do so, **this "
-                           "will kick ALL users** in the room. If you just want to remove the "
-                           "bridge, use `$cmdprefix+sp unbridge` instead.")
-
-
-@command_handler(needs_auth=False)
-async def unbridge(evt: CommandEvent):
-    portal, ok = await _get_portal_and_check_permission(evt, "unbridge_room")
-    if not ok:
-        return
-
-    evt.sender.command_status = _get_portal_murder_function("Room unbridging", portal.mxid,
-                                                            portal.unbridge, "unbridge",
-                                                            "Room successfully unbridged.")
-    return await evt.reply(f"Please confirm unbridging chat \"{portal.title}\" from room "
-                           f"[{portal.alias or portal.mxid}](https://matrix.to/#/{portal.mxid}) "
-                           "by typing `$cmdprefix+sp confirm-unbridge`")
-
-
-@command_handler(needs_auth=False)
-async def bridge(evt: CommandEvent):
-    if len(evt.args) == 0:
-        return await evt.reply("**Usage:** "
-                               "`$cmdprefix+sp bridge <Telegram chat ID> [Matrix room ID]`")
-    room_id = evt.args[1] if len(evt.args) > 1 else evt.room_id
-    that_this = "This" if room_id == evt.room_id else "That"
-
-    portal = po.Portal.get_by_mxid(room_id)
-    if portal:
-        return await evt.reply(f"{that_this} room is already a portal room.")
-
-    if not await _has_access_to(room_id, evt.az.intent, evt.sender, "bridge"):
-        return await evt.reply("You do not have the permissions to bridge that room.")
-
-    # The /id bot command provides the prefixed ID, so we assume
-    tgid = evt.args[0]
-    if tgid.startswith("-100"):
-        tgid = int(tgid[4:])
-        peer_type = "channel"
-    elif tgid.startswith("-"):
-        tgid = -int(tgid)
-        peer_type = "chat"
-    else:
-        return await evt.reply("That doesn't seem like a prefixed Telegram chat ID.\n\n"
-                               "If you did not get the ID using the `/id` bot command, please "
-                               "prefix channel IDs with `-100` and normal group IDs with `-`.\n\n"
-                               "Bridging private chats to existing rooms is not allowed.")
-
-    portal = po.Portal.get_by_tgid(tgid, peer_type=peer_type)
-    if not portal.allow_bridging():
-        return await evt.reply("This bridge doesn't allow bridging that Telegram chat.\n"
-                               "If you're the bridge admin, try"
-                               "`$cmdprefix+sp whitelist <Telegram chat ID>` first.")
-    if portal.mxid:
-        has_portal_message = (
-            "That Telegram chat already has a portal at "
-            f"[{portal.alias or portal.mxid}](https://matrix.to/#/{portal.mxid}). ")
-        if not await _has_access_to(portal.mxid, evt.az.intent, evt.sender, "unbridge"):
-            return await evt.reply(f"{has_portal_message}"
-                                   "Additionally, you do not have the permissions to unbridge "
-                                   "that room.")
-        evt.sender.command_status = {
-            "next": confirm_bridge,
-            "action": "Room bridging",
-            "mxid": portal.mxid,
-            "bridge_to_mxid": room_id,
-            "tgid": portal.tgid,
-            "peer_type": portal.peer_type,
-        }
-        return await evt.reply(f"{has_portal_message}"
-                               "However, you have the permissions to unbridge that room.\n\n"
-                               "To delete that portal completely and continue bridging, use "
-                               "`$cmdprefix+sp delete-and-continue`. To unbridge the portal "
-                               "without kicking Matrix users, use `$cmdprefix+sp unbridge-and-"
-                               "continue`. To cancel, use `$cmdprefix+sp cancel`")
-    evt.sender.command_status = {
-        "next": confirm_bridge,
-        "action": "Room bridging",
-        "bridge_to_mxid": room_id,
-        "tgid": portal.tgid,
-        "peer_type": portal.peer_type,
-    }
-    return await evt.reply("That Telegram chat has no existing portal. To confirm bridging the "
-                           "chat to this room, use `$cmdprefix+sp continue`")
-
-
-async def cleanup_old_portal_while_bridging(evt, portal):
-    if not portal.mxid:
-        await evt.reply("The portal seems to have lost its Matrix room between you"
-                        "calling `$cmdprefix+sp bridge` and this command.\n\n"
-                        "Continuing without touching previous Matrix room...")
-        return True, None
-    elif evt.args[0] == "delete-and-continue":
-        return True, portal.cleanup_room(portal.main_intent, portal.mxid,
-                                         message="Portal deleted (moving to another room)")
-    elif evt.args[0] == "unbridge-and-continue":
-        return True, portal.cleanup_room(portal.main_intent, portal.mxid,
-                                         message="Room unbridged (portal moving to another room)",
-                                         puppets_only=True)
-    else:
-        await evt.reply(
-            "The chat you were trying to bridge already has a Matrix portal room.\n\n"
-            "Please use `$cmdprefix+sp delete-and-continue` or `$cmdprefix+sp unbridge-and-"
-            "continue` to either delete or unbridge the existing room (respectively) and "
-            "continue with the bridging.\n\n"
-            "If you changed your mind, use `$cmdprefix+sp cancel` to cancel.")
-        return False, None
-
-
-async def confirm_bridge(evt: CommandEvent):
-    status = evt.sender.command_status
-    try:
-        portal = po.Portal.get_by_tgid(status["tgid"], peer_type=status["peer_type"])
-        bridge_to_mxid = status["bridge_to_mxid"]
-    except KeyError:
-        evt.sender.command_status = None
-        return await evt.reply("Fatal error: tgid or peer_type missing from command_status. "
-                               "This shouldn't happen unless you're messing with the command "
-                               "handler code.")
-    if "mxid" in status:
-        ok, coro = await cleanup_old_portal_while_bridging(evt, portal)
-        if not ok:
-            return
-        elif coro:
-            asyncio.ensure_future(coro, loop=evt.loop)
-            await evt.reply("Cleaning up previous portal room...")
-    elif portal.mxid:
-        evt.sender.command_status = None
-        return await evt.reply("The portal seems to have created a Matrix room between you "
-                               "calling `$cmdprefix+sp bridge` and this command.\n\n"
-                               "Please start over by calling the bridge command again.")
-    elif evt.args[0] != "continue":
-        return await evt.reply("Please use `$cmdprefix+sp continue` to confirm the bridging or "
-                               "`$cmdprefix+sp cancel` to cancel.")
-
-    user = evt.sender if evt.sender.logged_in else evt.tgbot
-    try:
-        entity = await user.client.get_entity(portal.peer)
-    except Exception:
-        evt.log.exception("Failed to get_entity(%s) for manual bridging.", portal.peer)
-        if evt.sender.logged_in:
-            return await evt.reply("Failed to get info of telegram chat. "
-                                   "You are logged in, are you in that chat?")
-        else:
-            return await evt.reply("Failed to get info of telegram chat. "
-                                   "You're not logged in, is the relay bot in the chat?")
-    if isinstance(entity, (ChatForbidden, ChannelForbidden)):
-        if evt.sender.logged_in:
-            return await evt.reply("You don't seem to be in that chat.")
-        else:
-            return await evt.reply("The bot doesn't seem to be in that chat.")
-
-    direct = False
-
-    portal.mxid = bridge_to_mxid
-    portal.title, portal.about, levels = await _get_initial_state(evt)
-    portal.photo_id = ""
-    portal.save()
-
-    asyncio.ensure_future(portal.update_matrix_room(user, entity, direct, levels=levels),
-                          loop=evt.loop)
-
-    return await evt.reply("Bridging complete. Portal synchronization should begin momentarily.")
-
-
-async def _get_initial_state(evt: CommandEvent):
-    state = await evt.az.intent.get_room_state(evt.room_id)
-    title = None
-    about = None
-    levels = None
-    for event in state:
-        if event["type"] == "m.room.name":
-            title = event["content"]["name"]
-        elif event["type"] == "m.room.topic":
-            about = event["content"]["topic"]
-        elif event["type"] == "m.room.power_levels":
-            levels = event["content"]
-        elif event["type"] == "m.room.canonical_alias":
-            title = title or event["content"]["alias"]
-    return title, about, levels
-
-
-@command_handler()
-async def create(evt: CommandEvent):
-    type = evt.args[0] if len(evt.args) > 0 else "group"
-    if type not in {"chat", "group", "supergroup", "channel"}:
-        return await evt.reply(
-            "**Usage:** `$cmdprefix+sp create ['group'/'supergroup'/'channel']`")
-
-    if po.Portal.get_by_mxid(evt.room_id):
-        return await evt.reply("This is already a portal room.")
-
-    title, about, levels = await _get_initial_state(evt)
-    if not title:
-        return await evt.reply("Please set a title before creating a Telegram chat.")
-
-    supergroup = type == "supergroup"
-    type = {
-        "supergroup": "channel",
-        "channel": "channel",
-        "chat": "chat",
-        "group": "chat",
-    }[type]
-
-    portal = po.Portal(tgid=None, mxid=evt.room_id, title=title, about=about, peer_type=type)
-    try:
-        await portal.create_telegram_chat(evt.sender, supergroup=supergroup)
-    except ValueError as e:
-        portal.delete()
-        return await evt.reply(e.args[0])
-    return await evt.reply(f"Telegram chat created. ID: {portal.tgid}")
-
-
-@command_handler()
-async def upgrade(evt: CommandEvent):
-    portal = po.Portal.get_by_mxid(evt.room_id)
-    if not portal:
-        return await evt.reply("This is not a portal room.")
-    elif portal.peer_type == "channel":
-        return await evt.reply("This is already a supergroup or a channel.")
-    elif portal.peer_type == "user":
-        return await evt.reply("You can't upgrade private chats.")
-
-    try:
-        await portal.upgrade_telegram_chat(evt.sender)
-        return await evt.reply(f"Group upgraded to supergroup. New ID: {portal.tgid}")
-    except ChatAdminRequiredError:
-        return await evt.reply("You don't have the permission to upgrade this group.")
-    except ValueError as e:
-        return await evt.reply(e.args[0])
-
-
-@command_handler()
-async def group_name(evt: CommandEvent):
-    if len(evt.args) == 0:
-        return await evt.reply("**Usage:** `$cmdprefix+sp group-name <name/->`")
-
-    portal = po.Portal.get_by_mxid(evt.room_id)
-    if not portal:
-        return await evt.reply("This is not a portal room.")
-    elif portal.peer_type != "channel":
-        return await evt.reply("Only channels and supergroups have usernames.")
-
-    try:
-        await portal.set_telegram_username(evt.sender,
-                                           evt.args[0] if evt.args[0] != "-" else "")
-        if portal.username:
-            return await evt.reply(f"Username of channel changed to {portal.username}.")
-        else:
-            return await evt.reply(f"Channel is now private.")
-    except ChatAdminRequiredError:
-        return await evt.reply(
-            "You don't have the permission to set the username of this channel.")
-    except UsernameNotModifiedError:
-        if portal.username:
-            return await evt.reply("That is already the username of this channel.")
-        else:
-            return await evt.reply("This channel is already private")
-    except UsernameOccupiedError:
-        return await evt.reply("That username is already in use.")
-    except UsernameInvalidError:
-        return await evt.reply("Invalid username")
-
-
-@command_handler(needs_admin=True)
-async def filter_mode(evt: CommandEvent):
-    try:
-        mode = evt.args[0]
-        if mode not in ("whitelist", "blacklist"):
-            raise ValueError()
-    except (IndexError, ValueError):
-        return await evt.reply("**Usage:** `$cmdprefix+sp filter-mode <whitelist/blacklist>`")
-
-    evt.config["bridge.filter.mode"] = mode
-    evt.config.save()
-    po.Portal.filter_mode = mode
-    if mode == "whitelist":
-        return await evt.reply("The bridge will now disallow bridging chats by default.\n"
-                               "To allow bridging a specific chat, use"
-                               "`!filter whitelist <chat ID>`.")
-    else:
-        return await evt.reply("The bridge will now allow bridging chats by default.\n"
-                               "To disallow bridging a specific chat, use"
-                               "`!filter blacklist <chat ID>`.")
-
-
-@command_handler(needs_admin=True)
-async def filter(evt: CommandEvent):
-    try:
-        action = evt.args[0]
-        if action not in ("whitelist", "blacklist", "add", "remove"):
-            raise ValueError()
-
-        id = evt.args[1]
-        if id.startswith("-100"):
-            id = int(id[4:])
-        elif id.startswith("-"):
-            id = int(id[1:])
-        else:
-            id = int(id)
-    except (IndexError, ValueError):
-        return await evt.reply("**Usage:** `$cmdprefix+sp filter <whitelist/blacklist> <chat ID>`")
-
-    mode = evt.config["bridge.filter.mode"]
-    if mode not in ("blacklist", "whitelist"):
-        return await evt.reply(f"Unknown filter mode \"{mode}\". Please fix the bridge config.")
-
-    list = evt.config["bridge.filter.list"]
-
-    if action in ("blacklist", "whitelist"):
-        action = "add" if mode == action else "remove"
-
-    def save():
-        evt.config["bridge.filter.list"] = list
-        evt.config.save()
-        po.Portal.filter_list = list
-
-    if action == "add":
-        if id in list:
-            return await evt.reply(f"That chat is already {mode}ed.")
-        list.append(id)
-        save()
-        return await evt.reply(f"Chat ID added to {mode}.")
-    elif action == "remove":
-        if id not in list:
-            return await evt.reply(f"That chat is not {mode}ed.")
-        list.remove(id)
-        save()
-        return await evt.reply(f"Chat ID removed from {mode}.")

mautrix_telegram/commands/portal/__init__.py

@@ -0,0 +1 @@
+from . import admin, bridge, config, create_chat, filter, misc, unbridge