Bump version to 0.6.1rc2

Users who the bridge only saw via logged in users with the target user
in their contact lists wouldn't get their displayname updated due to an
invalid condition in the update_displayname function.

.codeclimate.yml

@@ -0,0 +1,8 @@
+engines:
+  sonar-python:
+    enabled: true
+    checks:
+      python:S107:
+        enabled: false
+exclude_patterns:
+- "alembic/"

.dockerignore

@@ -0,0 +1,4 @@
+.editorconfig
+.codeclimate.yml
+*.png
+*.md

.gitignore

@@ -1,12 +1,17 @@
 .idea/
 
 .venv
+env/
 pip-selfcheck.json
 *.pyc
 __pycache__
+build
+dist
+*.egg-info
+.eggs
 
 config.yaml
 registration.yaml
+*.log*
 *.db
-*.session
-*.json
+*.bak

.gitlab-ci.yml

@@ -0,0 +1,38 @@
+image: docker:stable
+
+stages:
+- build
+- push
+
+default:
+  before_script:
+  - docker login -u $CI_REGISTRY_USER -p $CI_REGISTRY_PASSWORD $CI_REGISTRY
+
+build:
+  stage: build
+  script:
+  - docker pull $CI_REGISTRY_IMAGE:latest || true
+  - docker build --pull --cache-from $CI_REGISTRY_IMAGE:latest --tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA .
+  - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA
+
+push latest:
+  stage: push
+  only:
+  - master
+  variables:
+    GIT_STRATEGY: none
+  script:
+  - docker pull $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA
+  - docker tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA $CI_REGISTRY_IMAGE:latest
+  - docker push $CI_REGISTRY_IMAGE:latest
+
+push tag:
+  stage: push
+  variables:
+    GIT_STRATEGY: none
+  except:
+  - master
+  script:
+  - docker pull $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA
+  - docker tag $CI_REGISTRY_IMAGE:$CI_COMMIT_SHA $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME
+  - docker push $CI_REGISTRY_IMAGE:$CI_COMMIT_REF_NAME

Dockerfile

@@ -1,30 +1,44 @@
-FROM docker.io/alpine:3.7
+FROM docker.io/alpine:3.10
 
 ENV UID=1337 \
-    GID=1337
+    GID=1337 \
+    FFMPEG_BINARY=/usr/bin/ffmpeg
 
-COPY . /opt/mautrixtelegram
+COPY . /opt/mautrix-telegram
+WORKDIR /opt/mautrix-telegram
 RUN apk add --no-cache \
-      python3-dev \
       py3-virtualenv \
       py3-pillow \
       py3-aiohttp \
-      py3-lxml \
       py3-magic \
-      py3-numpy \
-      py3-asn1crypto \
       py3-sqlalchemy \
+      py3-markdown \
+      py3-psycopg2 \
+      py3-ruamel.yaml \
+      # Indirect dependencies
+      #commonmark
+        py3-future \
+      #alembic
+        py3-mako \
+        py3-dateutil \
+        py3-markupsafe \
+      #moviepy
+        py3-decorator \
+        #py3-tqdm \
+        py3-requests \
+        #imageio
+          py3-numpy \
+      #telethon
+        py3-rsa \
+      # Other dependencies
+      python3-dev \
+      libffi-dev \
       build-base \
       ffmpeg \
-      bash \
       ca-certificates \
       su-exec \
-      s6 \
- && cd /opt/mautrixtelegram \
- && cp -r docker/root/* / \
- && rm docker -rf \
- && pip3 install -r requirements.txt -r optional-requirements.txt
+ && pip3 install .[all]
 
 VOLUME /data
 
-CMD ["/bin/s6-svscan", "/etc/s6.d"]
+CMD ["/opt/mautrix-telegram/docker-run.sh"]

ROADMAP.md

@@ -3,10 +3,11 @@
 * Matrix → Telegram
   * [x] Message content (text, formatting, files, etc..)
   * [x] Message redactions
+  * [x] Message edits
   * [ ] ‡ Message history
-  * [ ] † Presence
-  * [ ] † Typing notifications
-  * [ ] † Read receipts
+  * [x] Presence
+  * [x] Typing notifications
+  * [x] Read receipts
   * [x] Pinning messages
   * [x] Power level
     * [x] Normal chats
@@ -21,6 +22,10 @@
     * [ ] ‡ Changes to displayname/avatar
 * Telegram → Matrix
   * [x] Message content (text, formatting, files, etc..)
+  * [ ] Advanced message content/media
+    * [x] Polls
+	* [x] Games
+	* [ ] Buttons
   * [x] Message deletions
   * [x] Message edits
   * [ ] Message history
@@ -46,8 +51,10 @@
     * [x] When receiving invite or message
   * [x] Private chat creation by inviting Matrix puppet of Telegram user to new room
   * [x] Option to use bot to relay messages for unauthenticated Matrix users
-  * [ ] Option to use own Matrix account for messages sent from other Telegram clients
+  * [x] Option to use own Matrix account for messages sent from other Telegram clients
   * [ ] ‡ Calls (hard, not yet supported by Telethon)
+  * [ ] ‡ Secret chats (not yet supported by Telethon)
+  * [ ] ‡ E2EE in Matrix rooms (not yet supported 
 
 † Information not automatically sent from source, i.e. implementation may not be possible  
 ‡ Maybe, i.e. this feature may or may not be implemented at some point

alembic.ini

@@ -35,9 +35,6 @@ script_location = alembic
 # are written from script.py.mako
 # output_encoding = utf-8
 
-sqlalchemy.url = sqlite:///mautrix-telegram.db
-
-
 # Logging configuration
 [loggers]
 keys = root,sqlalchemy,alembic

alembic/env.py

@@ -1,19 +1,35 @@
-from __future__ import with_statement
 from alembic import context
 from sqlalchemy import engine_from_config, pool
 from logging.config import fileConfig
 
 import sys
 from os.path import abspath, dirname
+
 sys.path.insert(0, dirname(dirname(abspath(__file__))))
 
-from mautrix_telegram.base import Base
-import mautrix_telegram.db
+from mautrix_telegram.db import Base
+from mautrix_telegram.config import Config
+from alchemysession import AlchemySessionContainer
 
 # this is the Alembic Config object, which provides
 # access to the values within the .ini file in use.
 config = context.config
 
+mxtg_config_path = context.get_x_argument(as_dictionary=True).get("config", "config.yaml")
+mxtg_config = Config(mxtg_config_path, None, None)
+mxtg_config.load()
+config.set_main_option("sqlalchemy.url",
+                       mxtg_config.get("appservice.database", "sqlite:///mautrix-telegram.db"))
+
+
+class FakeDB:
+    @staticmethod
+    def query_property():
+        return None
+
+
+AlchemySessionContainer.create_table_classes(FakeDB(), "telethon_", Base)
+
 # Interpret the config file for Python logging.
 # This line sets up loggers basically.
 fileConfig(config.config_file_name)
@@ -24,6 +40,7 @@ fileConfig(config.config_file_name)
 # target_metadata = mymodel.Base.metadata
 target_metadata = Base.metadata
 
+
 # other values from the config, defined by the needs of env.py,
 # can be acquired:
 # my_important_option = config.get_main_option("my_important_option")
@@ -71,6 +88,7 @@ def run_migrations_online():
         with context.begin_transaction():
             context.run_migrations()
 
+
 if context.is_offline_mode():
     run_migrations_offline()
 else:

alembic/versions/17574c57f3f8_add_disable_updates_field_for_puppets.py

@@ -0,0 +1,27 @@
+"""Add disable_updates field for puppets
+
+Revision ID: 17574c57f3f8
+Revises: a9119be92164
+Create Date: 2019-05-15 00:24:46.967529
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '17574c57f3f8'
+down_revision = 'a9119be92164'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    with op.batch_alter_table("puppet") as batch_op:
+        batch_op.add_column(sa.Column("disable_updates", sa.Boolean(), nullable=False,
+                                      server_default=sa.sql.expression.false()))
+
+
+def downgrade():
+    with op.batch_alter_table("puppet") as batch_op:
+        batch_op.drop_column("disable_updates")

alembic/versions/1fa46383a9d3_add_is_bot_field_to_puppets.py

@@ -17,8 +17,10 @@ depends_on = None
 
 
 def upgrade():
-    op.add_column('puppet', sa.Column('is_bot', sa.Boolean(), nullable=True))
+    with op.batch_alter_table("puppet") as batch_op:
+        batch_op.add_column(sa.Column('is_bot', sa.Boolean(), nullable=True))
 
 
 def downgrade():
-    op.drop_column('puppet', 'is_bot')
+    with op.batch_alter_table("puppet") as batch_op:
+        batch_op.drop_column('is_bot')

alembic/versions/2228d49c383f_add_cascade_rules_to_userportal.py

@@ -0,0 +1,41 @@
+"""Add cascade rules to UserPortal
+
+Revision ID: 2228d49c383f
+Revises: bcfefa1f1299
+Create Date: 2018-05-31 11:11:59.482112
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = '2228d49c383f'
+down_revision = 'bcfefa1f1299'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    try:
+        with op.batch_alter_table("user_portal") as batch_op:
+            batch_op.drop_constraint("user_portal_user_fkey", type_="foreignkey")
+            batch_op.drop_constraint("user_portal_portal_fkey", type_="foreignkey")
+            batch_op.create_foreign_key("user_portal_user_fkey", "user", ["user"], ["tgid"],
+                                        onupdate="CASCADE", ondelete="CASCADE")
+            batch_op.create_foreign_key("user_portal_portal_fkey", "portal",
+                                        ["portal", "portal_receiver"], ["tgid", "tg_receiver"],
+                                        onupdate="CASCADE", ondelete="CASCADE")
+    except ValueError:
+        return
+
+
+def downgrade():
+    try:
+        with op.batch_alter_table("user_portal") as batch_op:
+                batch_op.drop_constraint("user_portal_user_fkey", type_="foreignkey")
+                batch_op.drop_constraint("user_portal_portal_fkey", type_="foreignkey")
+                batch_op.create_foreign_key("user_portal_user_fkey", "portal",
+                                            ["portal", "portal_receiver"], ["tgid", "tg_receiver"])
+                batch_op.create_foreign_key("user_portal_portal_fkey", "user", ["user"], ["tgid"])
+    except ValueError:
+        return

alembic/versions/30eca60587f1_add_megagroup_field_to_portals.py

@@ -16,8 +16,10 @@ depends_on = None
 
 
 def upgrade():
-    op.add_column('portal', sa.Column('megagroup', sa.Boolean()))
+    with op.batch_alter_table("portal") as batch_op:
+        batch_op.add_column(sa.Column('megagroup', sa.Boolean()))
 
 
 def downgrade():
-    op.drop_column('portal', 'megagroup')
+    with op.batch_alter_table("portal") as batch_op:
+        batch_op.drop_column('megagroup')

alembic/versions/501dad2868bc_move_sessions_to_main_database.py

@@ -19,31 +19,31 @@ depends_on = None
 
 def upgrade():
     Session = op.create_table('telethon_sessions',
-                              sa.Column('session_id', sa.VARCHAR(), nullable=False),
-                              sa.Column('dc_id', sa.INTEGER(), nullable=False),
-                              sa.Column('server_address', sa.VARCHAR(), nullable=True),
-                              sa.Column('port', sa.INTEGER(), nullable=True),
-                              sa.Column('auth_key', sa.BLOB(), nullable=True),
+                              sa.Column('session_id', sa.String, nullable=False),
+                              sa.Column('dc_id', sa.Integer, nullable=False),
+                              sa.Column('server_address', sa.String, nullable=True),
+                              sa.Column('port', sa.Integer, nullable=True),
+                              sa.Column('auth_key', sa.LargeBinary, nullable=True),
                               sa.PrimaryKeyConstraint('session_id', 'dc_id'))
     SentFile = op.create_table('telethon_sent_files',
-                               sa.Column('session_id', sa.VARCHAR(), nullable=False),
-                               sa.Column('md5_digest', sa.BLOB(), nullable=False),
-                               sa.Column('file_size', sa.INTEGER(), nullable=False),
-                               sa.Column('type', sa.INTEGER(), nullable=False),
-                               sa.Column('id', sa.INTEGER(), nullable=True),
-                               sa.Column('hash', sa.INTEGER(), nullable=True),
+                               sa.Column('session_id', sa.String, nullable=False),
+                               sa.Column('md5_digest', sa.LargeBinary, nullable=False),
+                               sa.Column('file_size', sa.Integer, nullable=False),
+                               sa.Column('type', sa.Integer, nullable=False),
+                               sa.Column('id', sa.BigInteger, nullable=True),
+                               sa.Column('hash', sa.BigInteger, nullable=True),
                                sa.PrimaryKeyConstraint('session_id', 'md5_digest', 'file_size',
                                                        'type'))
     Entity = op.create_table('telethon_entities',
-                             sa.Column('session_id', sa.VARCHAR(), nullable=False),
-                             sa.Column('id', sa.INTEGER(), nullable=False),
-                             sa.Column('hash', sa.INTEGER(), nullable=False),
-                             sa.Column('username', sa.VARCHAR(), nullable=True),
-                             sa.Column('phone', sa.INTEGER(), nullable=True),
-                             sa.Column('name', sa.VARCHAR(), nullable=True),
+                             sa.Column('session_id', sa.String, nullable=False),
+                             sa.Column('id', sa.Integer, nullable=False),
+                             sa.Column('hash', sa.Integer, nullable=False),
+                             sa.Column('username', sa.String, nullable=True),
+                             sa.Column('phone', sa.Integer, nullable=True),
+                             sa.Column('name', sa.String, nullable=True),
                              sa.PrimaryKeyConstraint('session_id', 'id'))
     Version = op.create_table('telethon_version',
-                              sa.Column('version', sa.INTEGER(), nullable=False),
+                              sa.Column('version', sa.Integer, nullable=False),
                               sa.PrimaryKeyConstraint('version'))
     conn = op.get_bind()
     sessions = [os.path.basename(f) for f in os.listdir(".") if f.endswith(".session")]

alembic/versions/6ca3d74d51e4_move_state_store_to_main_database.py

@@ -0,0 +1,137 @@
+"""Move state store to main database
+
+Revision ID: 6ca3d74d51e4
+Revises: 2228d49c383f
+Create Date: 2018-06-26 21:31:26.911307
+
+"""
+from alembic import context, op
+import sqlalchemy.orm as orm
+import sqlalchemy as sa
+import json
+import re
+
+from mautrix_telegram.config import Config
+from mautrix_telegram.db import Base
+
+# revision identifiers, used by Alembic.
+revision = "6ca3d74d51e4"
+down_revision = "2228d49c383f"
+branch_labels = None
+depends_on = None
+
+
+class RoomState(Base):
+    query = None
+    __tablename__ = "mx_room_state"
+    __table_args__ = {"extend_existing": True}
+
+    room_id = sa.Column(sa.String, primary_key=True)
+    power_levels = sa.Column("power_levels", sa.Text, nullable=True)
+
+
+class UserProfile(Base):
+    query = None
+    __tablename__ = "mx_user_profile"
+    __table_args__ = {"extend_existing": True}
+
+    room_id = sa.Column(sa.String, primary_key=True)
+    user_id = sa.Column(sa.String, primary_key=True)
+    membership = sa.Column(sa.String, nullable=False, default="leave")
+    displayname = sa.Column(sa.String, nullable=True)
+    avatar_url = sa.Column(sa.String, nullable=True)
+
+
+class Puppet(Base):
+    query = None
+    __tablename__ = "puppet"
+    __table_args__ = {"extend_existing": True}
+
+    id = sa.Column(sa.Integer, primary_key=True)
+    displayname = sa.Column(sa.String, nullable=True)
+    displayname_source = sa.Column(sa.Integer, nullable=True)
+    username = sa.Column(sa.String, nullable=True)
+    photo_id = sa.Column(sa.String, nullable=True)
+    is_bot = sa.Column(sa.Boolean, nullable=True)
+    matrix_registered = sa.Column(sa.Boolean, nullable=False, default=False)
+
+
+def upgrade():
+    with op.batch_alter_table("puppet") as batch_op:
+        batch_op.add_column(sa.Column("matrix_registered", sa.Boolean(), nullable=False,
+                                      server_default=sa.sql.expression.false()))
+    op.create_table("mx_room_state",
+                    sa.Column("room_id", sa.String(), nullable=False),
+                    sa.Column("power_levels", sa.Text(), nullable=True),
+                    sa.PrimaryKeyConstraint("room_id"))
+    op.create_table("mx_user_profile",
+                    sa.Column("room_id", sa.String(), nullable=False),
+                    sa.Column("user_id", sa.String(), nullable=False),
+                    sa.Column("membership", sa.String(), nullable=False,
+                              default="leave"),
+                    sa.Column("displayname", sa.String(), nullable=True),
+                    sa.Column("avatar_url", sa.String(), nullable=True),
+                    sa.PrimaryKeyConstraint("room_id", "user_id"))
+
+    try:
+        migrate_state_store()
+    except Exception as e:
+        print("Failed to migrate state store:", e)
+        print("Migrating the state store isn't required, but you can retry by alembic downgrading "
+              "to revision 2228d49c383f and upgrading again.")
+
+
+def migrate_state_store():
+    conn = op.get_bind()
+    session = orm.sessionmaker(bind=conn)()  # type: orm.Session
+
+    try:
+        with open("mx-state.json") as file:
+            data = json.load(file)
+    except FileNotFoundError:
+        return
+    if not data:
+        return
+    registrations = data.get("registrations", [])
+
+    mxtg_config_path = context.get_x_argument(as_dictionary=True).get("config", "config.yaml")
+    mxtg_config = Config(mxtg_config_path, None, None)
+    mxtg_config.load()
+
+    username_template = mxtg_config.get("bridge.username_template", "telegram_{userid}")
+    hs_domain = mxtg_config["homeserver.domain"]
+    localpart = username_template.format(userid="(.+)")
+    mxid_regex = re.compile("@{}:{}".format(localpart, hs_domain))
+    for user in registrations:
+        match = mxid_regex.match(user)
+        if not match:
+            continue
+
+        puppet = session.query(Puppet).get(match.group(1))
+        if not puppet:
+            continue
+
+        puppet.matrix_registered = True
+        session.merge(puppet)
+    session.commit()
+
+    user_profiles = [UserProfile(room_id=room, user_id=user,
+                                 membership=member.get("membership", "leave"),
+                                 displayname=member.get("displayname", None),
+                                 avatar_url=member.get("avatar_url", None))
+                     for room, members in data.get("members", {}).items()
+                     for user, member in members.items()]
+    session.add_all(user_profiles)
+    session.commit()
+
+    room_state = [RoomState(room_id=room, power_levels=json.dumps(levels))
+                  for room, levels in data.get("power_levels", {}).items()]
+    session.add_all(room_state)
+    session.commit()
+
+
+def downgrade():
+    op.drop_table("mx_user_profile")
+    op.drop_table("mx_room_state")
+    with op.batch_alter_table("puppet") as batch_op:
+        batch_op.drop_column("matrix_registered")

alembic/versions/7d47d84380b6_add_timestamp_to_telegramfile.py

@@ -17,9 +17,10 @@ depends_on = None
 
 def upgrade():
     op.add_column('telegram_file',
-                  sa.Column('timestamp', sa.BigInteger(), nullable=False, default=0,
-                            server_default="true"))
+                  sa.Column('timestamp', sa.BigInteger(), nullable=True, default=0,
+                            server_default="0"))
 
 
 def downgrade():
-    op.drop_column('telegram_file', 'timestamp')
+    with op.batch_alter_table("telegram_file") as batch_op:
+        batch_op.drop_column('timestamp')

alembic/versions/97d2a942bcf8_initial_revision.py

@@ -29,7 +29,7 @@ def upgrade():
                     sa.UniqueConstraint('mxid'))
     op.create_table('user',
                     sa.Column('mxid', sa.String),
-                    sa.Column('tgid', sa.Integer, nullable=True),
+                    sa.Column('tgid', sa.Integer, nullable=True, unique=True),
                     sa.Column('tg_username', sa.String, nullable=True),
                     sa.Column('saved_contacts', sa.Integer, nullable=False, default=0),
                     sa.PrimaryKeyConstraint('mxid'))
@@ -50,9 +50,13 @@ def upgrade():
                     sa.Column('portal', sa.Integer),
                     sa.Column('portal_receiver', sa.Integer),
                     sa.PrimaryKeyConstraint('user', 'portal', 'portal_receiver'),
-                    sa.ForeignKeyConstraint(("user",), ("user.tgid",)),
+                    sa.ForeignKeyConstraint(("user",), ("user.tgid",),
+                                            name="user_portal_user_fkey",
+                                            onupdate="CASCADE", ondelete="CASCADE"),
                     sa.ForeignKeyConstraint(("portal", "portal_receiver"),
-                                            ("portal.tgid", "portal.tg_receiver")))
+                                            ("portal.tgid", "portal.tg_receiver"),
+                                            name="user_portal_portal_fkey",
+                                            onupdate="CASCADE", ondelete="CASCADE"))
     op.create_table('message',
                     sa.Column('mxid', sa.String),
                     sa.Column('mx_room', sa.String),

alembic/versions/9e9c89b0b877_add_edit_index_to_messages.py

@@ -0,0 +1,48 @@
+"""Add edit index to messages
+
+Revision ID: 9e9c89b0b877
+Revises: 17574c57f3f8
+Create Date: 2019-05-29 15:28:23.128377
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = '9e9c89b0b877'
+down_revision = '17574c57f3f8'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.create_table('_message_temp',
+                    sa.Column('mxid', sa.String),
+                    sa.Column('mx_room', sa.String),
+                    sa.Column('tgid', sa.Integer),
+                    sa.Column('tg_space', sa.Integer),
+                    sa.Column('edit_index', sa.Integer),
+                    sa.PrimaryKeyConstraint('tgid', 'tg_space', 'edit_index'),
+                    sa.UniqueConstraint("mxid", "mx_room", "tg_space", name="_mx_id_room_2"))
+    c = op.get_bind()
+    c.execute("INSERT INTO _message_temp (mxid, mx_room, tgid, tg_space, edit_index) "
+              "SELECT message.mxid, message.mx_room, message.tgid, message.tg_space, 0 "
+              "FROM message")
+    c.execute("DROP TABLE message")
+    c.execute("ALTER TABLE _message_temp RENAME TO message")
+
+
+def downgrade():
+    op.create_table('_message_temp',
+                    sa.Column('mxid', sa.String),
+                    sa.Column('mx_room', sa.String),
+                    sa.Column('tgid', sa.Integer),
+                    sa.Column('tg_space', sa.Integer),
+                    sa.PrimaryKeyConstraint('tgid', 'tg_space'),
+                    sa.UniqueConstraint("mxid", "mx_room", "tg_space", name="_mx_id_room"))
+    c = op.get_bind()
+    c.execute("INSERT INTO _message_temp (mxid, mx_room, tgid, tg_space) "
+              "SELECT message.mxid, message.mx_room, message.tgid, message.tg_space "
+              "FROM message")
+    c.execute("DROP TABLE message")
+    c.execute("ALTER TABLE _message_temp RENAME TO message")

alembic/versions/a9119be92164_add_phone_number_field_to_users.py

@@ -0,0 +1,25 @@
+"""Add phone number field to users
+
+Revision ID: a9119be92164
+Revises: b54929c22c86
+Create Date: 2018-09-28 02:38:40.626282
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = "a9119be92164"
+down_revision = "b54929c22c86"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.add_column("user", sa.Column("tg_phone", sa.String(), nullable=True))
+
+
+def downgrade():
+    with op.batch_alter_table("user") as batch_op:
+        batch_op.drop_column("tg_phone")

alembic/versions/b54929c22c86_add_portal_specific_config.py

@@ -0,0 +1,25 @@
+"""Add portal-specific config
+
+Revision ID: b54929c22c86
+Revises: d5f7b8b4b456
+Create Date: 2018-09-24 23:40:33.528710
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = "b54929c22c86"
+down_revision = "d5f7b8b4b456"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.add_column("portal", sa.Column("config", sa.Text(), nullable=True))
+
+
+def downgrade():
+    with op.batch_alter_table("portal") as batch_op:
+        batch_op.drop_column("config")

alembic/versions/bcfefa1f1299_add_displayname_source_fields_for_.py

@@ -20,4 +20,5 @@ def upgrade():
 
 
 def downgrade():
-    op.drop_column('puppet', 'displayname_source')
+    with op.batch_alter_table("puppet") as batch_op:
+        batch_op.drop_column('displayname_source')

alembic/versions/d5f7b8b4b456_add_access_token_and_custom_mxid_fields_.py

@@ -0,0 +1,26 @@
+"""Add access_token and custom_mxid fields for puppets
+
+Revision ID: d5f7b8b4b456
+Revises: 6ca3d74d51e4
+Create Date: 2018-07-20 12:09:30.277960
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+# revision identifiers, used by Alembic.
+revision = "d5f7b8b4b456"
+down_revision = "6ca3d74d51e4"
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    op.add_column("puppet", sa.Column("access_token", sa.String(), nullable=True))
+    op.add_column("puppet", sa.Column("custom_mxid", sa.String(), nullable=True))
+
+
+def downgrade():
+    with op.batch_alter_table("puppet") as batch_op:
+        batch_op.drop_column("custom_mxid")
+        batch_op.drop_column("access_token")

docker-run.sh

@@ -1,23 +1,22 @@
-#!/bin/bash
+#!/bin/sh
 
-# Define functions
+# Define functions.
 function fixperms {
-	chown -R ${UID}:${GID} /data /opt/mautrixtelegram
+	chown -R $UID:$GID /data /opt/mautrix-telegram
 }
 
+cd /opt/mautrix-telegram
 
-# Go into env
-cd /opt/mautrixtelegram
-export FFMPEG_BINARY=/usr/bin/ffmpeg
-
-# Replace database path in alembic.ini
-sed -i "s#sqlite:///mautrix-telegram.db#sqlite:////data/mautrix-telegram.db#" alembic.ini
+# Replace database path in config.
 sed -i "s#sqlite:///mautrix-telegram.db#sqlite:////data/mautrix-telegram.db#" /data/config.yaml
 
+if [ -f /data/mx-state.json ]; then
+	ln -s /data/mx-state.json
+fi
 # Check that database is in the right state
-alembic upgrade head
+alembic -x config=/data/config.yaml upgrade head
 
-if [[ ! -f /data/config.yaml ]]; then
+if [ ! -f /data/config.yaml ]; then
 	cp example-config.yaml /data/config.yaml
 	echo "Didn't find a config file."
 	echo "Copied default config file to /data/config.yaml"
@@ -27,14 +26,14 @@ if [[ ! -f /data/config.yaml ]]; then
 	exit
 fi
 
-if [[ ! -f /data/registration.yaml ]]; then
+if [ ! -f /data/registration.yaml ]; then
 	python3 -m mautrix_telegram -g -c /data/config.yaml -r /data/registration.yaml
 	echo "Didn't find a registration file."
-	echo "Generated ode for you."
+	echo "Generated one for you."
 	echo "Copy that over to synapses app service directory."
 	fixperms
 	exit
 fi
 
 fixperms
-exec su-exec ${UID}:${GID} python3 -m mautrix_telegram -c /data/config.yaml
+exec su-exec $UID:$GID python3 -m mautrix_telegram -c /data/config.yaml

docker/root/etc/s6.d/.s6-svscan/finish

@@ -1 +0,0 @@
-#!/bin/sh

docker/root/etc/s6.d/mautrix-telegram/finish

@@ -1,2 +0,0 @@
-#!/bin/bash
-s6-svscanctl -t /etc/s6.d

example-config.yaml

@@ -11,15 +11,21 @@ homeserver:
 # Application service host/registration related details
 # Changing these values requires regeneration of the registration.
 appservice:
-    # The protocol the homeserver should use when connecting to this appservice.
-    # Usually "http" or "https".
-    protocol: http
+    # The address that the homeserver can use to connect to this appservice.
+    address: http://localhost:8080
 
-    # The hostname and port where the homeserver can find this appservice.
-    hostname: localhost
+    # The hostname and port where this appservice should listen.
+    hostname: 0.0.0.0
     port: 8080
+    # The maximum body size of appservice API requests (from the homeserver) in mebibytes
+    # Usually 1 is enough, but on high-traffic bridges you might need to increase this to avoid 413s
+    max_body_size: 1
 
-    # The full URI to the database.
+    # The full URI to the database. SQLite and Postgres are fully supported.
+    # Other DBMSes supported by SQLAlchemy may or may not work.
+    # Format examples:
+    #   SQLite:   sqlite:///filename.db
+    #   Postgres: postgres://username:password@hostname/dbname
     database: sqlite:///mautrix-telegram.db
 
     # Public part of web server for out-of-Matrix interaction with the bridge.
@@ -34,19 +40,39 @@ appservice:
         # implicitly.
         external: https://example.com/public
 
-    # Whether or not to enable debug messages in the console.
-    debug: true
+    # Provisioning API part of the web server for automated portal creation and fetching information.
+    # Used by things like Dimension (https://dimension.t2bot.io/).
+    provisioning:
+        # Whether or not the provisioning API should be enabled.
+        enabled: true
+        # The prefix to use in the provisioning API endpoints.
+        prefix: /_matrix/provision/v1
+        # The shared secret to authorize users of the API.
+        # Set to "generate" to generate and save a new token.
+        shared_secret: generate
 
     # The unique ID of this appservice.
     id: telegram
     # Username of the appservice bot.
     bot_username: telegrambot
+    # Display name and avatar for bot. Set to "remove" to remove display name/avatar, leave empty
+    # to leave display name/avatar as-is.
     bot_displayname: Telegram bridge bot
+    bot_avatar: mxc://maunium.net/tJCRmUyJDsgRNgqhOgoiHWbX
+
+    # Community ID for bridged users (changes registration file) and rooms.
+    # Must be created manually.
+    community_id: false
 
     # Authentication tokens for AS <-> HS communication. Autogenerated; do not modify.
     as_token: "This value is generated when generating the registration"
     hs_token: "This value is generated when generating the registration"
 
+# Prometheus telemetry config. Requires prometheus-client to be installed.
+metrics:
+    enabled: false
+    listen_port: 8000
+
 # Bridge config
 bridge:
     # Localpart template of MXIDs for Telegram users.
@@ -78,48 +104,116 @@ bridge:
     - username
     - phone number
 
-    # Show message editing as a reply to the original message.
-    # If this is false, message edits are not shown at all, as Matrix does not support editing yet.
-    edits_as_replies: false
-    # Highlight changed/added parts in edits. Requires lxml.
-    highlight_edits: false
-    # Whether or not Matrix bot messages (type m.notice) should be bridged.
-    bridge_notices: true
-    # Whether to bridge Telegram bot messages as m.notices or m.texts.
-    bot_messages_as_notices: true
     # Maximum number of members to sync per portal when starting up. Other members will be
     # synced when they send messages. The maximum is 10000, after which the Telegram server
     # will not send any more members.
     # Defaults to no local limit (-> limited to 10000 by server)
     max_initial_member_sync: -1
+    # Whether or not to sync the member list in channels.
+    # If no channel admins have logged into the bridge, the bridge won't be able to sync the member
+    # list regardless of this setting.
+    sync_channel_members: true
+    # Whether or not to skip deleted members when syncing members.
+    skip_deleted_members: true
+    # Whether or not to automatically synchronize contacts and chats of Matrix users logged into
+    # their Telegram account at startup.
+    startup_sync: true
+    # Number of most recently active dialogs to check when syncing chats.
+    # Dialogs include groups and private chats, but only groups are synced.
+    # Set to 0 to remove limit.
+    sync_dialog_limit: 30
     # The maximum number of simultaneous Telegram deletions to handle.
     # A large number of simultaneous redactions could put strain on your homeserver.
     max_telegram_delete: 10
+    # Whether or not to automatically sync the Matrix room state (mostly unpuppeted displaynames)
+    # at startup and when creating a bridge.
+    sync_matrix_state: true
     # Allow logging in within Matrix. If false, the only way to log in is using the out-of-Matrix
     # login website (see appservice.public config section)
     allow_matrix_login: true
-    # Use inline images instead of m.image to make rich captions possible.
-    # N.B. Inline images are not supported on all clients (e.g. Riot iOS).
-    inline_images: false
     # Whether or not to bridge plaintext highlights.
     # Only enable this if your displayname_template has some static part that the bridge can use to
     # reliably identify what is a plaintext highlight.
     plaintext_highlights: false
     # Whether or not to make portals of publicly joinable channels/supergroups publicly joinable on Matrix.
     public_portals: true
-    # Whether to send stickers as the new native m.sticker type or normal m.images.
-    # Old versions of Riot don't support the new type at all.
-    # Remember that proper sticker support always requires Pillow to convert webp into png.
-    native_stickers: true
     # Whether or not to fetch and handle Telegram updates at startup from the time the bridge was down.
-    # WARNING: Probably buggy, might get stuck in infinite loop.
+    # Currently only works for private chats and normal groups.
+    # WARNING: This feature seems to be broken in the telegram library.
     catch_up: false
+    # Whether or not to use /sync to get presence, read receipts and typing notifications when using
+    # your own Matrix account as the Matrix puppet for your Telegram account.
+    sync_with_custom_puppets: true
+    # Set to false to disable link previews in messages sent to Telegram.
+    telegram_link_preview: true
+    # Use inline images instead of a separate message for the caption.
+    # N.B. Inline images are not supported on all clients (e.g. Riot iOS).
+    inline_images: false
+    # Maximum size of image in megabytes before sending to Telegram as a document.
+    image_as_file_size: 10
+    # Maximum size of Telegram documents in megabytes to bridge.
+    max_document_size: 100
 
+    # Whether to bridge Telegram bot messages as m.notices or m.texts.
+    bot_messages_as_notices: true
+    bridge_notices:
+        # Whether or not Matrix bot messages (type m.notice) should be bridged.
+        default: false
+        # List of user IDs for whom the previous flag is flipped.
+        # e.g. if bridge_notices.default is false, notices from other users will not be bridged, but
+        #      notices from users listed here will be bridged.
+        exceptions:
+        - "@importantbot:example.com"
+
+    # Some config options related to Telegram message deduplication.
+    # The default values are usually fine, but some debug messages/warnings might recommend you
+    # change these.
+    deduplication:
+        # Whether or not to check the database if the message about to be sent is a duplicate.
+        pre_db_check: false
+        # The number of latest events to keep when checking for duplicates.
+        # You might need to increase this on high-traffic bridge instances.
+        cache_queue_length: 20
+
+
+    # The formats to use when sending messages to Telegram via the relay bot.
+    #
+    # Telegram doesn't have built-in emotes, so the m.emote format is also used for non-relaybot users.
+    #
+    # Available variables:
+    #   $sender_displayname    - The display name of the sender (e.g. Example User)
+    #   $sender_username       - The username (Matrix ID localpart) of the sender (e.g. exampleuser)
+    #   $sender_mxid           - The Matrix ID of the sender (e.g. @exampleuser:example.com)
+    #   $message               - The message content as HTML
+    message_formats:
+        m.text: "<b>$sender_displayname</b>: $message"
+        m.emote: "* <b>$sender_displayname</b> $message"
+        m.file: "<b>$sender_displayname</b> sent a file: $message"
+        m.image: "<b>$sender_displayname</b> sent an image: $message"
+        m.audio: "<b>$sender_displayname</b> sent an audio file: $message"
+        m.video: "<b>$sender_displayname</b> sent a video: $message"
+        m.location: "<b>$sender_displayname</b> sent a location: $message"
+
+    # The formats to use when sending state events to Telegram via the relay bot.
+    #
+    # Variables from `message_formats` that have the `sender_` prefix are available without the prefix.
+    # In name_change events, `$prev_displayname` is the previous displayname.
+    #
+    # Set format to an empty string to disable the messages for that event.
+    state_event_formats:
+        join: "<b>$displayname</b> joined the room."
+        leave: "<b>$displayname</b> left the room."
+        name_change: "<b>$prev_displayname</b> changed their name to <b>$displayname</b>"
+
+    # Filter rooms that can/can't be bridged. Can also be managed using the `filter` and
+    # `filter-mode` management commands.
+    #
+    # Filters do not affect direct chats.
+    # An empty blacklist will essentially disable the filter.
     filter:
         # Filter mode to use. Either "blacklist" or "whitelist".
-        # If the mode is "blacklist", the listed chats will never be bridged. An empty blacklist disables the filter.
+        # If the mode is "blacklist", the listed chats will never be bridged.
         # If the mode is "whitelist", only the listed chats can be bridged.
-        # Direct chats are not affected.
         mode: blacklist
         # The list of group/channel IDs to filter.
         list: []
@@ -130,7 +224,9 @@ bridge:
     # Permissions for using the bridge.
     # Permitted values:
     #   relaybot - Only use the bridge via the relaybot, no access to commands.
-    #       full - Full access to use the bridge via relaybot or logging in with Telegram account.
+    #       user - Relaybot level + access to commands to create bridges.
+    #  puppeting - User level + logging in with a Telegram account.
+    #       full - Full access to use the bridge, i.e. previous levels + Matrix login.
     #      admin - Full access to use the bridge and some extra administration commands.
     # Permitted keys:
     #        * - All Matrix users
@@ -138,8 +234,8 @@ bridge:
     #     mxid - Specific user
     permissions:
         "*": "relaybot"
+        "public.example.com": "user"
         "example.com": "full"
-        "public.example.com": "full"
         "@admin:example.com": "admin"
 
     # Options related to the message relay Telegram bot.
@@ -148,6 +244,8 @@ bridge:
         authless_portals: true
         # Whether or not to allow Telegram group admins to use the bot commands.
         whitelist_group_admins: true
+        # Whether or not to ignore incoming events sent by the relay bot.
+        ignore_own_incoming_events: true
         # List of usernames/user IDs who are also allowed to use the bot commands.
         whitelist:
         - myusername
@@ -160,3 +258,92 @@ telegram:
     api_hash: tjyd5yge35lbodk1xwzw2jstp90k55qz
     # (Optional) Create your own bot at https://t.me/BotFather
     bot_token: disabled
+
+    # Telethon connection options.
+    connection:
+        # The timeout in seconds to be used when connecting.
+        timeout: 120
+        # How many times the reconnection should retry, either on the initial connection or when
+        # Telegram disconnects us. May be set to a negative or null value for infinite retries, but
+        # this is not recommended, since the program can get stuck in an infinite loop.
+        retries: 5
+        # The delay in seconds to sleep between automatic reconnections.
+        retry_delay: 1
+        # The threshold below which the library should automatically sleep on flood wait errors
+        # (inclusive). For instance, if a FloodWaitError for 17s occurs and flood_sleep_threshold
+        # is 20s, the library will sleep automatically. If the error was for 21s, it would raise
+        # the error instead. Values larger than a day (86400) will be changed to a day.
+        flood_sleep_threshold: 60
+        # How many times a request should be retried. Request are retried when Telegram is having
+        # internal issues, when there is a FloodWaitError less than flood_sleep_threshold, or when
+        # there's a migrate error. May take a negative or null value for infinite retries, but this
+        # is not recommended, since some requests can always trigger a call fail (such as searching
+        # for messages).
+        request_retries: 5
+
+    # Device info sent to Telegram.
+    device_info:
+        # "auto" = OS name+version.
+        device_model: auto
+        # "auto" = Telethon version.
+        system_version: auto
+        # "auto" = mautrix-telegram version.
+        app_version: auto
+        lang_code: en
+        system_lang_code: en
+
+    # Custom server to connect to.
+    server:
+        # Set to true to use these server settings. If false, will automatically
+        # use production server assigned by Telegram. Set to false in production.
+        enabled: false
+        # The DC ID to connect to.
+        dc: 2
+        # The IP to connect to.
+        ip: 149.154.167.40
+        # The port to connect to. 443 may not work, 80 is better and both are equally secure.
+        port: 80
+
+    # Telethon proxy configuration.
+    # You must install PySocks from pip for proxies to work.
+    proxy:
+        # Allowed types: disabled, socks4, socks5, http
+        type: disabled
+        # Proxy IP address and port.
+        address: 127.0.0.1
+        port: 1080
+        # Whether or not to perform DNS resolving remotely.
+        rdns: true
+        # Proxy authentication (optional).
+        username: ""
+        password: ""
+
+# Python logging configuration.
+#
+# See section 16.7.2 of the Python documentation for more info:
+# https://docs.python.org/3.6/library/logging.config.html#configuration-dictionary-schema
+logging:
+    version: 1
+    formatters:
+        precise:
+            format: "[%(asctime)s] [%(levelname)s@%(name)s] %(message)s"
+    handlers:
+        file:
+            class: logging.handlers.RotatingFileHandler
+            formatter: precise
+            filename: ./mautrix-telegram.log
+            maxBytes: 10485760
+            backupCount: 10
+        console:
+            class: logging.StreamHandler
+            formatter: precise
+    loggers:
+        mau:
+            level: DEBUG
+        telethon:
+            level: DEBUG
+        aiohttp:
+            level: INFO
+    root:
+        level: DEBUG
+        handlers: [file, console]

mautrix_telegram/__init__.py

@@ -1,2 +1,2 @@
-__version__ = "0.2.0rc3"
+__version__ = "0.6.1rc2"
 __author__ = "Tulir Asokan <tulir@maunium.net>"

mautrix_telegram/__main__.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -14,36 +14,40 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Awaitable, List, Any
+from time import time
 import argparse
-import sys
-import logging
 import asyncio
+import logging.config
+import sys
+import copy
+import signal
+import os
 
 import sqlalchemy as sql
-from sqlalchemy import orm
 
-from alchemysession import AlchemySessionContainer
 from mautrix_appservice import AppService
+from alchemysession import AlchemySessionContainer
 
-from .base import Base
-from .config import Config, DictWithRecursion
-from .matrix import MatrixHandler
-
-from .db import init as init_db
+from .web.provisioning import ProvisioningAPI
+from .web.public import PublicBridgeWebsite
 from .abstract_user import init as init_abstract_user
-from .user import init as init_user, User
 from .bot import init as init_bot
+from .config import Config
+from .context import Context
+from .db import Base, init as init_db
+from .formatter import init as init_formatter
+from .matrix import MatrixHandler
 from .portal import init as init_portal
 from .puppet import init as init_puppet
-from .formatter import init as init_formatter
-from .public import PublicBridgeWebsite
-from .context import Context
+from .sqlstatestore import SQLStateStore
+from .user import User, init as init_user
+from . import __version__
 
-log = logging.getLogger("mau")
-time_formatter = logging.Formatter("[%(asctime)s] [%(levelname)s@%(name)s] %(message)s")
-handler = logging.StreamHandler()
-handler.setFormatter(time_formatter)
-log.addHandler(handler)
+try:
+    import prometheus_client as prometheus
+except ImportError:
+    prometheus = None
 
 parser = argparse.ArgumentParser(
     description="A Matrix-Telegram puppeting bridge.",
@@ -59,7 +63,7 @@ parser.add_argument("-r", "--registration", type=str, default="registration.yaml
                     metavar="<path>", help="the path to save the generated registration to")
 args = parser.parse_args()
 
-config = Config(args.config, args.registration, args.base_config)
+config = Config(args.config, args.registration, args.base_config, os.environ)
 config.load()
 config.update()
 
@@ -69,52 +73,91 @@ if args.generate_registration:
     print(f"Registration generated and saved to {config.registration_path}")
     sys.exit(0)
 
-if config["appservice.debug"]:
-    telethon_log = logging.getLogger("telethon")
-    telethon_log.addHandler(handler)
-    telethon_log.setLevel(logging.DEBUG)
-    log.setLevel(logging.DEBUG)
-    log.debug("Debug messages enabled.")
+logging.config.dictConfig(copy.deepcopy(config["logging"]))
+log = logging.getLogger("mau.init")  # type: logging.Logger
+log.debug(f"Initializing mautrix-telegram {__version__}")
 
-db_engine = sql.create_engine(config.get("appservice.database", "sqlite:///mautrix-telegram.db"))
-db_factory = orm.sessionmaker(bind=db_engine)
-db_session = orm.scoping.scoped_session(db_factory)
+db_engine = sql.create_engine(config["appservice.database"] or "sqlite:///mautrix-telegram.db")
 Base.metadata.bind = db_engine
 
-telethon_session_container = AlchemySessionContainer(engine=db_engine, session=db_session,
-                                                     table_base=Base, table_prefix="telethon_",
-                                                     manage_tables=False)
+session_container = AlchemySessionContainer(engine=db_engine, table_base=Base, session=False,
+                                            table_prefix="telethon_", manage_tables=False)
+session_container.core_mode = True
 
-loop = asyncio.get_event_loop()
+try:
+    import uvloop
 
+    asyncio.set_event_loop_policy(uvloop.EventLoopPolicy())
+    log.debug("Using uvloop for asyncio")
+except ImportError:
+    pass
+
+loop = asyncio.get_event_loop()  # type: asyncio.AbstractEventLoop
+
+state_store = SQLStateStore()
+mebibyte = 1024 ** 2
 appserv = AppService(config["homeserver.address"], config["homeserver.domain"],
                      config["appservice.as_token"], config["appservice.hs_token"],
                      config["appservice.bot_username"], log="mau.as", loop=loop,
-                     verify_ssl=config["homeserver.verify_ssl"])
-
-context = Context(appserv, db_session, config, loop, None, None, telethon_session_container)
+                     verify_ssl=config["homeserver.verify_ssl"], state_store=state_store,
+                     real_user_content_key="net.maunium.telegram.puppet",
+                     aiohttp_params={
+                         "client_max_size": config["appservice.max_body_size"] * mebibyte
+                     })
+bot = init_bot(config)
+context = Context(appserv, config, loop, session_container, bot)
 
 if config["appservice.public.enabled"]:
-    public = PublicBridgeWebsite(loop)
-    appserv.app.add_subapp(config.get("appservice.public.prefix", "/public"), public.app)
+    public_website = PublicBridgeWebsite(loop)
+    appserv.app.add_subapp(config["appservice.public.prefix"] or "/public", public_website.app)
+    context.public_website = public_website
+
+if config["appservice.provisioning.enabled"]:
+    provisioning_api = ProvisioningAPI(context)
+    appserv.app.add_subapp(config["appservice.provisioning.prefix"] or "/_matrix/provisioning",
+                           provisioning_api.app)
+    context.provisioning_api = provisioning_api
+
+context.mx = MatrixHandler(context)
+
+if config["metrics.enabled"]:
+    if prometheus:
+        prometheus.start_http_server(config["metrics.listen_port"])
+    else:
+        log.warn("Metrics are enabled in the config, but prometheus-async is not installed.")
 
 with appserv.run(config["appservice.hostname"], config["appservice.port"]) as start:
-    init_db(db_session)
+    start_ts = time()
+    init_db(db_engine)
     init_abstract_user(context)
-    context.bot = init_bot(context)
-    context.mx = MatrixHandler(context)
     init_formatter(context)
     init_portal(context)
-    init_puppet(context)
-    startup_actions = init_user(context) + [start, context.mx.init_as_bot()]
+    startup_actions = (init_puppet(context) +
+                       init_user(context) +
+                       [start, context.mx.init_as_bot()])  # type: List[Awaitable[Any]]
 
     if context.bot:
         startup_actions.append(context.bot.start())
 
+    signal.signal(signal.SIGINT, signal.default_int_handler)
+    signal.signal(signal.SIGTERM, signal.default_int_handler)
+
+    end_ts = time()
     try:
+        log.debug(f"Initialization complete in {round(end_ts - start_ts, 2)} seconds,"
+                  " running startup actions")
+        start_ts = time()
         loop.run_until_complete(asyncio.gather(*startup_actions, loop=loop))
+        end_ts = time()
+        log.debug(f"Startup actions complete in {round(end_ts - start_ts, 2)} seconds,"
+                  " now running forever")
         loop.run_forever()
     except KeyboardInterrupt:
-        for user in User.by_tgid.values():
-            user.stop()
+        log.debug("Interrupt received, stopping clients")
+        loop.run_until_complete(
+            asyncio.gather(*[user.stop() for user in User.by_tgid.values()], loop=loop))
+        log.debug("Clients stopped, shutting down")
         sys.exit(0)
+    except Exception as e:
+        log.exception("Unexpected error")
+        sys.exit(1)

mautrix_telegram/abstract_user.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -14,65 +14,164 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Tuple, Optional, List, Union, Dict, TYPE_CHECKING
+from abc import ABC, abstractmethod
+import asyncio
+import logging
 import platform
-import os
+import time
 
-from telethon.tl.types import *
-from mautrix_appservice import MatrixRequestError
+from telethon.tl.patched import MessageService, Message
+from telethon.tl.types import (
+    Channel, ChannelForbidden, Chat, ChatForbidden, MessageActionChannelMigrateFrom, PeerUser,
+    TypeUpdate, UpdateChannelPinnedMessage, UpdateChatPinnedMessage, UpdateChatParticipantAdmin,
+    UpdateChatParticipants, UpdateChatUserTyping, UpdateDeleteChannelMessages, UpdateDeleteMessages,
+    UpdateEditChannelMessage, UpdateEditMessage, UpdateNewChannelMessage, UpdateNewMessage,
+    UpdateReadHistoryOutbox, UpdateShortChatMessage, UpdateShortMessage, UpdateUserName,
+    UpdateUserPhoto, UpdateUserStatus, UpdateUserTyping, User, UserStatusOffline, UserStatusOnline)
+
+from mautrix_appservice import MatrixRequestError, AppService
+from alchemysession import AlchemySessionContainer
 
-from .tgclient import MautrixTelegramClient
-from .db import Message as DBMessage
 from . import portal as po, puppet as pu, __version__
+from .db import Message as DBMessage
+from .types import TelegramID, MatrixUserID
+from .tgclient import MautrixTelegramClient
 
-config = None
+if TYPE_CHECKING:
+    from .context import Context
+    from .config import Config
+    from .bot import Bot
+
+config = None  # type: Config
 # Value updated from config in init()
-MAX_DELETIONS = 10
+MAX_DELETIONS = 10  # type: int
 
+UpdateMessage = Union[UpdateShortChatMessage, UpdateShortMessage, UpdateNewChannelMessage,
+                      UpdateNewMessage, UpdateEditMessage, UpdateEditChannelMessage]
+UpdateMessageContent = Union[UpdateShortMessage, UpdateShortChatMessage, Message, MessageService]
 
-class AbstractUser:
-    session_container = None
-    loop = None
-    log = None
-    db = None
-    az = None
+try:
+    from prometheus_client import Histogram
 
-    def __init__(self):
-        self.connected = False
-        self.whitelisted = False
-        self.client = None
-        self.tgid = None
-        self.mxid = None
-        self.is_relaybot = False
+    UPDATE_TIME = Histogram("telegram_update", "Time spent processing Telegram updates",
+                            ["update_type"])
+except ImportError:
+    Histogram = None
+    UPDATE_TIME = None
 
-    async def _init_client(self):
+class AbstractUser(ABC):
+    session_container = None  # type: AlchemySessionContainer
+    loop = None  # type: asyncio.AbstractEventLoop
+    log = None  # type: logging.Logger
+    az = None  # type: AppService
+    bot = None  # type: Bot
+    ignore_incoming_bot_events = True  # type: bool
+
+    def __init__(self) -> None:
+        self.is_admin = False  # type: bool
+        self.matrix_puppet_whitelisted = False  # type: bool
+        self.puppet_whitelisted = False  # type: bool
+        self.whitelisted = False  # type: bool
+        self.relaybot_whitelisted = False  # type: bool
+        self.client = None  # type: MautrixTelegramClient
+        self.tgid = None  # type: TelegramID
+        self.mxid = None  # type: MatrixUserID
+        self.is_relaybot = False  # type: bool
+        self.is_bot = False  # type: bool
+        self.relaybot = None  # type: Optional[Bot]
+
+    @property
+    def connected(self) -> bool:
+        return self.client and self.client.is_connected()
+
+    @property
+    def _proxy_settings(self) -> Optional[Tuple[int, str, str, str, str, str]]:
+        proxy_type = config["telegram.proxy.type"].lower()
+        if proxy_type == "disabled":
+            return None
+        elif proxy_type == "socks4":
+            proxy_type = 1
+        elif proxy_type == "socks5":
+            proxy_type = 2
+        elif proxy_type == "http":
+            proxy_type = 3
+
+        return (proxy_type,
+                config["telegram.proxy.address"], config["telegram.proxy.port"],
+                config["telegram.proxy.rdns"],
+                config["telegram.proxy.username"], config["telegram.proxy.password"])
+
+    def _init_client(self) -> None:
         self.log.debug(f"Initializing client for {self.name}")
-        device = f"{platform.system()} {platform.release()}"
-        sysversion = MautrixTelegramClient.__version__
-        self.session = self.session_container.new_session(self.name)
-        self.client = MautrixTelegramClient(session=self.session,
-                                            api_id=config["telegram.api_id"],
-                                            api_hash=config["telegram.api_hash"],
-                                            loop=self.loop,
-                                            app_version=__version__,
-                                            system_version=sysversion,
-                                            device_model=device,
-                                            report_errors=False)
-        await self.client.add_event_handler(self._update_catch)
 
-    async def update(self, update):
+        self.session = self.session_container.new_session(self.name)
+        if config["telegram.server.enabled"]:
+            self.session.set_dc(config["telegram.server.dc"],
+                                config["telegram.server.ip"],
+                                config["telegram.server.port"])
+
+        if self.is_relaybot:
+            base_logger = logging.getLogger("telethon.relaybot")
+        else:
+            base_logger = logging.getLogger(f"telethon.{self.tgid or -hash(self.mxid)}")
+
+        device = config["telegram.device_info.device_model"]
+        sysversion = config["telegram.device_info.system_version"]
+        appversion = config["telegram.device_info.app_version"]
+
+        self.client = MautrixTelegramClient(
+            session=self.session,
+
+            api_id=config["telegram.api_id"],
+            api_hash=config["telegram.api_hash"],
+
+            app_version=__version__ if appversion == "auto" else appversion,
+            system_version=MautrixTelegramClient.__version__ if sysversion == "auto" else sysversion,
+            device_model=f"{platform.system()} {platform.release()}" if device == "auto" else device,
+
+            timeout=config["telegram.connection.timeout"],
+            connection_retries=config["telegram.connection.retries"],
+            retry_delay=config["telegram.connection.retry_delay"],
+            flood_sleep_threshold=config["telegram.connection.flood_sleep_threshold"],
+            request_retries=config["telegram.connection.request_retries"],
+
+            proxy=self._proxy_settings,
+
+            loop=self.loop,
+            base_logger=base_logger
+        )
+        self.client.add_event_handler(self._update_catch)
+
+    @abstractmethod
+    async def update(self, update: TypeUpdate) -> bool:
         return False
 
-    async def post_login(self):
+    @abstractmethod
+    async def post_login(self) -> None:
         raise NotImplementedError()
 
-    async def _update_catch(self, update):
+    @abstractmethod
+    def register_portal(self, portal: po.Portal) -> None:
+        raise NotImplementedError()
+
+    @abstractmethod
+    def unregister_portal(self, portal: po.Portal) -> None:
+        raise NotImplementedError()
+
+    async def _update_catch(self, update: TypeUpdate) -> None:
+        start_time = time.time()
         try:
             if not await self.update(update):
                 await self._update(update)
         except Exception:
             self.log.exception("Failed to handle Telegram update")
+        if UPDATE_TIME:
+            UPDATE_TIME.labels(update_type=type(update).__name__).observe(time.time() - start_time)
 
-    async def _get_dialogs(self, limit=None):
+    async def get_dialogs(self, limit: int = None) -> List[Union[Chat, Channel]]:
+        if self.is_bot:
+            return []
         dialogs = await self.client.get_dialogs(limit=limit)
         return [dialog.entity for dialog in dialogs if (
             not isinstance(dialog.entity, (User, ChatForbidden, ChannelForbidden))
@@ -80,37 +179,42 @@ class AbstractUser:
                      and (dialog.entity.deactivated or dialog.entity.left)))]
 
     @property
-    def name(self):
+    @abstractmethod
+    def name(self) -> str:
         raise NotImplementedError()
 
-    @property
-    def logged_in(self):
-        return self.client and self.client.is_user_authorized()
+    async def is_logged_in(self) -> bool:
+        return self.client and self.client.is_connected() and await self.client.is_user_authorized()
 
-    @property
-    def has_full_access(self):
-        return self.logged_in and self.whitelisted
+    async def has_full_access(self, allow_bot: bool = False) -> bool:
+        return (self.puppet_whitelisted
+                and (not self.is_bot or allow_bot)
+                and await self.is_logged_in())
 
-    async def start(self):
+    async def start(self, delete_unless_authenticated: bool = False) -> 'AbstractUser':
         if not self.client:
-            await self._init_client()
-        self.connected = await self.client.connect()
-
-    async def ensure_started(self, even_if_no_session=False):
-        if not self.whitelisted:
-            return self
-        elif not self.connected and (even_if_no_session or os.path.exists(f"{self.name}.session")):
-            return await self.start()
+            self._init_client()
+        await self.client.connect()
+        self.log.debug("%s connected: %s", self.mxid, self.connected)
         return self
 
-    def stop(self):
-        self.client.disconnect()
+    async def ensure_started(self, even_if_no_session=False) -> 'AbstractUser':
+        if not self.puppet_whitelisted or self.connected:
+            return self
+        self.log.debug("ensure_started(%s, even_if_no_session=%s)", self.mxid, even_if_no_session)
+        if even_if_no_session or self.session_container.has_session(self.mxid):
+            await self.start(delete_unless_authenticated=not even_if_no_session)
+        return self
+
+    async def stop(self) -> None:
+        await self.client.disconnect()
         self.client = None
-        self.connected = False
 
     # region Telegram update handling
 
-    async def _update(self, update):
+    async def _update(self, update: TypeUpdate) -> None:
+        asyncio.ensure_future(self._handle_entity_updates(getattr(update, "_entities", {})),
+                              loop=self.loop)
         if isinstance(update, (UpdateShortChatMessage, UpdateShortMessage, UpdateNewChannelMessage,
                                UpdateNewMessage, UpdateEditMessage, UpdateEditChannelMessage)):
             await self.update_message(update)
@@ -122,11 +226,11 @@ class AbstractUser:
             await self.update_typing(update)
         elif isinstance(update, UpdateUserStatus):
             await self.update_status(update)
-        elif isinstance(update, (UpdateChatAdmins, UpdateChatParticipantAdmin)):
+        elif isinstance(update, UpdateChatParticipantAdmin):
             await self.update_admin(update)
         elif isinstance(update, UpdateChatParticipants):
             await self.update_participants(update)
-        elif isinstance(update, UpdateChannelPinnedMessage):
+        elif isinstance(update, (UpdateChannelPinnedMessage, UpdateChatPinnedMessage)):
             await self.update_pinned_messages(update)
         elif isinstance(update, (UpdateUserName, UpdateUserPhoto)):
             await self.update_others_info(update)
@@ -135,79 +239,99 @@ class AbstractUser:
         else:
             self.log.debug("Unhandled update: %s", update)
 
-    async def update_pinned_messages(self, update):
-        portal = po.Portal.get_by_tgid(update.channel_id)
+    async def update_pinned_messages(self, update: Union[UpdateChannelPinnedMessage,
+                                                         UpdateChatPinnedMessage]) -> None:
+        if isinstance(update, UpdateChatPinnedMessage):
+            portal = po.Portal.get_by_tgid(TelegramID(update.chat_id))
+        else:
+            portal = po.Portal.get_by_tgid(TelegramID(update.channel_id))
         if portal and portal.mxid:
-            await portal.receive_telegram_pin_id(update.id)
+            await portal.receive_telegram_pin_id(update.id, self.tgid)
 
-    async def update_participants(self, update):
-        portal = po.Portal.get_by_tgid(update.participants.chat_id)
+    @staticmethod
+    async def update_participants(update: UpdateChatParticipants) -> None:
+        portal = po.Portal.get_by_tgid(TelegramID(update.participants.chat_id))
         if portal and portal.mxid:
             await portal.update_telegram_participants(update.participants.participants)
 
-    async def update_read_receipt(self, update):
+    async def update_read_receipt(self, update: UpdateReadHistoryOutbox) -> None:
         if not isinstance(update.peer, PeerUser):
             self.log.debug("Unexpected read receipt peer: %s", update.peer)
             return
 
-        portal = po.Portal.get_by_tgid(update.peer.user_id, self.tgid)
+        portal = po.Portal.get_by_tgid(TelegramID(update.peer.user_id), self.tgid)
         if not portal or not portal.mxid:
             return
 
         # We check that these are user read receipts, so tg_space is always the user ID.
-        message = DBMessage.query.get((update.max_id, self.tgid))
+        message = DBMessage.get_one_by_tgid(TelegramID(update.max_id), self.tgid, edit_index=-1)
         if not message:
             return
 
-        puppet = pu.Puppet.get(update.peer.user_id)
+        puppet = pu.Puppet.get(TelegramID(update.peer.user_id))
         await puppet.intent.mark_read(portal.mxid, message.mxid)
 
-    async def update_admin(self, update):
+    async def update_admin(self, update: UpdateChatParticipantAdmin) -> None:
         # TODO duplication not checked
-        portal = po.Portal.get_by_tgid(update.chat_id, peer_type="chat")
-        if isinstance(update, UpdateChatAdmins):
-            await portal.set_telegram_admins_enabled(update.enabled)
-        elif isinstance(update, UpdateChatParticipantAdmin):
-            await portal.set_telegram_admin(update.user_id)
-        else:
-            self.log.warning("Unexpected admin status update: %s", update)
+        portal = po.Portal.get_by_tgid(TelegramID(update.chat_id), peer_type="chat")
+        if not portal or not portal.mxid:
+            return
 
-    async def update_typing(self, update):
+        await portal.set_telegram_admin(TelegramID(update.user_id))
+
+    async def update_typing(self, update: Union[UpdateUserTyping, UpdateChatUserTyping]) -> None:
         if isinstance(update, UpdateUserTyping):
-            portal = po.Portal.get_by_tgid(update.user_id, self.tgid, "user")
+            portal = po.Portal.get_by_tgid(TelegramID(update.user_id), self.tgid, "user")
         else:
-            portal = po.Portal.get_by_tgid(update.chat_id, peer_type="chat")
-        sender = pu.Puppet.get(update.user_id)
+            portal = po.Portal.get_by_tgid(TelegramID(update.chat_id), peer_type="chat")
+
+        if not portal or not portal.mxid:
+            return
+
+        sender = pu.Puppet.get(TelegramID(update.user_id))
         await portal.handle_telegram_typing(sender, update)
 
-    async def update_others_info(self, update):
+    async def _handle_entity_updates(self, entities: Dict[int, Union[User, Chat, Channel]]
+                                     ) -> None:
+        try:
+            users = (entity for entity in entities.values() if isinstance(entity, User))
+            puppets = ((pu.Puppet.get(TelegramID(user.id)), user) for user in users)
+            await asyncio.gather(*[puppet.update_info(self, info)
+                                   for puppet, info in puppets if puppet])
+        except Exception:
+            self.log.exception("Failed to handle entity updates")
+
+    async def update_others_info(self, update: Union[UpdateUserName, UpdateUserPhoto]) -> None:
         # TODO duplication not checked
-        puppet = pu.Puppet.get(update.user_id)
+        puppet = pu.Puppet.get(TelegramID(update.user_id))
         if isinstance(update, UpdateUserName):
+            puppet.username = update.username
             if await puppet.update_displayname(self, update):
                 puppet.save()
         elif isinstance(update, UpdateUserPhoto):
-            if await puppet.update_avatar(self, update.photo.photo_big):
+            if await puppet.update_avatar(self, update.photo):
                 puppet.save()
         else:
             self.log.warning("Unexpected other user info update: %s", update)
 
-    async def update_status(self, update):
-        puppet = pu.Puppet.get(update.user_id)
+    async def update_status(self, update: UpdateUserStatus) -> None:
+        puppet = pu.Puppet.get(TelegramID(update.user_id))
         if isinstance(update.status, UserStatusOnline):
-            await puppet.intent.set_presence("online")
+            await puppet.default_mxid_intent.set_presence("online")
         elif isinstance(update.status, UserStatusOffline):
-            await puppet.intent.set_presence("offline")
+            await puppet.default_mxid_intent.set_presence("offline")
         else:
             self.log.warning("Unexpected user status update: %s", update)
         return
 
-    def get_message_details(self, update):
+    def get_message_details(self, update: UpdateMessage) -> Tuple[UpdateMessageContent,
+                                                                  Optional[pu.Puppet],
+                                                                  Optional[po.Portal]]:
         if isinstance(update, UpdateShortChatMessage):
-            portal = po.Portal.get_by_tgid(update.chat_id, peer_type="chat")
-            sender = pu.Puppet.get(update.from_id)
+            portal = po.Portal.get_by_tgid(TelegramID(update.chat_id), peer_type="chat")
+            sender = pu.Puppet.get(TelegramID(update.from_id))
         elif isinstance(update, UpdateShortMessage):
-            portal = po.Portal.get_by_tgid(update.user_id, self.tgid, "user")
+            portal = po.Portal.get_by_tgid(TelegramID(update.user_id), self.tgid, "user")
             sender = pu.Puppet.get(self.tgid if update.out else update.user_id)
         elif isinstance(update, (UpdateNewMessage, UpdateNewChannelMessage,
                                  UpdateEditMessage, UpdateEditChannelMessage)):
@@ -225,7 +349,8 @@ class AbstractUser:
         return update, sender, portal
 
     @staticmethod
-    async def _try_redact(portal, message):
+    async def _try_redact(message: DBMessage) -> None:
+        portal = po.Portal.get_by_mxid(message.mx_room)
         if not portal:
             return
         try:
@@ -233,41 +358,43 @@ class AbstractUser:
         except MatrixRequestError:
             pass
 
-    async def delete_message(self, update):
+    async def delete_message(self, update: UpdateDeleteMessages) -> None:
         if len(update.messages) > MAX_DELETIONS:
             return
 
-        for message in update.messages:
-            message = DBMessage.query.get((message, self.tgid))
-            if not message:
-                continue
-            self.db.delete(message)
-            number_left = DBMessage.query.filter(DBMessage.mxid == message.mxid,
-                                                 DBMessage.mx_room == message.mx_room).count()
-            if number_left == 0:
-                portal = po.Portal.get_by_mxid(message.mx_room)
-                await self._try_redact(portal, message)
-        self.db.commit()
+        for message_id in update.messages:
+            messages = DBMessage.get_all_by_tgid(TelegramID(message_id), self.tgid)
+            for message in messages:
+                message.delete()
+                number_left = DBMessage.count_spaces_by_mxid(message.mxid, message.mx_room)
+                if number_left == 0:
+                    portal = po.Portal.get_by_mxid(message.mx_room)
+                    await self._try_redact(message)
 
-    async def delete_channel_message(self, update):
+    async def delete_channel_message(self, update: UpdateDeleteChannelMessages) -> None:
         if len(update.messages) > MAX_DELETIONS:
             return
 
-        portal = po.Portal.get_by_tgid(update.channel_id)
-        if not portal:
-            return
+        channel_id = TelegramID(update.channel_id)
 
-        for message in update.messages:
-            message = DBMessage.query.get((message, portal.tgid))
-            if not message:
-                continue
-            self.db.delete(message)
-            await self._try_redact(portal, message)
-        self.db.commit()
+        for message_id in update.messages:
+            messages = DBMessage.get_all_by_tgid(TelegramID(message_id), channel_id)
+            for message in messages:
+                message.delete()
+                await self._try_redact(message)
 
-    async def update_message(self, original_update):
+    async def update_message(self, original_update: UpdateMessage) -> None:
         update, sender, portal = self.get_message_details(original_update)
 
+        if self.is_bot and not portal.mxid:
+            self.log.debug(f"Ignoring message received by bot in unbridged chat %s",
+                           portal.tgid_log)
+            return
+
+        if self.ignore_incoming_bot_events and self.bot and sender.id == self.bot.tgid:
+            self.log.debug(f"Ignoring relaybot-sent message %s to %s", update, portal.tgid_log)
+            return
+
         if isinstance(update, MessageService):
             if isinstance(update.action, MessageActionChannelMigrateFrom):
                 self.log.debug(f"Ignoring action %s to %s by %d", update.action,
@@ -280,10 +407,7 @@ class AbstractUser:
 
         user = sender.tgid if sender else "admin"
         if isinstance(original_update, (UpdateEditMessage, UpdateEditChannelMessage)):
-            if config["bridge.edits_as_replies"]:
-                self.log.debug("Handling edit %s to %s by %s", update, portal.tgid_log, user)
-                return await portal.handle_telegram_edit(self, sender, update)
-            return
+            return await portal.handle_telegram_edit(self, sender, update)
 
         self.log.debug("Handling message %s to %s by %s", update, portal.tgid_log, user)
         return await portal.handle_telegram_message(self, sender, update)
@@ -291,8 +415,9 @@ class AbstractUser:
     # endregion
 
 
-def init(context):
+def init(context: "Context") -> None:
     global config, MAX_DELETIONS
-    AbstractUser.az, AbstractUser.db, config, AbstractUser.loop, _ = context
-    AbstractUser.session_container = context.telethon_session_container
+    AbstractUser.az, config, AbstractUser.loop, AbstractUser.relaybot = context.core
+    AbstractUser.ignore_incoming_bot_events = config["bridge.relaybot.ignore_own_incoming_events"]
+    AbstractUser.session_container = context.session_container
     MAX_DELETIONS = config.get("bridge.max_telegram_delete", 10)

mautrix_telegram/base.py

@@ -1,2 +0,0 @@
-from sqlalchemy.ext.declarative import declarative_base
-Base = declarative_base()

mautrix_telegram/bot.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -14,78 +14,102 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from typing import Awaitable, Callable
+from typing import Awaitable, Callable, Dict, List, Optional, Pattern, Tuple, TYPE_CHECKING
 import logging
 import re
 
-from telethon.tl.types import *
+from telethon.tl.patched import Message, MessageService
+from telethon.tl.types import (
+    ChannelParticipantAdmin, ChannelParticipantCreator, ChatForbidden, ChatParticipantAdmin,
+    ChatParticipantCreator, InputChannel, InputUser, MessageActionChatAddUser,
+    MessageActionChatDeleteUser, MessageEntityBotCommand, PeerChannel, PeerChat, TypePeer,
+    UpdateNewChannelMessage, UpdateNewMessage, MessageActionChatMigrateTo, User)
 from telethon.tl.functions.messages import GetChatsRequest, GetFullChatRequest
 from telethon.tl.functions.channels import GetChannelsRequest, GetParticipantRequest
 from telethon.errors import ChannelInvalidError, ChannelPrivateError
 
+from .types import MatrixUserID
 from .abstract_user import AbstractUser
 from .db import BotChat
+from .types import TelegramID
 from . import puppet as pu, portal as po, user as u
 
-config = None
+if TYPE_CHECKING:
+    from .config import Config
+    from .context import Context
+
+config = None  # type: Config
 
 ReplyFunc = Callable[[str], Awaitable[Message]]
 
 
 class Bot(AbstractUser):
-    log = logging.getLogger("mau.bot")
-    mxid_regex = re.compile("@.+:.+")
+    log = logging.getLogger("mau.bot")  # type: logging.Logger
+    mxid_regex = re.compile("@.+:.+")  # type: Pattern
 
-    def __init__(self, token: str):
+    def __init__(self, token: str) -> None:
         super().__init__()
-        self.token = token
-        self.whitelisted = True
-        self.username = None
-        self.is_relaybot = True
-        self.chats = {chat.id: chat.type for chat in BotChat.query.all()}
-        self.tg_whitelist = []
-        self.whitelist_group_admins = config["bridge.relaybot.whitelist_group_admins"] or False
+        self.token = token  # type: str
+        self.puppet_whitelisted = True  # type: bool
+        self.whitelisted = True  # type: bool
+        self.relaybot_whitelisted = True  # type: bool
+        self.username = None  # type: str
+        self.is_relaybot = True  # type: bool
+        self.is_bot = True  # type: bool
+        self.chats = {}  # type: Dict[int, str]
+        self.tg_whitelist = []  # type: List[int]
+        self.whitelist_group_admins = (config["bridge.relaybot.whitelist_group_admins"]
+                                       or False)  # type: bool
+        self._me_info = None  # type: Optional[User]
+        self._me_mxid = None  # type: Optional[MatrixUserID]
 
-    async def init_permissions(self):
+    async def get_me(self, use_cache: bool = True) -> Tuple[User, MatrixUserID]:
+        if not use_cache or not self._me_mxid:
+            self._me_info = await self.client.get_me()
+            self._me_mxid = pu.Puppet.get_mxid_from_id(TelegramID(self._me_info.id))
+        return self._me_info, self._me_mxid
+
+    async def init_permissions(self) -> None:
         whitelist = config["bridge.relaybot.whitelist"] or []
-        for id in whitelist:
-            if isinstance(id, str):
-                entity = await self.client.get_input_entity(id)
+        for user_id in whitelist:
+            if isinstance(user_id, str):
+                entity = await self.client.get_input_entity(user_id)
                 if isinstance(entity, InputUser):
-                    id = entity.user_id
+                    user_id = entity.user_id
                 else:
-                    id = None
-            if isinstance(id, int):
-                self.tg_whitelist.append(id)
+                    user_id = None
+            if isinstance(user_id, int):
+                self.tg_whitelist.append(user_id)
 
-    async def start(self):
-        await super().start()
-        if not self.logged_in:
+    async def start(self, delete_unless_authenticated: bool = False) -> 'Bot':
+        self.chats = {chat.id: chat.type for chat in BotChat.all()}
+        await super().start(delete_unless_authenticated)
+        if not await self.is_logged_in():
             await self.client.sign_in(bot_token=self.token)
         await self.post_login()
         return self
 
-    async def post_login(self):
+    async def post_login(self) -> None:
         await self.init_permissions()
         info = await self.client.get_me()
         self.tgid = info.id
         self.username = info.username
         self.mxid = pu.Puppet.get_mxid_from_id(self.tgid)
 
-        chat_ids = [id for id, type in self.chats.items() if type == "chat"]
+        chat_ids = [chat_id for chat_id, chat_type in self.chats.items() if chat_type == "chat"]
         response = await self.client(GetChatsRequest(chat_ids))
         for chat in response.chats:
             if isinstance(chat, ChatForbidden) or chat.left or chat.deactivated:
-                self.remove_chat(chat.id)
+                self.remove_chat(TelegramID(chat.id))
 
-        channel_ids = [InputChannel(id, 0)
-                       for id, type in self.chats.items()
-                       if type == "channel"]
-        for id in channel_ids:
+        channel_ids = [InputChannel(chat_id, 0)
+                       for chat_id, chat_type in self.chats.items()
+                       if chat_type == "channel"]
+        for channel_id in channel_ids:
             try:
-                await self.client(GetChannelsRequest([id]))
+                await self.client(GetChannelsRequest([channel_id]))
             except (ChannelPrivateError, ChannelInvalidError):
-                self.remove_chat(id.channel_id)
+                self.remove_chat(TelegramID(channel_id.channel_id))
 
         if config["bridge.catch_up"]:
             try:
@@ -93,29 +117,25 @@ class Bot(AbstractUser):
             except Exception:
                 self.log.exception("Failed to run catch_up() for bot")
 
-    def register_portal(self, portal: po.Portal):
+    def register_portal(self, portal: po.Portal) -> None:
         self.add_chat(portal.tgid, portal.peer_type)
 
-    def unregister_portal(self, portal: po.Portal):
+    def unregister_portal(self, portal: po.Portal) -> None:
         self.remove_chat(portal.tgid)
 
-    def add_chat(self, id: int, type: str):
-        if id not in self.chats:
-            self.chats[id] = type
-            self.db.add(BotChat(id=id, type=type))
-            self.db.commit()
+    def add_chat(self, chat_id: TelegramID, chat_type: str) -> None:
+        if chat_id not in self.chats:
+            self.chats[chat_id] = chat_type
+            BotChat(id=TelegramID(chat_id), type=chat_type).insert()
 
-    def remove_chat(self, id: int):
+    def remove_chat(self, chat_id: TelegramID) -> None:
         try:
-            del self.chats[id]
+            del self.chats[chat_id]
         except KeyError:
             pass
-        existing_chat = BotChat.query.get(id)
-        if existing_chat:
-            self.db.delete(existing_chat)
-            self.db.commit()
+        BotChat.delete(chat_id)
 
-    async def _can_use_commands(self, chat, tgid):
+    async def _can_use_commands(self, chat: TypePeer, tgid: TelegramID) -> bool:
         if tgid in self.tg_whitelist:
             return True
 
@@ -134,14 +154,15 @@ class Bot(AbstractUser):
                 for p in participants:
                     if p.user_id == tgid:
                         return isinstance(p, (ChatParticipantCreator, ChatParticipantAdmin))
+        return False
 
-    async def check_can_use_commands(self, event: Message, reply: ReplyFunc):
+    async def check_can_use_commands(self, event: Message, reply: ReplyFunc) -> bool:
         if not await self._can_use_commands(event.to_id, event.from_id):
             await reply("You do not have the permission to use that command.")
             return False
         return True
 
-    async def handle_command_portal(self, portal: po.Portal, reply: ReplyFunc):
+    async def handle_command_portal(self, portal: po.Portal, reply: ReplyFunc) -> Message:
         if not config["bridge.relaybot.authless_portals"]:
             return await reply("This bridge doesn't allow portal creation from Telegram.")
 
@@ -157,18 +178,19 @@ class Bot(AbstractUser):
                 return await reply(
                     "Portal is not public. Use `/invite <mxid>` to get an invite.")
 
-    async def handle_command_invite(self, portal: po.Portal, reply: ReplyFunc, mxid: str):
-        if len(mxid) == 0:
+    async def handle_command_invite(self, portal: po.Portal, reply: ReplyFunc,
+                                    mxid_input: MatrixUserID) -> Message:
+        if len(mxid_input) == 0:
             return await reply("Usage: `/invite <mxid>`")
         elif not portal.mxid:
             return await reply("Portal does not have Matrix room. "
                                "Create one with /portal first.")
-        if not self.mxid_regex.match(mxid):
+        if not self.mxid_regex.match(mxid_input):
             return await reply("That doesn't look like a Matrix ID.")
-        user = await u.User.get_by_mxid(mxid).ensure_started()
-        if not user.whitelisted:
+        user = await u.User.get_by_mxid(MatrixUserID(mxid_input)).ensure_started()
+        if not user.relaybot_whitelisted:
             return await reply("That user is not whitelisted to use the bridge.")
-        elif user.logged_in:
+        elif await user.is_logged_in():
             displayname = f"@{user.username}" if user.username else user.displayname
             return await reply("That user seems to be logged in. "
                                f"Just invite [{displayname}](tg://user?id={user.tgid})")
@@ -176,7 +198,8 @@ class Bot(AbstractUser):
             await portal.main_intent.invite(portal.mxid, user.mxid)
             return await reply(f"Invited `{user.mxid}` to the portal.")
 
-    def handle_command_id(self, message: Message, reply: ReplyFunc):
+    @staticmethod
+    def handle_command_id(message: Message, reply: ReplyFunc) -> Awaitable[Message]:
         # Provide the prefixed ID to the user so that the user wouldn't need to specify whether the
         # chat is a normal group or a supergroup/channel when using the ID.
         if isinstance(message.to_id, PeerChannel):
@@ -198,15 +221,15 @@ class Bot(AbstractUser):
 
         return False
 
-    async def handle_command(self, message: Message):
-        def reply(reply_text):
-            return self.client.send_message(message.to_id, reply_text, markdown=True,
-                                            reply_to=message.id)
+    async def handle_command(self, message: Message) -> None:
+        def reply(reply_text: str) -> Awaitable[Message]:
+            return self.client.send_message(message.to_id, reply_text, reply_to=message.id)
 
         text = message.message
 
         if self.match_command(text, "id"):
-            return await self.handle_command_id(message, reply)
+            await self.handle_command_id(message, reply)
+            return
 
         portal = po.Portal.get_by_entity(message.to_id)
 
@@ -221,36 +244,42 @@ class Bot(AbstractUser):
                 mxid = text[text.index(" ") + 1:]
             except ValueError:
                 mxid = ""
-            await self.handle_command_invite(portal, reply, mxid=mxid)
+            await self.handle_command_invite(portal, reply, mxid_input=mxid)
 
-    def handle_service_message(self, message: MessageService):
-        to_id = message.to_id
+    def handle_service_message(self, message: MessageService) -> None:
+        to_id = message.to_id  # type: TelegramID
         if isinstance(to_id, PeerChannel):
             to_id = to_id.channel_id
-            type = "channel"
+            chat_type = "channel"
         elif isinstance(to_id, PeerChat):
             to_id = to_id.chat_id
-            type = "chat"
+            chat_type = "chat"
         else:
             return
 
         action = message.action
         if isinstance(action, MessageActionChatAddUser) and self.tgid in action.users:
-            self.add_chat(to_id, type)
+            self.add_chat(to_id, chat_type)
         elif isinstance(action, MessageActionChatDeleteUser) and action.user_id == self.tgid:
             self.remove_chat(to_id)
+        elif isinstance(action, MessageActionChatMigrateTo):
+            self.remove_chat(to_id)
+            self.add_chat(TelegramID(action.channel_id), "channel")
 
-    async def update(self, update):
+    async def update(self, update) -> bool:
         if not isinstance(update, (UpdateNewMessage, UpdateNewChannelMessage)):
-            return
+            return False
         if isinstance(update.message, MessageService):
-            return self.handle_service_message(update.message)
+            self.handle_service_message(update.message)
+            return False
 
         is_command = (isinstance(update.message, Message)
                       and update.message.entities and len(update.message.entities) > 0
                       and isinstance(update.message.entities[0], MessageEntityBotCommand))
         if is_command:
-            return await self.handle_command(update.message)
+            await self.handle_command(update.message)
+            return True
+        return False
 
     def is_in_chat(self, peer_id) -> bool:
         return peer_id in self.chats
@@ -260,9 +289,9 @@ class Bot(AbstractUser):
         return "bot"
 
 
-def init(context):
+def init(cfg: 'Config') -> Optional[Bot]:
     global config
-    config = context.config
+    config = cfg
     token = config["telegram.bot_token"]
     if token and not token.lower().startswith("disable"):
         return Bot(token)

mautrix_telegram/commands/__init__.py

@@ -1,2 +1,5 @@
-from .handler import command_handler, CommandHandler, CommandEvent
-from . import clean_rooms, auth, meta, telegram, portal
+from .handler import (command_handler, command_handlers as _command_handlers,
+                      CommandHandler, CommandProcessor, CommandEvent,
+                      SECTION_GENERAL, SECTION_AUTH, SECTION_CREATING_PORTALS,
+                      SECTION_PORTAL_MANAGEMENT, SECTION_MISC, SECTION_ADMIN)
+from . import portal, telegram, clean_rooms, matrix_auth, meta

mautrix_telegram/commands/clean_rooms.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -14,20 +14,27 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from mautrix_appservice import MatrixRequestError
+from typing import Dict, List, NewType, Optional, Tuple, Union
 
-from . import command_handler
+from mautrix_appservice import MatrixRequestError, IntentAPI
+
+from ..types import MatrixRoomID, MatrixUserID
+from . import command_handler, CommandEvent, SECTION_ADMIN
 from .. import puppet as pu, portal as po
 
+ManagementRoom = NewType('ManagementRoom', Tuple[MatrixRoomID, MatrixUserID])
 
-async def _find_rooms(intent):
-    management_rooms = []
-    unidentified_rooms = []
-    portals = []
-    empty_portals = []
+
+async def _find_rooms(intent: IntentAPI) -> Tuple[List[ManagementRoom], List[MatrixRoomID],
+                                                  List['po.Portal'], List['po.Portal']]:
+    management_rooms = []  # type: List[ManagementRoom]
+    unidentified_rooms = []  # type: List[MatrixRoomID]
+    portals = []  # type: List[po.Portal]
+    empty_portals = []  # type: List[po.Portal]
 
     rooms = await intent.get_joined_rooms()
-    for room in rooms:
+    for room_str in rooms:
+        room = MatrixRoomID(room_str)
         portal = po.Portal.get_by_mxid(room)
         if not portal:
             try:
@@ -35,11 +42,11 @@ async def _find_rooms(intent):
             except MatrixRequestError:
                 members = []
             if len(members) == 2:
-                other_member = members[0] if members[0] != intent.mxid else members[1]
+                other_member = MatrixUserID(members[0] if members[0] != intent.mxid else members[1])
                 if pu.Puppet.get_id_from_mxid(other_member):
                     unidentified_rooms.append(room)
                 else:
-                    management_rooms.append((room, other_member))
+                    management_rooms.append(ManagementRoom((room, other_member)))
             else:
                 unidentified_rooms.append(room)
         else:
@@ -52,12 +59,10 @@ async def _find_rooms(intent):
     return management_rooms, unidentified_rooms, portals, empty_portals
 
 
-@command_handler(needs_admin=True, needs_auth=False, name="clean-rooms")
-async def clean_rooms(evt):
-    if not evt.is_management:
-        return await evt.reply("`clean-rooms` is a particularly spammy command. Please don't "
-                               "run it in non-management rooms.")
-
+@command_handler(needs_admin=True, needs_auth=False, management_only=True, name="clean-rooms",
+                 help_section=SECTION_ADMIN,
+                 help_text="Clean up unused portal/management rooms.")
+async def clean_rooms(evt: CommandEvent) -> Optional[Dict]:
     management_rooms, unidentified_rooms, portals, empty_portals = await _find_rooms(evt.az.intent)
 
     reply = ["#### Management rooms (M)"]
@@ -65,7 +70,7 @@ async def clean_rooms(evt):
                for n, (room, other_member) in enumerate(management_rooms)]
               or ["No management rooms found."])
     reply.append("#### Active portal rooms (A)")
-    reply += ([f"{n+1}. [P{n+1}](https://matrix.to/#/{portal.mxid}) "
+    reply += ([f"{n+1}. [A{n+1}](https://matrix.to/#/{portal.mxid}) "
                f"(to Telegram chat \"{portal.title}\")"
                for n, portal in enumerate(portals)]
               or ["No active portal rooms found."])
@@ -74,7 +79,7 @@ async def clean_rooms(evt):
                for n, room in enumerate(unidentified_rooms)]
               or ["No unidentified rooms found."])
     reply.append("#### Inactive portal rooms (I)")
-    reply += ([f"{n}. [E{n}](https://matrix.to/#/{portal.mxid}) "
+    reply += ([f"{n}. [I{n}](https://matrix.to/#/{portal.mxid}) "
                f"(to Telegram chat \"{portal.title}\")"
                for n, portal in enumerate(empty_portals)]
               or ["No inactive portal rooms found."])
@@ -88,9 +93,9 @@ async def clean_rooms(evt):
               "",
               ("To clean specific rooms, type `$cmdprefix+sp clean-range <range>` "
                "where `range` is the range (e.g. `5-21`) prefixed with the first letter of"
-               "the group name."),
+               "the group name. (e.g. `I2-6`)"),
               "",
-              ("Please note that you will have to re-run `$cmdprefix+sp cleanrooms` "
+              ("Please note that you will have to re-run `$cmdprefix+sp clean-rooms` "
                "between each use of the commands above.")]
 
     evt.sender.command_status = {
@@ -102,17 +107,20 @@ async def clean_rooms(evt):
     return await evt.reply("\n".join(reply))
 
 
-async def set_rooms_to_clean(evt, management_rooms, unidentified_rooms, portals, empty_portals):
+async def set_rooms_to_clean(evt, management_rooms: List[ManagementRoom],
+                             unidentified_rooms: List[MatrixRoomID], portals: List["po.Portal"],
+                             empty_portals: List["po.Portal"]) -> None:
     command = evt.args[0]
-    rooms_to_clean = []
+    rooms_to_clean = []  # type: List[Union[po.Portal, MatrixRoomID]]
     if command == "clean-recommended":
-        rooms_to_clean = empty_portals + unidentified_rooms
+        rooms_to_clean += empty_portals
+        rooms_to_clean += unidentified_rooms
     elif command == "clean-groups":
         if len(evt.args) < 2:
             return await evt.reply("**Usage:** `$cmdprefix+sp clean-groups [M][A][U][I]")
-        groups_to_clean = evt.args[1]
+        groups_to_clean = evt.args[1].upper()
         if "M" in groups_to_clean:
-            rooms_to_clean += management_rooms
+            rooms_to_clean += [room_id for (room_id, user_id) in management_rooms]
         if "A" in groups_to_clean:
             rooms_to_clean += portals
         if "U" in groups_to_clean:
@@ -121,12 +129,12 @@ async def set_rooms_to_clean(evt, management_rooms, unidentified_rooms, portals,
             rooms_to_clean += empty_portals
     elif command == "clean-range":
         try:
-            range = evt.args[1]
-            group, range = range[0], range[1:]
-            start, end = range.split("-")
+            clean_range = evt.args[1]
+            group, clean_range = clean_range[0], clean_range[1:]
+            start, end = clean_range.split("-")
             start, end = int(start), int(end)
             if group == "M":
-                group = management_rooms
+                group = [room_id for (room_id, user_id) in management_rooms]
             elif group == "A":
                 group = portals
             elif group == "U":
@@ -152,7 +160,7 @@ async def set_rooms_to_clean(evt, management_rooms, unidentified_rooms, portals,
                     "`$cmdprefix+sp confirm-clean`.")
 
 
-async def execute_room_cleanup(evt, rooms_to_clean):
+async def execute_room_cleanup(evt, rooms_to_clean: List[Union[po.Portal, MatrixRoomID]]) -> None:
     if len(evt.args) > 0 and evt.args[0] == "confirm-clean":
         await evt.reply(f"Cleaning {len(rooms_to_clean)} rooms. "
                         "This might take a while.")
@@ -161,7 +169,7 @@ async def execute_room_cleanup(evt, rooms_to_clean):
             if isinstance(room, po.Portal):
                 await room.cleanup_and_delete()
                 cleaned += 1
-            elif isinstance(room, str):
+            elif isinstance(room, str):  # str is aliased by MatrixRoomID
                 await po.Portal.cleanup_room(evt.az.intent, room, message="Room deleted")
                 cleaned += 1
         evt.sender.command_status = None

mautrix_telegram/commands/handler.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -14,90 +14,352 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-import markdown
+"""This module contains classes handling commands issued by Matrix users."""
+from typing import Awaitable, Callable, Dict, List, NamedTuple, Optional
 import logging
+import traceback
+
+import commonmark
 
 from telethon.errors import FloodWaitError
 
+from ..types import MatrixRoomID, MatrixEventID
 from ..util import format_duration
+from .. import user as u, context as c
 
-command_handlers = {}
+command_handlers = {}  # type: Dict[str, CommandHandler]
+
+HelpSection = NamedTuple('HelpSection', [('name', str), ('order', int), ('description', str)])
+
+SECTION_GENERAL = HelpSection("General", 0, "")
+SECTION_AUTH = HelpSection("Authentication", 10, "")
+SECTION_CREATING_PORTALS = HelpSection("Creating portals", 20, "")
+SECTION_PORTAL_MANAGEMENT = HelpSection("Portal management", 30, "")
+SECTION_MISC = HelpSection("Miscellaneous", 40, "")
+SECTION_ADMIN = HelpSection("Administration", 50, "")
 
 
-def command_handler(needs_auth=True, management_only=False, needs_admin=False, name=None):
-    def decorator(func):
-        def wrapper(evt):
-            if management_only and not evt.is_management:
-                return evt.reply(f"`{evt.command}` is a restricted command:"
-                                 "you may only run it in management rooms.")
-            elif needs_auth and not evt.sender.logged_in:
-                return evt.reply("This command requires you to be logged in.")
-            elif needs_admin and not evt.sender.is_admin:
-                return evt.reply("This is command requires administrator privileges.")
-            return func(evt)
+class HtmlEscapingRenderer(commonmark.HtmlRenderer):
+    def __init__(self, allow_html: bool = False):
+        super().__init__()
+        self.allow_html = allow_html
 
-        command_handlers[name or func.__name__.replace("_", "-")] = wrapper
-        return wrapper
+    def lit(self, s):
+        if self.allow_html:
+            return super().lit(s)
+        return super().lit(s.replace("<", "&lt;").replace(">", "&gt;"))
 
-    return decorator
+    def image(self, node, entering):
+        prev = self.allow_html
+        self.allow_html = True
+        super().image(node, entering)
+        self.allow_html = prev
+
+
+md_parser = commonmark.Parser()
+md_renderer = HtmlEscapingRenderer()
+
+
+def ensure_trailing_newline(s: str) -> str:
+    """Returns the passed string, but with a guaranteed trailing newline."""
+    return s + ("" if s[-1] == "\n" else "\n")
 
 
 class CommandEvent:
-    def __init__(self, handler, room, sender, command, args, is_management, is_portal):
-        self.az = handler.az
-        self.log = handler.log
-        self.loop = handler.loop
-        self.tgbot = handler.tgbot
-        self.config = handler.config
-        self.command_prefix = handler.command_prefix
+    """Holds information about a command issued in a Matrix room.
+
+    When a Matrix command was issued to the bot, CommandEvent will hold
+    information regarding the event.
+
+    Attributes:
+        room_id: The id of the Matrix room in which the command was issued.
+        event_id: The id of the matrix event which contained the command.
+        sender: The user who issued the command.
+        command: The issued command.
+        args: Arguments given with the issued command.
+        is_management: Determines whether the room in which the command wa
+            issued is a management room.
+        is_portal: Determines whether the room in which the command was issued
+            is a portal.
+    """
+
+    def __init__(self, processor: 'CommandProcessor', room: MatrixRoomID, event: MatrixEventID,
+                 sender: u.User, command: str, args: List[str], is_management: bool,
+                 is_portal: bool) -> None:
+        self.az = processor.az
+        self.log = processor.log
+        self.loop = processor.loop
+        self.tgbot = processor.tgbot
+        self.config = processor.config
+        self.public_website = processor.public_website
+        self.command_prefix = processor.command_prefix
         self.room_id = room
+        self.event_id = event
         self.sender = sender
         self.command = command
         self.args = args
         self.is_management = is_management
         self.is_portal = is_portal
 
-    def reply(self, message, allow_html=False, render_markdown=True):
-        message = message.replace("$cmdprefix+sp ",
-                                  "" if self.is_management else f"{self.command_prefix} ")
-        message = message.replace("$cmdprefix", self.command_prefix)
-        html = None
+    def reply(self, message: str, allow_html: bool = False, render_markdown: bool = True
+              ) -> Awaitable[Dict]:
+        """Write a reply to the room in which the command was issued.
+
+        Replaces occurences of "$cmdprefix" in the message with the command
+        prefix and replaces occurences of "$cmdprefix+sp " with the command
+        prefix if the command was not issued in a management room.
+        If allow_html and render_markdown are both False, the message will not
+        be rendered to html and sending of html is disabled.
+
+        Args:
+            message: The message to post in the room.
+            allow_html: Escape html in the message or don't render html at all
+                if markdown is disabled.
+            render_markdown: Use markdown formatting to render the passed
+                message to html.
+
+        Returns:
+            Handler for the message sending function.
+        """
+        message_cmd = self._replace_command_prefix(message)
+        html = self._render_message(message_cmd, allow_html=allow_html,
+                                    render_markdown=render_markdown)
+
+        return self.az.intent.send_notice(self.room_id, message_cmd, html=html)
+
+    def mark_read(self) -> Awaitable[Dict]:
+        """Marks the command as read by the bot."""
+        return self.az.intent.mark_read(self.room_id, self.event_id)
+
+    def _replace_command_prefix(self, message: str) -> str:
+        """Returns the string with the proper command prefix entered."""
+        message = message.replace(
+            "$cmdprefix+sp ", "" if self.is_management else f"{self.command_prefix} "
+        )
+        return message.replace("$cmdprefix", self.command_prefix)
+
+    @staticmethod
+    def _render_message(message: str, allow_html: bool, render_markdown: bool) -> Optional[str]:
+        """Renders the message as HTML.
+
+        Args:
+            allow_html: Flag to allow custom HTML in the message.
+            render_markdown: If true, markdown styling is applied to the message.
+
+        Returns:
+            The message rendered as HTML.
+            None is returned if no styled output is required.
+        """
+        html = ""
         if render_markdown:
-            html = markdown.markdown(message, safe_mode="escape" if allow_html else False)
+            md_renderer.allow_html = allow_html
+            html = md_renderer.render(md_parser.parse(message))
         elif allow_html:
             html = message
-        return self.az.intent.send_notice(self.room_id, message, html=html)
+        return ensure_trailing_newline(html) if html else None
 
 
 class CommandHandler:
+    """A command which can be executed from a Matrix room.
+
+    The command manages its permission and help texts.
+    When called, it will check the permission of the command event and execute
+    the command or, in case of error, report back to the user.
+
+    Attributes:
+        needs_auth: Flag indicating if the sender is required to be logged in.
+        needs_puppeting: Flag indicating if the sender is required to use
+            Telegram puppeteering for this command.
+        needs_matrix_puppeting: Flag indicating if the sender is required to use
+            Matrix pupeteering.
+        needs_admin: Flag for whether only admin users can issue this command.
+        management_only: Whether the command can exclusively be issued in a
+            management room.
+        name: The name of this command.
+        help_section: Section of the help in which this command will appear.
+    """
+
+    def __init__(self, handler: Callable[[CommandEvent], Awaitable[Dict]], needs_auth: bool,
+                 needs_puppeting: bool, needs_matrix_puppeting: bool, needs_admin: bool,
+                 management_only: bool, name: str, help_text: str, help_args: str,
+                 help_section: HelpSection) -> None:
+        """
+        Args:
+            handler: The function handling the execution of this command.
+            needs_auth: Flag indicating if the sender is required to be logged in.
+            needs_puppeting: Flag indicating if the sender is required to use
+                Telegram puppeteering for this command.
+            needs_matrix_puppeting: Flag indicating if the sender is required to
+                use Matrix pupeteering.
+            needs_admin: Flag for whether only admin users can issue this command.
+            management_only: Whether the command can exclusively be issued
+                in a management room.
+            name: The name of this command.
+            help_text: The text displayed in the help for this command.
+            help_args: Help text for the arguments of this command.
+            help_section: Section of the help in which this command will appear.
+        """
+        self._handler = handler
+        self.needs_auth = needs_auth
+        self.needs_puppeting = needs_puppeting
+        self.needs_matrix_puppeting = needs_matrix_puppeting
+        self.needs_admin = needs_admin
+        self.management_only = management_only
+        self.name = name
+        self._help_text = help_text
+        self._help_args = help_args
+        self.help_section = help_section
+
+    async def get_permission_error(self, evt: CommandEvent) -> Optional[str]:
+        """Returns the reason why the command could not be issued.
+
+        Args:
+            evt: The event for which to get the error information.
+
+        Returns:
+            A string describing the error or None if there was no error.
+        """
+        if self.management_only and not evt.is_management:
+            return (f"`{evt.command}` is a restricted command: "
+                    "you may only run it in management rooms.")
+        elif self.needs_puppeting and not evt.sender.puppet_whitelisted:
+            return "This command requires puppeting privileges."
+        elif self.needs_matrix_puppeting and not evt.sender.matrix_puppet_whitelisted:
+            return "This command requires Matrix puppeting privileges."
+        elif self.needs_admin and not evt.sender.is_admin:
+            return "This command requires administrator privileges."
+        elif self.needs_auth and not await evt.sender.is_logged_in():
+            return "This command requires you to be logged in."
+        return None
+
+    def has_permission(self, is_management: bool, puppet_whitelisted: bool,
+                       matrix_puppet_whitelisted: bool, is_admin: bool, is_logged_in: bool) -> bool:
+        """Checks the permission for this command with the given status.
+
+        Args:
+            is_management: If the room in which the command will be issued is a
+                management room.
+            puppet_whitelisted: If the connected Telegram account puppet is
+                allowed to issue the command.
+            matrix_puppet_whitelisted: If the connected Matrix account puppet is
+                allowed to issue the command.
+            is_admin: If the issuing user is an admin.
+            is_logged_in: If the issuing user is logged in.
+
+        Returns:
+            True if a user with the given state is allowed to issue the
+            command.
+        """
+        return ((not self.management_only or is_management) and
+                (not self.needs_puppeting or puppet_whitelisted) and
+                (not self.needs_matrix_puppeting or matrix_puppet_whitelisted) and
+                (not self.needs_admin or is_admin) and
+                (not self.needs_auth or is_logged_in))
+
+    async def __call__(self, evt: CommandEvent) -> Dict:
+        """Executes the command if evt was issued with proper rights.
+
+        Args:
+            evt: The CommandEvent for which to check permissions.
+
+        Returns:
+            The result of the command or the error message function.
+
+        Raises:
+            FloodWaitError
+        """
+        error = await self.get_permission_error(evt)
+        if error is not None:
+            return await evt.reply(error)
+        return await self._handler(evt)
+
+    @property
+    def has_help(self) -> bool:
+        """Returns true if this command has a help text."""
+        return bool(self.help_section) and bool(self._help_text)
+
+    @property
+    def help(self) -> str:
+        """Returns the help text to this command."""
+        return f"**{self.name}** {self._help_args} - {self._help_text}"
+
+
+def command_handler(_func: Optional[Callable[[CommandEvent], Awaitable[Dict]]] = None, *,
+                    needs_auth: bool = True, needs_puppeting: bool = True,
+                    needs_matrix_puppeting: bool = False, needs_admin: bool = False,
+                    management_only: bool = False, name: Optional[str] = None,
+                    help_text: str = "", help_args: str = "", help_section: HelpSection = None
+                    ) -> Callable[[Callable[[CommandEvent], Awaitable[Optional[Dict]]]],
+                                  CommandHandler]:
+    def decorator(func: Callable[[CommandEvent], Awaitable[Optional[Dict]]]) -> CommandHandler:
+        actual_name = name or func.__name__.replace("_", "-")
+        handler = CommandHandler(func, needs_auth, needs_puppeting, needs_matrix_puppeting,
+                                 needs_admin, management_only, actual_name, help_text, help_args,
+                                 help_section)
+        command_handlers[handler.name] = handler
+        return handler
+
+    return decorator if _func is None else decorator(_func)
+
+
+class CommandProcessor:
+    """Handles the raw commands issued by a user to the Matrix bot."""
     log = logging.getLogger("mau.commands")
 
-    def __init__(self, context):
-        self.az, self.db, self.config, self.loop, self.tgbot = context
+    def __init__(self, context: c.Context) -> None:
+        self.az, self.config, self.loop, self.tgbot = context.core
+        self.public_website = context.public_website
         self.command_prefix = self.config["bridge.command_prefix"]
 
-    # region Utility functions for handling commands
+    async def handle(self, room: MatrixRoomID, event_id: MatrixEventID, sender: u.User,
+                     command: str, args: List[str], is_management: bool, is_portal: bool
+                     ) -> Optional[Dict]:
+        """Handles the raw commands issued by a user to the Matrix bot.
 
-    async def handle(self, room, sender, command, args, is_management, is_portal):
-        evt = CommandEvent(self, room, sender, command, args,
-                           is_management, is_portal)
+        If the command is not known, it might be a followup command and is
+        delegated to a command handler registered for that purpose in the
+        senders command_status as "next".
+
+        Args:
+            room: ID of the Matrix room in which the command was issued.
+            event_id: ID of the event by which the command was issued.
+            sender: The sender who issued the command.
+            command: The issued command, case insensitive.
+            args: Arguments given with the command.
+            is_management: Whether the room is a management room.
+            is_portal: Whether the room is a portal.
+
+        Returns:
+            The result of the error message function or None if no error
+            occured. Unknown and delegated commands do not count as errors.
+        """
+        if not command_handlers or "unknown-command" not in command_handlers:
+            raise ValueError("command_handlers are not properly initialized.")
+
+        evt = CommandEvent(self, room, event_id, sender, command, args, is_management, is_portal)
         orig_command = command
         command = command.lower()
         try:
-            command = command_handlers[command]
+            handler = command_handlers[command]
         except KeyError:
             if sender.command_status and "next" in sender.command_status:
                 args.insert(0, orig_command)
                 evt.command = ""
-                command = sender.command_status["next"]
+                handler = sender.command_status["next"]
             else:
-                command = command_handlers["unknown-command"]
+                handler = command_handlers["unknown-command"]
         try:
-            await command(evt)
+            await handler(evt)
         except FloodWaitError as e:
             return await evt.reply(f"Flood error: Please wait {format_duration(e.seconds)}")
         except Exception:
-            self.log.exception("Fatal error handling command "
+            self.log.exception("Unhandled error while handling command "
                                f"{evt.command} {' '.join(args)} from {sender.mxid}")
-            return await evt.reply("Fatal error while handling command. "
+            if evt.sender.is_admin and evt.is_management:
+                return await evt.reply("Unhandled error while handling command:\n\n"
+                                       "```traceback\n"
+                                       f"{traceback.format_exc()}"
+                                       "```")
+            return await evt.reply("Unhandled error while handling command. "
                                    "Check logs for more details.")
+        return None

mautrix_telegram/commands/matrix_auth.py

@@ -0,0 +1,101 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Dict, Optional
+
+from . import command_handler, CommandEvent, SECTION_AUTH
+from .. import puppet as pu
+
+
+@command_handler(needs_auth=True, needs_matrix_puppeting=True,
+                 help_section=SECTION_AUTH,
+                 help_text="Revert your Telegram account's Matrix puppet to use the default Matrix "
+                           "account.")
+async def logout_matrix(evt: CommandEvent) -> Optional[Dict]:
+    puppet = pu.Puppet.get(evt.sender.tgid)
+    if not puppet.is_real_user:
+        return await evt.reply("You are not logged in with your Matrix account.")
+    await puppet.switch_mxid(None, None)
+    return await evt.reply("Reverted your Telegram account's Matrix puppet back to the default.")
+
+
+@command_handler(needs_auth=True, management_only=True, needs_matrix_puppeting=True,
+                 help_section=SECTION_AUTH,
+                 help_text="Replace your Telegram account's Matrix puppet with your own Matrix "
+                           "account.")
+async def login_matrix(evt: CommandEvent) -> Optional[Dict]:
+    puppet = pu.Puppet.get(evt.sender.tgid)
+    if puppet.is_real_user:
+        return await evt.reply("You have already logged in with your Matrix account. "
+                               "Log out with `$cmdprefix+sp logout-matrix` first.")
+    allow_matrix_login = evt.config.get("bridge.allow_matrix_login", True)
+    if allow_matrix_login:
+        evt.sender.command_status = {
+            "next": enter_matrix_token,
+            "action": "Matrix login",
+        }
+    if evt.config["appservice.public.enabled"]:
+        prefix = evt.config["appservice.public.external"]
+        token = evt.public_website.make_token(evt.sender.mxid, "/matrix-login")
+        url = f"{prefix}/matrix-login?token={token}"
+        if allow_matrix_login:
+            return await evt.reply(
+                "This bridge instance allows you to log in inside or outside Matrix.\n\n"
+                "If you would like to log in within Matrix, please send your Matrix access token "
+                "here.\n"
+                f"If you would like to log in outside of Matrix, [click here]({url}).\n\n"
+                "Logging in outside of Matrix is recommended, because in-Matrix login would save "
+                "your access token in the message history.")
+        return await evt.reply("This bridge instance does not allow logging in inside Matrix.\n\n"
+                               f"Please visit [the login page]({url}) to log in.")
+    elif allow_matrix_login:
+        return await evt.reply(
+            "This bridge instance does not allow you to log in outside of Matrix.\n\n"
+            "Please send your Matrix access token here to log in.")
+    return await evt.reply("This bridge instance has been configured to not allow logging in.")
+
+
+@command_handler(needs_auth=True, needs_matrix_puppeting=True,
+                 help_section=SECTION_AUTH,
+                 help_text="Pings the server with the stored matrix authentication.")
+async def ping_matrix(evt: CommandEvent) -> Optional[Dict]:
+    puppet = pu.Puppet.get(evt.sender.tgid)
+    if not puppet.is_real_user:
+        return await evt.reply("You are not logged in with your Matrix account.")
+    resp = await puppet.init_custom_mxid()
+    if resp == pu.PuppetError.InvalidAccessToken:
+        return await evt.reply("Your access token is invalid.")
+    elif resp == pu.PuppetError.Success:
+        return await evt.reply("Your Matrix login is working.")
+    return await evt.reply(f"Unknown response while checking your Matrix login: {resp}.")
+
+
+async def enter_matrix_token(evt: CommandEvent) -> Dict:
+    evt.sender.command_status = None
+
+    puppet = pu.Puppet.get(evt.sender.tgid)
+    if puppet.is_real_user:
+        return await evt.reply("You have already logged in with your Matrix account. "
+                               "Log out with `$cmdprefix+sp logout-matrix` first.")
+
+    resp = await puppet.switch_mxid(" ".join(evt.args), evt.sender.mxid)
+    if resp == pu.PuppetError.OnlyLoginSelf:
+        return await evt.reply("You can only log in as your own Matrix user.")
+    elif resp == pu.PuppetError.InvalidAccessToken:
+        return await evt.reply("Failed to verify access token.")
+    assert resp == pu.PuppetError.Success, "Encountered an unhandled PuppetError."
+    return await evt.reply(
+        f"Replaced your Telegram account's Matrix puppet with {puppet.custom_mxid}.")

mautrix_telegram/commands/meta.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -14,75 +14,59 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from . import command_handler
+from typing import Dict, List, Optional, Tuple
+
+from . import command_handler, CommandEvent, _command_handlers, SECTION_GENERAL
+from .handler import HelpSection
 
 
-@command_handler(needs_auth=False)
-def cancel(evt):
+@command_handler(needs_auth=False, needs_puppeting=False,
+                 help_section=SECTION_GENERAL,
+                 help_text="Cancel an ongoing action (such as login)")
+async def cancel(evt: CommandEvent) -> Optional[Dict]:
     if evt.sender.command_status:
         action = evt.sender.command_status["action"]
         evt.sender.command_status = None
-        return evt.reply(f"{action} cancelled.")
+        return await evt.reply(f"{action} cancelled.")
     else:
-        return evt.reply("No ongoing command.")
+        return await evt.reply("No ongoing command.")
 
 
-@command_handler(needs_auth=False)
-def unknown_command(evt):
-    return evt.reply("Unknown command. Try `$cmdprefix+sp help` for help.")
+@command_handler(needs_auth=False, needs_puppeting=False)
+async def unknown_command(evt: CommandEvent) -> Optional[Dict]:
+    return await evt.reply("Unknown command. Try `$cmdprefix+sp help` for help.")
 
 
-@command_handler(needs_auth=False)
-def help(evt):
+help_cache = {}  # type: Dict[Tuple[bool, bool, bool, bool, bool], str]
+
+
+async def _get_help_text(evt: CommandEvent) -> str:
+    cache_key = (evt.is_management, evt.sender.puppet_whitelisted,
+                 evt.sender.matrix_puppet_whitelisted, evt.sender.is_admin,
+                 await evt.sender.is_logged_in())
+    if cache_key not in help_cache:
+        help_sections = {}  # type: Dict[HelpSection, List[str]]
+        for handler in _command_handlers.values():
+            if handler.has_help and handler.has_permission(*cache_key):
+                help_sections.setdefault(handler.help_section, [])
+                help_sections[handler.help_section].append(handler.help + "  ")
+        help_sorted = sorted(help_sections.items(), key=lambda item: item[0].order)
+        helps = ["#### {}\n{}\n".format(key.name, "\n".join(value)) for key, value in help_sorted]
+        help_cache[cache_key] = "\n".join(helps)
+    return help_cache[cache_key]
+
+
+def _get_management_status(evt: CommandEvent) -> str:
     if evt.is_management:
-        management_status = ("This is a management room: prefixing commands "
-                             "with `$cmdprefix` is not required.\n")
+        return "This is a management room: prefixing commands with `$cmdprefix` is not required."
     elif evt.is_portal:
-        management_status = ("**This is a portal room**: you must always "
-                             "prefix commands with `$cmdprefix`.\n"
-                             "Management commands will not be sent to Telegram.")
-    else:
-        management_status = ("**This is not a management room**: you must "
-                             "prefix commands with `$cmdprefix`.\n")
-    help = """\n
-#### Generic bridge commands
-**help**   - Show this help message.  
-**cancel** - Cancel an ongoing action (such as login).
+        return ("**This is a portal room**: you must always prefix commands with `$cmdprefix`.\n"
+                "Management commands will not be sent to Telegram.")
+    return "**This is not a management room**: you must prefix commands with `$cmdprefix`."
 
-#### Authentication
-**login**  - Request an authentication code.  
-**logout** - Log out from Telegram.  
-**ping**   - Check if you're logged into Telegram.
 
-#### Miscellaneous things
-**search** [_-r|--remote_] <_query_> - Search your contacts or the Telegram servers for users.  
-**sync** [`chats`|`contacts`|`me`]   - Synchronize your chat portals, contacts and/or own info.  
-**ping-bot**                         - Get info of the message relay Telegram bot.  
-**set-pl** <_level_> [_mxid_]        - Set a temporary power level without affecting Telegram.
-
-#### Initiating chats
-**pm** <_identifier_> - Open a private chat with the given Telegram user. The identifier is either
-                        the internal user ID, the username or the phone number.  
-**join** <_link_>     - Join a chat with an invite link.  
-**create** [_type_]   - Create a Telegram chat of the given type for the current Matrix room. The
-                        type is either `group`, `supergroup` or `channel` (defaults to `group`).
-
-#### Portal management  
-**upgrade**                 - Upgrade a normal Telegram group to a supergroup.  
-**invite-link**             - Get a Telegram invite link to the current chat.  
-**delete-portal**           - Remove all users from the current portal room and forget the portal.
-                              Only works for group chats; to delete a private chat portal, simply
-                              leave the room.  
-**unbridge**                - Remove puppets from the current portal room and forget the portal.  
-**bridge** [_id_]           - Bridge the current Matrix room to the Telegram chat with the given
-                              ID. The ID must be the prefixed version that you get with the `/id`
-                              command of the Telegram-side bot.  
-**group-name** <_name_|`-`> - Change the username of a supergroup/channel. To disable, use a dash
-                             (`-`) as the name.  
-**clean-rooms**             - Clean up unused portal/management rooms.
-
-**filter** <`whitelist`|`blacklist`> <_chat ID_> - Allow or disallow bridging a specific chat.  
-**filter-mode** <`whitelist`|`blacklist`>      - Change whether the bridge will allow or disallow
-                                                 bridging rooms by default.
-"""
-    return evt.reply(management_status + help)
+@command_handler(name="help", needs_auth=False, needs_puppeting=False,
+                 help_section=SECTION_GENERAL,
+                 help_text="Show this help message.")
+async def help_cmd(evt: CommandEvent) -> Optional[Dict]:
+    return await evt.reply(_get_management_status(evt) + "\n" + await _get_help_text(evt))

mautrix_telegram/commands/portal.py

@@ -1,426 +0,0 @@
-# -*- coding: future_fstrings -*-
-# mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
-import asyncio
-
-from telethon.errors import *
-from mautrix_appservice import MatrixRequestError
-
-from .. import portal as po
-from . import command_handler, CommandEvent
-
-
-@command_handler(needs_admin=True, needs_auth=False, name="set-pl")
-async def set_power_level(evt: CommandEvent):
-    try:
-        level = int(evt.args[0])
-    except KeyError:
-        return await evt.reply("**Usage:** `$cmdprefix+sp set-power <level> [mxid]`")
-    except ValueError:
-        return await evt.reply("The level must be an integer.")
-    levels = await evt.az.intent.get_power_levels(evt.room_id)
-    mxid = evt.args[1] if len(evt.args) > 1 else evt.sender.mxid
-    levels["users"][mxid] = level
-    try:
-        await evt.az.intent.set_power_levels(evt.room_id, levels)
-    except MatrixRequestError:
-        evt.log.exception("Failed to set power level.")
-        return await evt.reply("Failed to set power level.")
-
-
-@command_handler()
-async def invite_link(evt: CommandEvent):
-    portal = po.Portal.get_by_mxid(evt.room_id)
-    if not portal:
-        return await evt.reply("This is not a portal room.")
-
-    if portal.peer_type == "user":
-        return await evt.reply("You can't invite users to private chats.")
-
-    try:
-        link = await portal.get_invite_link(evt.sender)
-        return await evt.reply(f"Invite link to {portal.title}: {link}")
-    except ValueError as e:
-        return await evt.reply(e.args[0])
-    except ChatAdminRequiredError:
-        return await evt.reply("You don't have the permission to create an invite link.")
-
-
-async def _has_access_to(room, intent, sender, event, default=50):
-    if sender.is_admin:
-        return True
-    # Make sure the state store contains the power levels.
-    try:
-        await intent.get_power_levels(room)
-    except MatrixRequestError:
-        return False
-    return intent.state_store.has_power_level(room, sender.mxid,
-                                              event=f"net.maunium.telegram.{event}",
-                                              default=default)
-
-
-async def _get_portal_and_check_permission(evt, permission, action=None):
-    room_id = evt.args[0] if len(evt.args) > 0 else evt.room_id
-
-    portal = po.Portal.get_by_mxid(room_id)
-    if not portal:
-        that_this = "This" if room_id == evt.room_id else "That"
-        return await evt.reply(f"{that_this} is not a portal room."), False
-
-    if not await _has_access_to(portal.mxid, evt.az.intent, evt.sender, permission):
-        action = action or f"{permission.replace('_', ' ')}s"
-        return await evt.reply(f"You do not have the permissions to {action} that portal."), False
-    return portal, True
-
-
-def _get_portal_murder_function(action, room_id, function, command, completed_message):
-    async def post_confirm(confirm):
-        confirm.sender.command_status = None
-        if len(confirm.args) > 0 and confirm.args[0] == f"confirm-{command}":
-            await function()
-            if confirm.room_id != room_id:
-                return await confirm.reply(completed_message)
-        else:
-            return await confirm.reply(f"{action} cancelled.")
-
-    return {
-        "next": post_confirm,
-        "action": action,
-    }
-
-
-@command_handler()
-async def delete_portal(evt: CommandEvent):
-    portal, ok = await _get_portal_and_check_permission(evt, "delete_portal")
-    if not ok:
-        return
-
-    evt.sender.command_status = _get_portal_murder_function("Portal deletion", portal.mxid,
-                                                            portal.cleanup_and_delete, "delete",
-                                                            "Portal successfully deleted.")
-    return await evt.reply("Please confirm deletion of portal "
-                           f"[{portal.alias or portal.mxid}](https://matrix.to/#/{portal.mxid}) "
-                           f"to Telegram chat \"{portal.title}\" "
-                           "by typing `$cmdprefix+sp confirm-delete`")
-
-
-@command_handler()
-async def unbridge(evt: CommandEvent):
-    portal, ok = await _get_portal_and_check_permission(evt, "unbridge_room")
-    if not ok:
-        return
-
-    evt.sender.command_status = _get_portal_murder_function("Room unbridging", portal.mxid,
-                                                            portal.unbridge, "unbridge",
-                                                            "Room successfully unbridged.")
-    return await evt.reply(f"Please confirm unbridging chat \"{portal.title}\" from room "
-                           f"[{portal.alias or portal.mxid}](https://matrix.to/#/{portal.mxid}) "
-                           "by typing `$cmdprefix+sp confirm-unbridge`")
-
-
-@command_handler()
-async def bridge(evt: CommandEvent):
-    if len(evt.args) == 0:
-        return await evt.reply("**Usage:** "
-                               "`$cmdprefix+sp bridge <Telegram chat ID> [Matrix room ID]`")
-    room_id = evt.args[1] if len(evt.args) > 1 else evt.room_id
-    that_this = "This" if room_id == evt.room_id else "That"
-
-    portal = po.Portal.get_by_mxid(room_id)
-    if portal:
-        return await evt.reply(f"{that_this} room is already a portal room.")
-
-    if not await _has_access_to(room_id, evt.az.intent, evt.sender, "bridge"):
-        return await evt.reply("You do not have the permissions to bridge that room.")
-
-    # The /id bot command provides the prefixed ID, so we assume
-    tgid = evt.args[0]
-    if tgid.startswith("-100"):
-        tgid = int(tgid[4:])
-        peer_type = "channel"
-    elif tgid.startswith("-"):
-        tgid = -int(tgid)
-        peer_type = "chat"
-    else:
-        return await evt.reply("That doesn't seem like a prefixed Telegram chat ID.\n\n"
-                               "If you did not get the ID using the `/id` bot command, please "
-                               "prefix channel IDs with `-100` and normal group IDs with `-`.\n\n"
-                               "Bridging private chats to existing rooms is not allowed.")
-
-    portal = po.Portal.get_by_tgid(tgid, peer_type=peer_type)
-    if not portal.allow_bridging():
-        return await evt.reply("This bridge doesn't allow bridging that Telegram chat.\n"
-                               "If you're the bridge admin, try"
-                               "`$cmdprefix+sp whitelist <Telegram chat ID>` first.")
-    if portal.mxid:
-        has_portal_message = (
-            "That Telegram chat already has a portal at "
-            f"[{portal.alias or portal.mxid}](https://matrix.to/#/{portal.mxid}). ")
-        if not await _has_access_to(portal.mxid, evt.az.intent, evt.sender, "unbridge"):
-            return await evt.reply(f"{has_portal_message}"
-                                   "Additionally, you do not have the permissions to unbridge "
-                                   "that room.")
-        evt.sender.command_status = {
-            "next": confirm_bridge,
-            "action": "Room bridging",
-            "mxid": portal.mxid,
-            "bridge_to_mxid": room_id,
-            "tgid": portal.tgid,
-            "peer_type": portal.peer_type,
-        }
-        return await evt.reply(f"{has_portal_message}"
-                               "However, you have the permissions to unbridge that room.\n\n"
-                               "To delete that portal completely and continue bridging, use "
-                               "`$cmdprefix+sp delete-and-continue`. To unbridge the portal "
-                               "without kicking Matrix users, use `$cmdprefix+sp unbridge-and-"
-                               "continue`. To cancel, use `$cmdprefix+sp cancel`")
-    evt.sender.command_status = {
-        "next": confirm_bridge,
-        "action": "Room bridging",
-        "bridge_to_mxid": room_id,
-        "tgid": portal.tgid,
-        "peer_type": portal.peer_type,
-    }
-    return await evt.reply("That Telegram chat has no existing portal. To confirm bridging the "
-                           "chat to this room, use `$cmdprefix+sp continue`")
-
-
-async def cleanup_old_portal_while_bridging(evt, portal):
-    if not portal.mxid:
-        await evt.reply("The portal seems to have lost its Matrix room between you"
-                        "calling `$cmdprefix+sp bridge` and this command.\n\n"
-                        "Continuing without touching previous Matrix room...")
-        return True, None
-    elif evt.args[0] == "delete-and-continue":
-        return True, portal.cleanup_room(portal.main_intent, portal.mxid,
-                                         message="Portal deleted (moving to another room)")
-    elif evt.args[0] == "unbridge-and-continue":
-        return True, portal.cleanup_room(portal.main_intent, portal.mxid,
-                                         message="Room unbridged (portal moving to another room)",
-                                         puppets_only=True)
-    else:
-        await evt.reply(
-            "The chat you were trying to bridge already has a Matrix portal room.\n\n"
-            "Please use `$cmdprefix+sp delete-and-continue` or `$cmdprefix+sp unbridge-and-"
-            "continue` to either delete or unbridge the existing room (respectively) and "
-            "continue with the bridging.\n\n"
-            "If you changed your mind, use `$cmdprefix+sp cancel` to cancel.")
-        return False, None
-
-
-async def confirm_bridge(evt: CommandEvent):
-    status = evt.sender.command_status
-    try:
-        portal = po.Portal.get_by_tgid(status["tgid"], peer_type=status["peer_type"])
-        bridge_to_mxid = status["bridge_to_mxid"]
-    except KeyError:
-        evt.sender.command_status = None
-        return await evt.reply("Fatal error: tgid or peer_type missing from command_status. "
-                               "This shouldn't happen unless you're messing with the command "
-                               "handler code.")
-    if "mxid" in status:
-        ok, coro = await cleanup_old_portal_while_bridging(evt, portal)
-        if not ok:
-            return
-        elif coro:
-            asyncio.ensure_future(coro, loop=evt.loop)
-            await evt.reply("Cleaning up previous portal room...")
-    elif portal.mxid:
-        evt.sender.command_status = None
-        return await evt.reply("The portal seems to have created a Matrix room between you "
-                               "calling `$cmdprefix+sp bridge` and this command.\n\n"
-                               "Please start over by calling the bridge command again.")
-    elif evt.args[0] != "continue":
-        return await evt.reply("Please use `$cmdprefix+sp continue` to confirm the bridging or "
-                               "`$cmdprefix+sp cancel` to cancel.")
-
-    portal.mxid = bridge_to_mxid
-    portal.title, portal.about, levels = await _get_initial_state(evt)
-    portal.photo_id = ""
-    portal.save()
-
-    entity = await evt.sender.client.get_entity(portal.peer)
-    direct = False
-    asyncio.ensure_future(portal.update_matrix_room(evt.sender, entity, direct, levels=levels),
-                          loop=evt.loop)
-
-    return await evt.reply("Bridging complete. Portal synchronization should begin momentarily.")
-
-
-async def _get_initial_state(evt: CommandEvent):
-    state = await evt.az.intent.get_room_state(evt.room_id)
-    title = None
-    about = None
-    levels = None
-    for event in state:
-        if event["type"] == "m.room.name":
-            title = event["content"]["name"]
-        elif event["type"] == "m.room.topic":
-            about = event["content"]["topic"]
-        elif event["type"] == "m.room.power_levels":
-            levels = event["content"]
-        elif event["type"] == "m.room.canonical_alias":
-            title = title or event["content"]["alias"]
-    return title, about, levels
-
-
-@command_handler()
-async def create(evt: CommandEvent):
-    type = evt.args[0] if len(evt.args) > 0 else "group"
-    if type not in {"chat", "group", "supergroup", "channel"}:
-        return await evt.reply(
-            "**Usage:** `$cmdprefix+sp create ['group'/'supergroup'/'channel']`")
-
-    if po.Portal.get_by_mxid(evt.room_id):
-        return await evt.reply("This is already a portal room.")
-
-    title, about, levels = await _get_initial_state(evt)
-    if not title:
-        return await evt.reply("Please set a title before creating a Telegram chat.")
-
-    supergroup = type == "supergroup"
-    type = {
-        "supergroup": "channel",
-        "channel": "channel",
-        "chat": "chat",
-        "group": "chat",
-    }[type]
-
-    portal = po.Portal(tgid=None, mxid=evt.room_id, title=title, about=about, peer_type=type)
-    try:
-        await portal.create_telegram_chat(evt.sender, supergroup=supergroup)
-    except ValueError as e:
-        portal.delete()
-        return await evt.reply(e.args[0])
-    return await evt.reply(f"Telegram chat created. ID: {portal.tgid}")
-
-
-@command_handler()
-async def upgrade(evt: CommandEvent):
-    portal = po.Portal.get_by_mxid(evt.room_id)
-    if not portal:
-        return await evt.reply("This is not a portal room.")
-    elif portal.peer_type == "channel":
-        return await evt.reply("This is already a supergroup or a channel.")
-    elif portal.peer_type == "user":
-        return await evt.reply("You can't upgrade private chats.")
-
-    try:
-        await portal.upgrade_telegram_chat(evt.sender)
-        return await evt.reply(f"Group upgraded to supergroup. New ID: {portal.tgid}")
-    except ChatAdminRequiredError:
-        return await evt.reply("You don't have the permission to upgrade this group.")
-    except ValueError as e:
-        return await evt.reply(e.args[0])
-
-
-@command_handler()
-async def group_name(evt: CommandEvent):
-    if len(evt.args) == 0:
-        return await evt.reply("**Usage:** `$cmdprefix+sp group-name <name/->`")
-
-    portal = po.Portal.get_by_mxid(evt.room_id)
-    if not portal:
-        return await evt.reply("This is not a portal room.")
-    elif portal.peer_type != "channel":
-        return await evt.reply("Only channels and supergroups have usernames.")
-
-    try:
-        await portal.set_telegram_username(evt.sender,
-                                           evt.args[0] if evt.args[0] != "-" else "")
-        if portal.username:
-            return await evt.reply(f"Username of channel changed to {portal.username}.")
-        else:
-            return await evt.reply(f"Channel is now private.")
-    except ChatAdminRequiredError:
-        return await evt.reply(
-            "You don't have the permission to set the username of this channel.")
-    except UsernameNotModifiedError:
-        if portal.username:
-            return await evt.reply("That is already the username of this channel.")
-        else:
-            return await evt.reply("This channel is already private")
-    except UsernameOccupiedError:
-        return await evt.reply("That username is already in use.")
-    except UsernameInvalidError:
-        return await evt.reply("Invalid username")
-
-
-@command_handler(needs_admin=True)
-async def filter_mode(evt: CommandEvent):
-    try:
-        mode = evt.args[0]
-        if mode not in ("whitelist", "blacklist"):
-            raise ValueError()
-    except (IndexError, ValueError):
-        return await evt.reply("**Usage:** `$cmdprefix+sp filter-mode <whitelist/blacklist>`")
-
-    evt.config["bridge.filter.mode"] = mode
-    evt.config.save()
-    po.Portal.filter_mode = mode
-    if mode == "whitelist":
-        return await evt.reply("The bridge will now disallow bridging chats by default.\n"
-                               "To allow bridging a specific chat, use"
-                               "`!filter whitelist <chat ID>`.")
-    else:
-        return await evt.reply("The bridge will now allow bridging chats by default.\n"
-                               "To disallow bridging a specific chat, use"
-                               "`!filter blacklist <chat ID>`.")
-
-
-@command_handler(needs_admin=True)
-async def filter(evt: CommandEvent):
-    try:
-        action = evt.args[0]
-        if action not in ("whitelist", "blacklist", "add", "remove"):
-            raise ValueError()
-
-        id = evt.args[1]
-        if id.startswith("-100"):
-            id = int(id[4:])
-        elif id.startswith("-"):
-            id = int(id[1:])
-        else:
-            id = int(id)
-    except (IndexError, ValueError):
-        return await evt.reply("**Usage:** `$cmdprefix+sp filter <whitelist/blacklist> <chat ID>`")
-
-    mode = evt.config["bridge.filter.mode"]
-    if mode not in ("blacklist", "whitelist"):
-        return await evt.reply(f"Unknown filter mode \"{mode}\". Please fix the bridge config.")
-
-    list = evt.config["bridge.filter.list"]
-
-    if action in ("blacklist", "whitelist"):
-        action = "add" if mode == action else "remove"
-
-    def save():
-        evt.config["bridge.filter.list"] = list
-        evt.config.save()
-        po.Portal.filter_list = list
-
-    if action == "add":
-        if id in list:
-            return await evt.reply(f"That chat is already {mode}ed.")
-        list.append(id)
-        save()
-        return await evt.reply(f"Chat ID added to {mode}.")
-    elif action == "remove":
-        if id not in list:
-            return await evt.reply(f"That chat is not {mode}ed.")
-        list.remove(id)
-        save()
-        return await evt.reply(f"Chat ID removed from {mode}.")

mautrix_telegram/commands/portal/__init__.py

@@ -0,0 +1 @@
+from . import admin, bridge, config, create_chat, filter, misc, unbridge

mautrix_telegram/commands/portal/admin.py

@@ -0,0 +1,101 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Dict
+import asyncio
+
+from mautrix_appservice import MatrixRequestError
+
+from ... import portal as po, puppet as pu, user as u
+from .. import command_handler, CommandEvent, SECTION_ADMIN
+
+
+@command_handler(needs_admin=True, needs_auth=False, name="set-pl",
+                 help_section=SECTION_ADMIN,
+                 help_args="<_level_> [_mxid_]",
+                 help_text="Set a temporary power level without affecting Telegram.")
+async def set_power_level(evt: CommandEvent) -> Dict:
+    try:
+        level = int(evt.args[0])
+    except KeyError:
+        return await evt.reply("**Usage:** `$cmdprefix+sp set-pl <level> [mxid]`")
+    except ValueError:
+        return await evt.reply("The level must be an integer.")
+    levels = await evt.az.intent.get_power_levels(evt.room_id)
+    mxid = evt.args[1] if len(evt.args) > 1 else evt.sender.mxid
+    levels["users"][mxid] = level
+    try:
+        await evt.az.intent.set_power_levels(evt.room_id, levels)
+    except MatrixRequestError:
+        evt.log.exception("Failed to set power level.")
+        return await evt.reply("Failed to set power level.")
+    return {}
+
+
+@command_handler(needs_admin=True, needs_auth=False,
+                 help_section=SECTION_ADMIN,
+                 help_args="<`portal`|`puppet`|`user`>",
+                 help_text="Clear internal bridge caches")
+async def clear_db_cache(evt: CommandEvent) -> Dict:
+    try:
+        section = evt.args[0].lower()
+    except IndexError:
+        return await evt.reply("**Usage:** `$cmdprefix+sp clear-db-cache <section>`")
+    if section == "portal":
+        po.Portal.by_tgid = {}
+        po.Portal.by_mxid = {}
+        await evt.reply("Cleared portal cache")
+    elif section == "puppet":
+        pu.Puppet.cache = {}
+        for puppet in pu.Puppet.by_custom_mxid.values():
+            puppet.sync_task.cancel()
+        pu.Puppet.by_custom_mxid = {}
+        await asyncio.gather(
+            *[puppet.init_custom_mxid() for puppet in pu.Puppet.all_with_custom_mxid()],
+            loop=evt.loop)
+        await evt.reply("Cleared puppet cache and restarted custom puppet syncers")
+    elif section == "user":
+        u.User.by_mxid = {
+            user.mxid: user
+            for user in u.User.by_tgid.values()
+        }
+        await evt.reply("Cleared non-logged-in user cache")
+    else:
+        return await evt.reply("**Usage:** `$cmdprefix+sp clear-db-cache <section>`")
+
+
+@command_handler(needs_admin=True, needs_auth=False,
+                 help_section=SECTION_ADMIN,
+                 help_args="[_mxid_]",
+                 help_text="Reload and reconnect a user")
+async def reload_user(evt: CommandEvent) -> Dict:
+    if len(evt.args) > 0:
+        mxid = evt.args[0]
+    else:
+        mxid = evt.sender.mxid
+    user = u.User.get_by_mxid(mxid, create=False)
+    if not user:
+        return await evt.reply("User not found")
+    puppet = pu.Puppet.get_by_custom_mxid(mxid)
+    if puppet:
+        puppet.sync_task.cancel()
+    await user.stop()
+    user.delete(delete_db=False)
+    user = u.User.get_by_mxid(mxid)
+    await user.ensure_started()
+    if puppet:
+        await puppet.init_custom_mxid()
+    await evt.reply(f"Reloaded and reconnected {user.mxid} (telegram: {user.human_tg_id})")

mautrix_telegram/commands/portal/bridge.py

@@ -0,0 +1,187 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Dict, Optional, Tuple, Coroutine
+import asyncio
+
+from telethon.tl.types import ChatForbidden, ChannelForbidden
+
+from ...types import MatrixRoomID, TelegramID
+from ...util import ignore_coro
+from ... import portal as po
+from .. import command_handler, CommandEvent, SECTION_CREATING_PORTALS
+from .util import user_has_power_level, get_initial_state
+
+
+@command_handler(needs_auth=False, needs_puppeting=False,
+                 help_section=SECTION_CREATING_PORTALS,
+                 help_args="[_id_]",
+                 help_text="Bridge the current Matrix room to the Telegram chat with the given "
+                           "ID. The ID must be the prefixed version that you get with the `/id` "
+                           "command of the Telegram-side bot.")
+async def bridge(evt: CommandEvent) -> Dict:
+    if len(evt.args) == 0:
+        return await evt.reply("**Usage:** "
+                               "`$cmdprefix+sp bridge <Telegram chat ID> [Matrix room ID]`")
+    force_use_bot = False
+    if evt.args[0] == "--usebot" and evt.sender.is_admin:
+        force_use_bot = True
+        evt.args = evt.args[1:]
+    room_id = MatrixRoomID(evt.args[1]) if len(evt.args) > 1 else evt.room_id
+    that_this = "This" if room_id == evt.room_id else "That"
+
+    portal = po.Portal.get_by_mxid(room_id)
+    if portal:
+        return await evt.reply(f"{that_this} room is already a portal room.")
+
+    if not await user_has_power_level(room_id, evt.az.intent, evt.sender, "bridge"):
+        return await evt.reply(f"You do not have the permissions to bridge {that_this} room.")
+
+    # The /id bot command provides the prefixed ID, so we assume
+    tgid_str = evt.args[0]
+    if tgid_str.startswith("-100"):
+        tgid = TelegramID(int(tgid_str[4:]))
+        peer_type = "channel"
+    elif tgid_str.startswith("-"):
+        tgid = TelegramID(-int(tgid_str))
+        peer_type = "chat"
+    else:
+        return await evt.reply("That doesn't seem like a prefixed Telegram chat ID.\n\n"
+                               "If you did not get the ID using the `/id` bot command, please "
+                               "prefix channel IDs with `-100` and normal group IDs with `-`.\n\n"
+                               "Bridging private chats to existing rooms is not allowed.")
+
+    portal = po.Portal.get_by_tgid(tgid, peer_type=peer_type)
+    if not portal.allow_bridging():
+        return await evt.reply("This bridge doesn't allow bridging that Telegram chat.\n"
+                               "If you're the bridge admin, try "
+                               "`$cmdprefix+sp filter whitelist <Telegram chat ID>` first.")
+    if portal.mxid:
+        has_portal_message = (
+            "That Telegram chat already has a portal at "
+            f"[{portal.alias or portal.mxid}](https://matrix.to/#/{portal.mxid}). ")
+        if not await user_has_power_level(portal.mxid, evt.az.intent, evt.sender, "unbridge"):
+            return await evt.reply(f"{has_portal_message}"
+                                   "Additionally, you do not have the permissions to unbridge "
+                                   "that room.")
+        evt.sender.command_status = {
+            "next": confirm_bridge,
+            "action": "Room bridging",
+            "mxid": portal.mxid,
+            "bridge_to_mxid": room_id,
+            "tgid": portal.tgid,
+            "peer_type": portal.peer_type,
+            "force_use_bot": force_use_bot,
+        }
+        return await evt.reply(f"{has_portal_message}"
+                               "However, you have the permissions to unbridge that room.\n\n"
+                               "To delete that portal completely and continue bridging, use "
+                               "`$cmdprefix+sp delete-and-continue`. To unbridge the portal "
+                               "without kicking Matrix users, use `$cmdprefix+sp unbridge-and-"
+                               "continue`. To cancel, use `$cmdprefix+sp cancel`")
+    evt.sender.command_status = {
+        "next": confirm_bridge,
+        "action": "Room bridging",
+        "bridge_to_mxid": room_id,
+        "tgid": portal.tgid,
+        "peer_type": portal.peer_type,
+        "force_use_bot": force_use_bot,
+    }
+    return await evt.reply("That Telegram chat has no existing portal. To confirm bridging the "
+                           "chat to this room, use `$cmdprefix+sp continue`")
+
+
+async def cleanup_old_portal_while_bridging(evt: CommandEvent, portal: "po.Portal"
+                                            ) -> Tuple[bool, Optional[Coroutine[None, None, None]]]:
+    if not portal.mxid:
+        await evt.reply("The portal seems to have lost its Matrix room between you"
+                        "calling `$cmdprefix+sp bridge` and this command.\n\n"
+                        "Continuing without touching previous Matrix room...")
+        return True, None
+    elif evt.args[0] == "delete-and-continue":
+        return True, portal.cleanup_room(portal.main_intent, portal.mxid,
+                                         message="Portal deleted (moving to another room)")
+    elif evt.args[0] == "unbridge-and-continue":
+        return True, portal.cleanup_room(portal.main_intent, portal.mxid,
+                                         message="Room unbridged (portal moving to another room)",
+                                         puppets_only=True)
+    else:
+        await evt.reply(
+            "The chat you were trying to bridge already has a Matrix portal room.\n\n"
+            "Please use `$cmdprefix+sp delete-and-continue` or `$cmdprefix+sp unbridge-and-"
+            "continue` to either delete or unbridge the existing room (respectively) and "
+            "continue with the bridging.\n\n"
+            "If you changed your mind, use `$cmdprefix+sp cancel` to cancel.")
+        return False, None
+
+
+async def confirm_bridge(evt: CommandEvent) -> Optional[Dict]:
+    status = evt.sender.command_status
+    try:
+        portal = po.Portal.get_by_tgid(status["tgid"], peer_type=status["peer_type"])
+        bridge_to_mxid = status["bridge_to_mxid"]
+    except KeyError:
+        evt.sender.command_status = None
+        return await evt.reply("Fatal error: tgid or peer_type missing from command_status. "
+                               "This shouldn't happen unless you're messing with the command "
+                               "handler code.")
+    if "mxid" in status:
+        ok, coro = await cleanup_old_portal_while_bridging(evt, portal)
+        if not ok:
+            return None
+        elif coro:
+            ignore_coro(asyncio.ensure_future(coro, loop=evt.loop))
+            await evt.reply("Cleaning up previous portal room...")
+    elif portal.mxid:
+        evt.sender.command_status = None
+        return await evt.reply("The portal seems to have created a Matrix room between you "
+                               "calling `$cmdprefix+sp bridge` and this command.\n\n"
+                               "Please start over by calling the bridge command again.")
+    elif evt.args[0] != "continue":
+        return await evt.reply("Please use `$cmdprefix+sp continue` to confirm the bridging or "
+                               "`$cmdprefix+sp cancel` to cancel.")
+
+    evt.sender.command_status = None
+    is_logged_in = await evt.sender.is_logged_in() and not status["force_use_bot"]
+    user = evt.sender if is_logged_in else evt.tgbot
+    try:
+        entity = await user.client.get_entity(portal.peer)
+    except Exception:
+        evt.log.exception("Failed to get_entity(%s) for manual bridging.", portal.peer)
+        if is_logged_in:
+            return await evt.reply("Failed to get info of telegram chat. "
+                                   "You are logged in, are you in that chat?")
+        else:
+            return await evt.reply("Failed to get info of telegram chat. "
+                                   "You're not logged in, is the relay bot in the chat?")
+    if isinstance(entity, (ChatForbidden, ChannelForbidden)):
+        if is_logged_in:
+            return await evt.reply("You don't seem to be in that chat.")
+        else:
+            return await evt.reply("The bot doesn't seem to be in that chat.")
+
+    direct = False
+
+    portal.mxid = bridge_to_mxid
+    portal.title, portal.about, levels = await get_initial_state(evt.az.intent, evt.room_id)
+    portal.photo_id = ""
+    portal.save()
+
+    ignore_coro(asyncio.ensure_future(portal.update_matrix_room(user, entity, direct,
+                                                                levels=levels),
+                                      loop=evt.loop))
+
+    return await evt.reply("Bridging complete. Portal synchronization should begin momentarily.")

mautrix_telegram/commands/portal/config.py

@@ -0,0 +1,132 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Dict, Awaitable
+from io import StringIO
+
+from ...config import yaml
+from ... import portal as po, util
+from .. import command_handler, CommandEvent, SECTION_PORTAL_MANAGEMENT
+
+
+@command_handler(help_section=SECTION_PORTAL_MANAGEMENT,
+                 help_text="View or change per-portal settings.",
+                 help_args="<`help`|_subcommand_> [...]")
+async def config(evt: CommandEvent) -> None:
+    cmd = evt.args[0].lower() if len(evt.args) > 0 else "help"
+    if cmd not in ("view", "defaults", "set", "unset", "add", "del"):
+        await config_help(evt)
+        return
+    elif cmd == "defaults":
+        await config_defaults(evt)
+        return
+
+    portal = po.Portal.get_by_mxid(evt.room_id)
+    if not portal:
+        await evt.reply("This is not a portal room.")
+        return
+    elif cmd == "view":
+        await config_view(evt, portal)
+        return
+
+    key = evt.args[1] if len(evt.args) > 1 else None
+    value = yaml.load(" ".join(evt.args[2:])) if len(evt.args) > 2 else None
+    if cmd == "set":
+        await config_set(evt, portal, key, value)
+    elif cmd == "unset":
+        await config_unset(evt, portal, key)
+    elif cmd == "add" or cmd == "del":
+        await config_add_del(evt, portal, key, value, cmd)
+    else:
+        return
+    portal.save()
+
+
+def config_help(evt: CommandEvent) -> Awaitable[Dict]:
+    return evt.reply("""**Usage:** `$cmdprefix config <subcommand> [...]`. Subcommands:
+
+* **help** - View this help text.
+* **view** - View the current config data.
+* **defaults** - View the default config values.
+* **set** <_key_> <_value_> - Set a config value.
+* **unset** <_key_> - Remove a config value.
+* **add** <_key_> <_value_> - Add a value to an array.
+* **del** <_key_> <_value_> - Remove a value from an array.
+""")
+
+
+def config_view(evt: CommandEvent, portal: po.Portal) -> Awaitable[Dict]:
+    stream = StringIO()
+    yaml.dump(portal.local_config, stream)
+    return evt.reply(f"Room-specific config:\n\n```yaml\n{stream.getvalue()}```")
+
+
+def config_defaults(evt: CommandEvent) -> Awaitable[Dict]:
+    stream = StringIO()
+    yaml.dump({
+        "bridge_notices": {
+            "default": evt.config["bridge.bridge_notices.default"],
+            "exceptions": evt.config["bridge.bridge_notices.exceptions"],
+        },
+        "bot_messages_as_notices": evt.config["bridge.bot_messages_as_notices"],
+        "inline_images": evt.config["bridge.inline_images"],
+        "message_formats": evt.config["bridge.message_formats"],
+        "state_event_formats": evt.config["bridge.state_event_formats"],
+        "telegram_link_preview": evt.config["bridge.telegram_link_preview"],
+    }, stream)
+    return evt.reply(f"Bridge instance wide config:\n\n```yaml\n{stream.getvalue()}```")
+
+
+def config_set(evt: CommandEvent, portal: po.Portal, key: str, value: str) -> Awaitable[Dict]:
+    if not key or value is None:
+        return evt.reply(f"**Usage:** `$cmdprefix+sp config set <key> <value>`")
+    elif util.recursive_set(portal.local_config, key, value):
+        return evt.reply(f"Successfully set the value of `{key}` to `{value}`.")
+    else:
+        return evt.reply(f"Failed to set value of `{key}`. "
+                         "Does the path contain non-map types?")
+
+
+def config_unset(evt: CommandEvent, portal: po.Portal, key: str) -> Awaitable[Dict]:
+    if not key:
+        return evt.reply(f"**Usage:** `$cmdprefix+sp config unset <key>`")
+    elif util.recursive_del(portal.local_config, key):
+        return evt.reply(f"Successfully deleted `{key}` from config.")
+    else:
+        return evt.reply(f"`{key}` not found in config.")
+
+
+def config_add_del(evt: CommandEvent, portal: po.Portal, key: str, value: str, cmd: str
+                   ) -> Awaitable[Dict]:
+    if not key or value is None:
+        return evt.reply(f"**Usage:** `$cmdprefix+sp config {cmd} <key> <value>`")
+
+    arr = util.recursive_get(portal.local_config, key)
+    if not arr:
+        return evt.reply(f"`{key}` not found in config. "
+                         f"Maybe do `$cmdprefix+sp config set {key} []` first?")
+    elif not isinstance(arr, list):
+        return evt.reply("`{key}` does not seem to be an array.")
+    elif cmd == "add":
+        if value in arr:
+            return evt.reply(f"The array at `{key}` already contains `{value}`.")
+        arr.append(value)
+        return evt.reply(f"Successfully added `{value}` to the array at `{key}`")
+    else:
+        if value not in arr:
+            return evt.reply(f"The array at `{key}` does not contain `{value}`.")
+        arr.remove(value)
+        return evt.reply(f"Successfully removed `{value}` from the array at `{key}`")

mautrix_telegram/commands/portal/create_chat.py

@@ -0,0 +1,61 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Dict
+
+from ... import portal as po
+from ...types import TelegramID
+from .. import command_handler, CommandEvent, SECTION_CREATING_PORTALS
+from .util import user_has_power_level, get_initial_state
+
+
+@command_handler(help_section=SECTION_CREATING_PORTALS,
+                 help_args="[_type_]",
+                 help_text="Create a Telegram chat of the given type for the current Matrix room. "
+                           "The type is either `group`, `supergroup` or `channel` (defaults to "
+                           "`group`).")
+async def create(evt: CommandEvent) -> Dict:
+    type = evt.args[0] if len(evt.args) > 0 else "group"
+    if type not in {"chat", "group", "supergroup", "channel"}:
+        return await evt.reply(
+            "**Usage:** `$cmdprefix+sp create ['group'/'supergroup'/'channel']`")
+
+    if po.Portal.get_by_mxid(evt.room_id):
+        return await evt.reply("This is already a portal room.")
+
+    if not await user_has_power_level(evt.room_id, evt.az.intent, evt.sender, "bridge"):
+        return await evt.reply("You do not have the permissions to bridge this room.")
+
+    title, about, levels = await get_initial_state(evt.az.intent, evt.room_id)
+    if not title:
+        return await evt.reply("Please set a title before creating a Telegram chat.")
+
+    supergroup = type == "supergroup"
+    type = {
+        "supergroup": "channel",
+        "channel": "channel",
+        "chat": "chat",
+        "group": "chat",
+    }[type]
+
+    portal = po.Portal(tgid=TelegramID(0), peer_type=type,
+                       mxid=evt.room_id, title=title, about=about)
+    try:
+        await portal.create_telegram_chat(evt.sender, supergroup=supergroup)
+    except ValueError as e:
+        portal.delete()
+        return await evt.reply(e.args[0])
+    return await evt.reply(f"Telegram chat created. ID: {portal.tgid}")

mautrix_telegram/commands/portal/filter.py

@@ -0,0 +1,95 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Dict, Optional
+
+from ... import portal as po
+from .. import command_handler, CommandEvent, SECTION_ADMIN
+
+
+@command_handler(needs_admin=True,
+                 help_section=SECTION_ADMIN,
+                 help_args="<`whitelist`|`blacklist`>",
+                 help_text="Change whether the bridge will allow or disallow bridging rooms by "
+                           "default.")
+async def filter_mode(evt: CommandEvent) -> Dict:
+    try:
+        mode = evt.args[0]
+        if mode not in ("whitelist", "blacklist"):
+            raise ValueError()
+    except (IndexError, ValueError):
+        return await evt.reply("**Usage:** `$cmdprefix+sp filter-mode <whitelist/blacklist>`")
+
+    evt.config["bridge.filter.mode"] = mode
+    evt.config.save()
+    po.Portal.filter_mode = mode
+    if mode == "whitelist":
+        return await evt.reply("The bridge will now disallow bridging chats by default.\n"
+                               "To allow bridging a specific chat, use"
+                               "`!filter whitelist <chat ID>`.")
+    else:
+        return await evt.reply("The bridge will now allow bridging chats by default.\n"
+                               "To disallow bridging a specific chat, use"
+                               "`!filter blacklist <chat ID>`.")
+
+
+@command_handler(name="filter", needs_admin=True,
+                 help_section=SECTION_ADMIN,
+                 help_args="<`whitelist`|`blacklist`> <_chat ID_>",
+                 help_text="Allow or disallow bridging a specific chat.")
+async def edit_filter(evt: CommandEvent) -> Optional[Dict]:
+    try:
+        action = evt.args[0]
+        if action not in ("whitelist", "blacklist", "add", "remove"):
+            raise ValueError()
+
+        id_str = evt.args[1]
+        if id_str.startswith("-100"):
+            filter_id = int(id_str[4:])
+        elif id_str.startswith("-"):
+            filter_id = int(id_str[1:])
+        else:
+            filter_id = int(id_str)
+    except (IndexError, ValueError):
+        return await evt.reply("**Usage:** `$cmdprefix+sp filter <whitelist/blacklist> <chat ID>`")
+
+    mode = evt.config["bridge.filter.mode"]
+    if mode not in ("blacklist", "whitelist"):
+        return await evt.reply(f"Unknown filter mode \"{mode}\". Please fix the bridge config.")
+
+    filter_id_list = evt.config["bridge.filter.list"]
+
+    if action in ("blacklist", "whitelist"):
+        action = "add" if mode == action else "remove"
+
+    def save() -> None:
+        evt.config["bridge.filter.list"] = filter_id_list
+        evt.config.save()
+        po.Portal.filter_list = filter_id_list
+
+    if action == "add":
+        if filter_id in filter_id_list:
+            return await evt.reply(f"That chat is already {mode}ed.")
+        filter_id_list.append(filter_id)
+        save()
+        return await evt.reply(f"Chat ID added to {mode}.")
+    elif action == "remove":
+        if filter_id not in filter_id_list:
+            return await evt.reply(f"That chat is not {mode}ed.")
+        filter_id_list.remove(filter_id)
+        save()
+        return await evt.reply(f"Chat ID removed from {mode}.")
+    return None

mautrix_telegram/commands/portal/misc.py

@@ -0,0 +1,127 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Dict
+
+from telethon.errors import (ChatAdminRequiredError, UsernameInvalidError,
+                             UsernameNotModifiedError, UsernameOccupiedError)
+
+from ... import portal as po
+from .. import command_handler, CommandEvent, SECTION_PORTAL_MANAGEMENT, SECTION_MISC
+from .util import user_has_power_level
+
+
+@command_handler(needs_admin=False, needs_puppeting=False, needs_auth=False,
+                 help_section=SECTION_MISC,
+                 help_text="Fetch Matrix room state to ensure the bridge has up-to-date info.")
+async def sync_state(evt: CommandEvent) -> Dict:
+    portal = po.Portal.get_by_mxid(evt.room_id)
+    if not portal:
+        return await evt.reply("This is not a portal room.")
+    elif not await user_has_power_level(evt.room_id, evt.az.intent, evt.sender, "bridge"):
+        return await evt.reply(f"You do not have the permissions to synchronize this room.")
+
+    await portal.sync_matrix_members()
+    await evt.reply("Synchronization complete")
+
+
+@command_handler(name="id", needs_admin=False, needs_puppeting=False, needs_auth=False,
+                 help_section=SECTION_MISC,
+                 help_text="Get the ID of the Telegram chat where this room is bridged.")
+async def get_id(evt: CommandEvent) -> Dict:
+    portal = po.Portal.get_by_mxid(evt.room_id)
+    if not portal:
+        return await evt.reply("This is not a portal room.")
+    tgid = portal.tgid
+    if portal.peer_type == "chat":
+        tgid = -tgid
+    elif portal.peer_type == "channel":
+        tgid = f"-100{tgid}"
+    await evt.reply(f"This room is bridged to Telegram chat ID `{tgid}`.")
+
+
+@command_handler(help_section=SECTION_PORTAL_MANAGEMENT,
+                 help_text="Get a Telegram invite link to the current chat.")
+async def invite_link(evt: CommandEvent) -> Dict:
+    portal = po.Portal.get_by_mxid(evt.room_id)
+    if not portal:
+        return await evt.reply("This is not a portal room.")
+
+    if portal.peer_type == "user":
+        return await evt.reply("You can't invite users to private chats.")
+
+    try:
+        link = await portal.get_invite_link(evt.sender)
+        return await evt.reply(f"Invite link to {portal.title}: {link}")
+    except ValueError as e:
+        return await evt.reply(e.args[0])
+    except ChatAdminRequiredError:
+        return await evt.reply("You don't have the permission to create an invite link.")
+
+
+@command_handler(help_section=SECTION_PORTAL_MANAGEMENT,
+                 help_text="Upgrade a normal Telegram group to a supergroup.")
+async def upgrade(evt: CommandEvent) -> Dict:
+    portal = po.Portal.get_by_mxid(evt.room_id)
+    if not portal:
+        return await evt.reply("This is not a portal room.")
+    elif portal.peer_type == "channel":
+        return await evt.reply("This is already a supergroup or a channel.")
+    elif portal.peer_type == "user":
+        return await evt.reply("You can't upgrade private chats.")
+
+    try:
+        await portal.upgrade_telegram_chat(evt.sender)
+        return await evt.reply(f"Group upgraded to supergroup. New ID: -100{portal.tgid}")
+    except ChatAdminRequiredError:
+        return await evt.reply("You don't have the permission to upgrade this group.")
+    except ValueError as e:
+        return await evt.reply(e.args[0])
+
+
+@command_handler(help_section=SECTION_PORTAL_MANAGEMENT,
+                 help_args="<_name_|`-`>",
+                 help_text="Change the username of a supergroup/channel. "
+                           "To disable, use a dash (`-`) as the name.")
+async def group_name(evt: CommandEvent) -> Dict:
+    if len(evt.args) == 0:
+        return await evt.reply("**Usage:** `$cmdprefix+sp group-name <name/->`")
+
+    portal = po.Portal.get_by_mxid(evt.room_id)
+    if not portal:
+        return await evt.reply("This is not a portal room.")
+    elif portal.peer_type != "channel":
+        return await evt.reply("Only channels and supergroups have usernames.")
+
+    try:
+        await portal.set_telegram_username(evt.sender,
+                                           evt.args[0] if evt.args[0] != "-" else "")
+        if portal.username:
+            return await evt.reply(f"Username of channel changed to {portal.username}.")
+        else:
+            return await evt.reply(f"Channel is now private.")
+    except ChatAdminRequiredError:
+        return await evt.reply(
+            "You don't have the permission to set the username of this channel.")
+    except UsernameNotModifiedError:
+        if portal.username:
+            return await evt.reply("That is already the username of this channel.")
+        else:
+            return await evt.reply("This channel is already private")
+    except UsernameOccupiedError:
+        return await evt.reply("That username is already in use.")
+    except UsernameInvalidError:
+        return await evt.reply("Invalid username")

mautrix_telegram/commands/portal/unbridge.py

@@ -0,0 +1,97 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Dict, Callable, Optional
+
+from ...types import MatrixRoomID
+from ... import portal as po
+from .. import command_handler, CommandEvent, SECTION_PORTAL_MANAGEMENT
+from .util import user_has_power_level
+
+
+async def _get_portal_and_check_permission(evt: CommandEvent, permission: str,
+                                           action: Optional[str] = None
+                                           ) -> Optional[po.Portal]:
+    room_id = MatrixRoomID(evt.args[0]) if len(evt.args) > 0 else evt.room_id
+
+    portal = po.Portal.get_by_mxid(room_id)
+    if not portal:
+        that_this = "This" if room_id == evt.room_id else "That"
+        await evt.reply(f"{that_this} is not a portal room.")
+        return None
+
+    if not await user_has_power_level(portal.mxid, evt.az.intent, evt.sender, permission):
+        action = action or f"{permission.replace('_', ' ')}s"
+        await evt.reply(f"You do not have the permissions to {action} that portal.")
+        return None
+    return portal
+
+
+def _get_portal_murder_function(action: str, room_id: str, function: Callable, command: str,
+                                completed_message: str) -> Dict:
+    async def post_confirm(confirm) -> Optional[Dict]:
+        confirm.sender.command_status = None
+        if len(confirm.args) > 0 and confirm.args[0] == f"confirm-{command}":
+            await function()
+            if confirm.room_id != room_id:
+                return await confirm.reply(completed_message)
+        else:
+            return await confirm.reply(f"{action} cancelled.")
+        return None
+
+    return {
+        "next": post_confirm,
+        "action": action,
+    }
+
+
+@command_handler(needs_auth=False, needs_puppeting=False,
+                 help_section=SECTION_PORTAL_MANAGEMENT,
+                 help_text="Remove all users from the current portal room and forget the portal. "
+                           "Only works for group chats; to delete a private chat portal, simply "
+                           "leave the room.")
+async def delete_portal(evt: CommandEvent) -> Optional[Dict]:
+    portal = await _get_portal_and_check_permission(evt, "unbridge")
+    if not portal:
+        return None
+
+    evt.sender.command_status = _get_portal_murder_function("Portal deletion", portal.mxid,
+                                                            portal.cleanup_and_delete, "delete",
+                                                            "Portal successfully deleted.")
+    return await evt.reply("Please confirm deletion of portal "
+                           f"[{portal.alias or portal.mxid}](https://matrix.to/#/{portal.mxid}) "
+                           f"to Telegram chat \"{portal.title}\" "
+                           "by typing `$cmdprefix+sp confirm-delete`"
+                           "\n\n"
+                           "**WARNING:** If the bridge bot has the power level to do so, **this "
+                           "will kick ALL users** in the room. If you just want to remove the "
+                           "bridge, use `$cmdprefix+sp unbridge` instead.")
+
+
+@command_handler(needs_auth=False, needs_puppeting=False,
+                 help_section=SECTION_PORTAL_MANAGEMENT,
+                 help_text="Remove puppets from the current portal room and forget the portal.")
+async def unbridge(evt: CommandEvent) -> Optional[Dict]:
+    portal = await _get_portal_and_check_permission(evt, "unbridge")
+    if not portal:
+        return None
+
+    evt.sender.command_status = _get_portal_murder_function("Room unbridging", portal.mxid,
+                                                            portal.unbridge, "unbridge",
+                                                            "Room successfully unbridged.")
+    return await evt.reply(f"Please confirm unbridging chat \"{portal.title}\" from room "
+                           f"[{portal.alias or portal.mxid}](https://matrix.to/#/{portal.mxid}) "
+                           "by typing `$cmdprefix+sp confirm-unbridge`")

mautrix_telegram/commands/portal/util.py

@@ -0,0 +1,56 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Dict, Tuple
+
+from mautrix_appservice import MatrixRequestError, IntentAPI
+
+from ... import user as u
+
+
+async def get_initial_state(intent: IntentAPI, room_id: str) -> Tuple[str, str, Dict]:
+    state = await intent.get_room_state(room_id)
+    title = None
+    about = None
+    levels = None
+    for event in state:
+        try:
+            if event["type"] == "m.room.name":
+                title = event["content"]["name"]
+            elif event["type"] == "m.room.topic":
+                about = event["content"]["topic"]
+            elif event["type"] == "m.room.power_levels":
+                levels = event["content"]
+            elif event["type"] == "m.room.canonical_alias":
+                title = title or event["content"]["alias"]
+        except KeyError:
+            # Some state event probably has empty content
+            pass
+    return title, about, levels
+
+
+async def user_has_power_level(room: str, intent, sender: u.User, event: str, default: int = 50
+                               ) -> bool:
+    if sender.is_admin:
+        return True
+    # Make sure the state store contains the power levels.
+    try:
+        await intent.get_power_levels(room)
+    except MatrixRequestError:
+        return False
+    return intent.state_store.has_power_level(room, sender.mxid,
+                                              event=f"net.maunium.telegram.{event}",
+                                              default=default)

mautrix_telegram/commands/telegram.py

@@ -1,142 +0,0 @@
-# -*- coding: future_fstrings -*-
-# mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from telethon.errors import *
-from telethon.tl.types import User as TLUser
-from telethon.tl.functions.messages import ImportChatInviteRequest, CheckChatInviteRequest
-from telethon.tl.functions.channels import JoinChannelRequest
-
-from .. import puppet as pu, portal as po
-from . import command_handler
-
-
-@command_handler()
-async def search(evt):
-    if len(evt.args) == 0:
-        return await evt.reply("**Usage:** `$cmdprefix+sp search [-r|--remote] <query>`")
-
-    force_remote = False
-    if evt.args[0] in {"-r", "--remote"}:
-        force_remote = True
-        evt.args.pop(0)
-
-    query = " ".join(evt.args)
-    if force_remote and len(query) < 5:
-        return await evt.reply("Minimum length of query for remote search is 5 characters.")
-
-    results, remote = await evt.sender.search(query, force_remote)
-
-    if not results:
-        if len(query) < 5 and remote:
-            return await evt.reply("No local results. "
-                                   "Minimum length of remote query is 5 characters.")
-        return await evt.reply("No results 3:")
-
-    reply = []
-    if remote:
-        reply += ["**Results from Telegram server:**", ""]
-    else:
-        reply += ["**Results in contacts:**", ""]
-    reply += [(f"* [{puppet.displayname}](https://matrix.to/#/{puppet.mxid}): "
-               f"{puppet.id} ({similarity}% match)")
-              for puppet, similarity in results]
-
-    # TODO somehow show remote channel results when joining by alias is possible?
-
-    return await evt.reply("\n".join(reply))
-
-
-@command_handler(name="pm")
-async def private_message(evt):
-    if len(evt.args) == 0:
-        return await evt.reply("**Usage:** `$cmdprefix+sp pm <user identifier>`")
-
-    try:
-        user = await evt.sender.client.get_entity(evt.args[0])
-    except ValueError:
-        return await evt.reply("Invalid user identifier or user not found.")
-
-    if not user:
-        return await evt.reply("User not found.")
-    elif not isinstance(user, TLUser):
-        return await evt.reply("That doesn't seem to be a user.")
-    portal = po.Portal.get_by_entity(user, evt.sender.tgid)
-    await portal.create_matrix_room(evt.sender, user, [evt.sender.mxid])
-    return await evt.reply("Created private chat room with "
-                           f"{pu.Puppet.get_displayname(user, False)}")
-
-
-async def _join(evt, arg):
-    if arg.startswith("joinchat/"):
-        invite_hash = arg[len("joinchat/"):]
-        try:
-            await evt.sender.client(CheckChatInviteRequest(invite_hash))
-        except InviteHashInvalidError:
-            return None, await evt.reply("Invalid invite link.")
-        except InviteHashExpiredError:
-            return None, await evt.reply("Invite link expired.")
-        try:
-            return (await evt.sender.client(ImportChatInviteRequest(invite_hash))), None
-        except UserAlreadyParticipantError:
-            return None, await evt.reply("You are already in that chat.")
-    else:
-        channel = await evt.sender.client.get_entity(arg)
-        if not channel:
-            return None, await evt.reply("Channel/supergroup not found.")
-        return await evt.sender.client(JoinChannelRequest(channel)), None
-
-
-@command_handler()
-async def join(evt):
-    if len(evt.args) == 0:
-        return await evt.reply("**Usage:** `$cmdprefix+sp join <invite link>`")
-
-    regex = re.compile(r"(?:https?://)?t(?:elegram)?\.(?:dog|me)(?:joinchat/)?/(.+)")
-    arg = regex.match(evt.args[0])
-    if not arg:
-        return await evt.reply("That doesn't look like a Telegram invite link.")
-
-    updates, _ = await _join(evt, arg.group(1))
-    if not updates:
-        return
-
-    for chat in updates.chats:
-        portal = po.Portal.get_by_entity(chat)
-        if portal.mxid:
-            await portal.invite_to_matrix([evt.sender.mxid])
-            return await evt.reply(f"Invited you to portal of {portal.title}")
-        else:
-            await evt.reply(f"Creating room for {chat.title}... This might take a while.")
-            await portal.create_matrix_room(evt.sender, chat, [evt.sender.mxid])
-            return await evt.reply(f"Created room for {portal.title}")
-
-
-@command_handler()
-async def sync(evt):
-    if len(evt.args) > 0:
-        sync_only = evt.args[0]
-        if sync_only not in ("chats", "contacts", "me"):
-            return await evt.reply("**Usage:** `$cmdprefix+sp sync [chats|contacts|me]`")
-    else:
-        sync_only = None
-
-    if not sync_only or sync_only == "chats":
-        await evt.sender.sync_dialogs(synchronous_create=True)
-    if not sync_only or sync_only == "contacts":
-        await evt.sender.sync_contacts()
-    if not sync_only or sync_only == "me":
-        await evt.sender.update_info()
-    return await evt.reply("Synchronization complete.")

mautrix_telegram/commands/telegram/__init__.py

@@ -0,0 +1 @@
+from . import account, auth, misc

mautrix_telegram/commands/telegram/account.py

@@ -0,0 +1,124 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Dict, Optional
+
+from telethon.errors import (UsernameInvalidError, UsernameNotModifiedError, UsernameOccupiedError,
+                             HashInvalidError, AuthKeyError, FirstNameInvalidError)
+from telethon.tl.types import Authorization
+from telethon.tl.functions.account import (UpdateUsernameRequest, GetAuthorizationsRequest,
+                                           ResetAuthorizationRequest, UpdateProfileRequest)
+
+from .. import command_handler, CommandEvent, SECTION_AUTH
+
+
+@command_handler(needs_auth=True,
+                 help_section=SECTION_AUTH,
+                 help_args="<_new username_>",
+                 help_text="Change your Telegram username.")
+async def username(evt: CommandEvent) -> Optional[Dict]:
+    if len(evt.args) == 0:
+        return await evt.reply("**Usage:** `$cmdprefix+sp username <new username>`")
+    if evt.sender.is_bot:
+        return await evt.reply("Bots can't set their own username.")
+    new_name = evt.args[0]
+    if new_name == "-":
+        new_name = ""
+    try:
+        await evt.sender.client(UpdateUsernameRequest(username=new_name))
+    except UsernameInvalidError:
+        return await evt.reply("Invalid username. Usernames must be between 5 and 30 alphanumeric "
+                               "characters.")
+    except UsernameNotModifiedError:
+        return await evt.reply("That is your current username.")
+    except UsernameOccupiedError:
+        return await evt.reply("That username is already in use.")
+    await evt.sender.update_info()
+    if not evt.sender.username:
+        await evt.reply("Username removed")
+    else:
+        await evt.reply(f"Username changed to {evt.sender.username}")
+
+
+@command_handler(needs_auth=True, help_section=SECTION_AUTH, help_args="<_new displayname_>",
+                 help_text="Change your Telegram displayname.")
+async def displayname(evt: CommandEvent) -> Optional[Dict]:
+    if len(evt.args) == 0:
+        return await evt.reply("**Usage:** `$cmdprefix+sp displayname <new displayname>`")
+    if evt.sender.is_bot:
+        return await evt.reply("Bots can't set their own displayname.")
+
+    first_name, last_name = ((evt.args[0], "")
+                             if len(evt.args) == 1
+                             else (" ".join(evt.args[:-1]), evt.args[-1]))
+    try:
+        await evt.sender.client(UpdateProfileRequest(first_name=first_name, last_name=last_name))
+    except FirstNameInvalidError:
+        return await evt.reply("Invalid first name")
+    await evt.sender.update_info()
+    await evt.reply("Displayname updated")
+
+
+def _format_session(sess: Authorization) -> str:
+    return (f"**{sess.app_name} {sess.app_version}**  \n"
+            f"  **Platform:** {sess.device_model} {sess.platform} {sess.system_version}  \n"
+            f"  **Active:** {sess.date_active} (created {sess.date_created})  \n"
+            f"  **From:** {sess.ip} - {sess.region}, {sess.country}")
+
+
+@command_handler(needs_auth=True,
+                 help_section=SECTION_AUTH,
+                 help_args="<`list`|`terminate`> [_hash_]",
+                 help_text="View or delete other Telegram sessions.")
+async def session(evt: CommandEvent) -> Optional[Dict]:
+    if len(evt.args) == 0:
+        return await evt.reply("**Usage:** `$cmdprefix+sp session <list|terminate> [hash]`")
+    elif evt.sender.is_bot:
+        return await evt.reply("Bots can't manage their sessions")
+    cmd = evt.args[0].lower()
+    if cmd == "list":
+        res = await evt.sender.client(GetAuthorizationsRequest())
+        session_list = res.authorizations
+        current = [s for s in session_list if s.current][0]
+        current_text = _format_session(current)
+        other_text = "\n".join(f"* {_format_session(sess)}  \n"
+                               f"  **Hash:** {sess.hash}"
+                               for sess in session_list if not sess.current)
+        return await evt.reply(f"### Current session\n"
+                               f"{current_text}\n"
+                               f"\n"
+                               f"### Other active sessions\n"
+                               f"{other_text}")
+    elif cmd == "terminate" and len(evt.args) > 1:
+        try:
+            session_hash = int(evt.args[1])
+        except ValueError:
+            return await evt.reply("Hash must be an integer")
+        try:
+            ok = await evt.sender.client(ResetAuthorizationRequest(hash=session_hash))
+        except HashInvalidError:
+            return await evt.reply("Invalid session hash.")
+        except AuthKeyError as e:
+            if e.message == "FRESH_RESET_AUTHORISATION_FORBIDDEN":
+                return await evt.reply("New sessions can't terminate other sessions. "
+                                       "Please wait a while.")
+            raise
+        if ok:
+            return await evt.reply("Session terminated successfully.")
+        else:
+            return await evt.reply("Session not found.")
+    else:
+        return await evt.reply("**Usage:** `$cmdprefix+sp session <list|terminate> [hash]`")

mautrix_telegram/commands/telegram/auth.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -14,46 +14,50 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Any, Dict, Optional
 import asyncio
 
-from telethon.errors import *
+from telethon.errors import (
+    AccessTokenExpiredError, AccessTokenInvalidError, FirstNameInvalidError, FloodWaitError,
+    PasswordHashInvalidError, PhoneCodeExpiredError, PhoneCodeInvalidError,
+    PhoneNumberAppSignupForbiddenError, PhoneNumberBannedError, PhoneNumberFloodError,
+    PhoneNumberOccupiedError, PhoneNumberUnoccupiedError, SessionPasswordNeededError)
 
-from . import command_handler
-from .. import puppet as pu
-from ..util import format_duration
+from ... import puppet as pu, user as u
+from ...commands import command_handler, CommandEvent, SECTION_AUTH
+from ...util import format_duration, ignore_coro
 
 
-@command_handler(needs_auth=False)
-async def ping(evt):
-    if not evt.sender.logged_in:
-        return await evt.reply("You're not logged in.")
-    me = await evt.sender.client.get_me()
+@command_handler(needs_auth=False,
+                 help_section=SECTION_AUTH,
+                 help_text="Check if you're logged into Telegram.")
+async def ping(evt: CommandEvent) -> Optional[Dict]:
+    me = await evt.sender.client.get_me() if await evt.sender.is_logged_in() else None
     if me:
-        return await evt.reply(f"You're logged in as @{me.username}")
+        human_tg_id = f"@{me.username}" if me.username else f"+{me.phone}"
+        return await evt.reply(f"You're logged in as {human_tg_id}")
     else:
         return await evt.reply("You're not logged in.")
 
 
-@command_handler()
-async def ping_bot(evt):
+@command_handler(needs_auth=False, needs_puppeting=False,
+                 help_section=SECTION_AUTH,
+                 help_text="Get the info of the message relay Telegram bot.")
+async def ping_bot(evt: CommandEvent) -> Optional[Dict]:
     if not evt.tgbot:
         return await evt.reply("Telegram message relay bot not configured.")
-    bot_info = await evt.tgbot.client.get_me()
-    mxid = pu.Puppet.get_mxid_from_id(bot_info.id)
-    displayname = bot_info.first_name
+    info, mxid = await evt.tgbot.get_me(use_cache=False)
     return await evt.reply("Telegram message relay bot is active: "
-                           f"[{displayname}](https://matrix.to/#/{mxid}) (ID {bot_info.id})\n\n"
+                           f"[{info.first_name}](https://matrix.to/#/{mxid}) (ID {info.id})\n\n"
                            "To use the bot, simply invite it to a portal room.")
 
 
-@command_handler(needs_auth=False, management_only=True)
-def register(evt):
-    return evt.reply("Not yet implemented.")
-
-
-@command_handler(needs_auth=False, management_only=True)
-async def register(evt):
-    if evt.sender.logged_in:
+@command_handler(needs_auth=False, management_only=True,
+                 help_section=SECTION_AUTH,
+                 help_args="<_phone_> <_full name_>",
+                 help_text="Register to Telegram")
+async def register(evt: CommandEvent) -> Optional[Dict]:
+    if await evt.sender.is_logged_in():
         return await evt.reply("You are already logged in.")
     elif len(evt.args) < 1:
         return await evt.reply("**Usage:** `$cmdprefix+sp register <phone> <full name>`")
@@ -64,21 +68,22 @@ async def register(evt):
     else:
         full_name = " ".join(evt.args[1:-1]), evt.args[-1]
 
-    await request_code(evt, phone_number, {
+    await _request_code(evt, phone_number, {
         "next": enter_code_register,
         "action": "Register",
         "full_name": full_name,
     })
+    return None
 
 
-async def enter_code_register(evt):
+async def enter_code_register(evt: CommandEvent) -> Dict:
     if len(evt.args) == 0:
         return await evt.reply("**Usage:** `$cmdprefix+sp <code>`")
     try:
         await evt.sender.ensure_started(even_if_no_session=True)
         first_name, last_name = evt.sender.command_status["full_name"]
         user = await evt.sender.client.sign_up(evt.args[0], first_name, last_name)
-        asyncio.ensure_future(evt.sender.post_login(user), loop=evt.loop)
+        ignore_coro(asyncio.ensure_future(evt.sender.post_login(user), loop=evt.loop))
         evt.sender.command_status = None
         return await evt.reply(f"Successfully registered to Telegram.")
     except PhoneNumberOccupiedError:
@@ -97,36 +102,47 @@ async def enter_code_register(evt):
                                "Check console for more details.")
 
 
-@command_handler(needs_auth=False, management_only=True)
-async def login(evt):
-    if evt.sender.logged_in:
-        return await evt.reply("You are already logged in.")
+@command_handler(needs_auth=False, management_only=True,
+                 help_section=SECTION_AUTH,
+                 help_text="Get instructions on how to log in.")
+async def login(evt: CommandEvent) -> Optional[Dict]:
+    override_sender = False
+    if len(evt.args) > 0 and evt.sender.is_admin:
+        evt.sender = await u.User.get_by_mxid(evt.args[0]).ensure_started()
+        override_sender = True
+    if await evt.sender.is_logged_in():
+        return await evt.reply(f"You are already logged in as {evt.sender.human_tg_id}.")
 
     allow_matrix_login = evt.config.get("bridge.allow_matrix_login", True)
-    if allow_matrix_login:
+    if allow_matrix_login and not override_sender:
         evt.sender.command_status = {
-            "next": enter_phone,
+            "next": enter_phone_or_token,
             "action": "Login",
         }
 
+    nb = "**N.B. Logging in grants the bridge full access to your Telegram account.**"
     if evt.config["appservice.public.enabled"]:
         prefix = evt.config["appservice.public.external"]
-        url = f"{prefix}/login?mxid={evt.sender.mxid}"
-        if evt.config.get("bridge.allow_matrix_login", True):
-            return await evt.reply("\n\n".join((
-                "This bridge instance allows you to log in inside or outside Matrix.",
-                "If you would like to log in within Matrix, please send your phone number here.",
-                f"If you would like to log in outside of Matrix, [click here]({url}).")))
-        return await evt.reply("This bridge instance does not allow logging in inside Matrix.\n\n"
-                               f"Please visit [the login page]({url}) to log in.")
+        url = f"{prefix}/login?token={evt.public_website.make_token(evt.sender.mxid, '/login')}"
+        if override_sender:
+            return await evt.reply(f"[Click here to log in]({url}) as "
+                                   f"[{evt.sender.mxid}](https://matrix.to/#/{evt.sender.mxid}).")
+        elif allow_matrix_login:
+            return await evt.reply(f"[Click here to log in]({url}). Alternatively, send your phone"
+                                   f" number (or bot auth token) here to log in.\n\n{nb}")
+        return await evt.reply(f"[Click here to log in]({url}).\n\n{nb}")
     elif allow_matrix_login:
-        return await evt.reply(
-            "This bridge instance does not allow you to log in outside of Matrix.\n\n"
-            "Please send your phone number here to start the login process.")
+        if override_sender:
+            return await evt.reply(
+                "This bridge instance does not allow you to log in outside of Matrix. "
+                "Logging in as another user inside Matrix is not currently possible.")
+        return await evt.reply("Please send your phone number (or bot auth token) here to start "
+                               f"the login process.\n\n{nb}")
     return await evt.reply("This bridge instance has been configured to not allow logging in.")
 
 
-async def request_code(evt, phone_number, next_status):
+async def _request_code(evt: CommandEvent, phone_number: str, next_status: Dict[str, Any]
+                        ) -> Dict:
     ok = False
     try:
         await evt.sender.ensure_started(even_if_no_session=True)
@@ -158,37 +174,85 @@ async def request_code(evt, phone_number, next_status):
 
 
 @command_handler(needs_auth=False)
-async def enter_phone(evt):
+async def enter_phone_or_token(evt: CommandEvent) -> Optional[Dict]:
     if len(evt.args) == 0:
-        return await evt.reply("**Usage:** `$cmdprefix+sp enter-phone <phone>`")
+        return await evt.reply("**Usage:** `$cmdprefix+sp enter-phone-or-token <phone-or-token>`")
     elif not evt.config.get("bridge.allow_matrix_login", True):
         return await evt.reply("This bridge instance does not allow in-Matrix login. "
                                "Please use `$cmdprefix+sp login` to get login instructions")
 
-    phone_number = evt.args[0]
-    await request_code(evt, phone_number, {
-        "next": enter_code,
-        "action": "Login",
-    })
+    # phone numbers don't contain colons but telegram bot auth tokens do
+    if evt.args[0].find(":") > 0:
+        try:
+            await _sign_in(evt, bot_token=evt.args[0])
+        except Exception:
+            evt.log.exception("Error sending auth token")
+            return await evt.reply("Unhandled exception while sending auth token. "
+                                   "Check console for more details.")
+    else:
+        await _request_code(evt, evt.args[0], {
+            "next": enter_code,
+            "action": "Login",
+        })
+    return None
 
 
 @command_handler(needs_auth=False)
-async def enter_code(evt):
+async def enter_code(evt: CommandEvent) -> Optional[Dict]:
     if len(evt.args) == 0:
         return await evt.reply("**Usage:** `$cmdprefix+sp enter-code <code>`")
     elif not evt.config.get("bridge.allow_matrix_login", True):
         return await evt.reply("This bridge instance does not allow in-Matrix login. "
                                "Please use `$cmdprefix+sp login` to get login instructions")
+    try:
+        await _sign_in(evt, code=evt.args[0])
+    except Exception:
+        evt.log.exception("Error sending phone code")
+        return await evt.reply("Unhandled exception while sending code. "
+                               "Check console for more details.")
+    return None
+
+
+@command_handler(needs_auth=False)
+async def enter_password(evt: CommandEvent) -> Optional[Dict]:
+    if len(evt.args) == 0:
+        return await evt.reply("**Usage:** `$cmdprefix+sp enter-password <password>`")
+    elif not evt.config.get("bridge.allow_matrix_login", True):
+        return await evt.reply("This bridge instance does not allow in-Matrix login. "
+                               "Please use `$cmdprefix+sp login` to get login instructions")
+    try:
+        await _sign_in(evt, password=" ".join(evt.args))
+    except AccessTokenInvalidError:
+        return await evt.reply("That bot token is not valid.")
+    except AccessTokenExpiredError:
+        return await evt.reply("That bot token has expired.")
+    except Exception:
+        evt.log.exception("Error sending password")
+        return await evt.reply("Unhandled exception while sending password. "
+                               "Check console for more details.")
+    return None
+
+
+async def _sign_in(evt: CommandEvent, **sign_in_info) -> Dict:
     try:
         await evt.sender.ensure_started(even_if_no_session=True)
-        user = await evt.sender.client.sign_in(code=evt.args[0])
-        asyncio.ensure_future(evt.sender.post_login(user), loop=evt.loop)
+        user = await evt.sender.client.sign_in(**sign_in_info)
+        existing_user = u.User.get_by_tgid(user.id)
+        if existing_user and existing_user != evt.sender:
+            await existing_user.log_out()
+            await evt.reply(f"[{existing_user.displayname}]"
+                            f"(https://matrix.to/#/{existing_user.mxid})"
+                            " was logged out from the account.")
+        ignore_coro(asyncio.ensure_future(evt.sender.post_login(user), loop=evt.loop))
         evt.sender.command_status = None
-        return await evt.reply(f"Successfully logged in as @{user.username}")
+        name = f"@{user.username}" if user.username else f"+{user.phone}"
+        return await evt.reply(f"Successfully logged in as {name}")
     except PhoneCodeExpiredError:
         return await evt.reply("Phone code expired. Try again with `$cmdprefix+sp login`.")
     except PhoneCodeInvalidError:
         return await evt.reply("Invalid phone code.")
+    except PasswordHashInvalidError:
+        return await evt.reply("Incorrect password.")
     except SessionPasswordNeededError:
         evt.sender.command_status = {
             "next": enter_password,
@@ -196,37 +260,12 @@ async def enter_code(evt):
         }
         return await evt.reply("Your account has two-factor authentication. "
                                "Please send your password here.")
-    except Exception:
-        evt.log.exception("Error sending phone code")
-        return await evt.reply("Unhandled exception while sending code. "
-                               "Check console for more details.")
 
 
-@command_handler(needs_auth=False)
-async def enter_password(evt):
-    if len(evt.args) == 0:
-        return await evt.reply("**Usage:** `$cmdprefix+sp enter-password <password>`")
-    elif not evt.config.get("bridge.allow_matrix_login", True):
-        return await evt.reply("This bridge instance does not allow in-Matrix login. "
-                               "Please use `$cmdprefix+sp login` to get login instructions")
-    try:
-        await evt.sender.ensure_started(even_if_no_session=True)
-        user = await evt.sender.client.sign_in(password=" ".join(evt.args))
-        asyncio.ensure_future(evt.sender.post_login(user), loop=evt.loop)
-        evt.sender.command_status = None
-        return await evt.reply(f"Successfully logged in as @{user.username}")
-    except PasswordHashInvalidError:
-        return await evt.reply("Incorrect password.")
-    except Exception:
-        evt.log.exception("Error sending password")
-        return await evt.reply("Unhandled exception while sending password. "
-                               "Check console for more details.")
-
-
-@command_handler(needs_auth=False)
-async def logout(evt):
-    if not evt.sender.logged_in:
-        return await evt.reply("You're not logged in.")
+@command_handler(needs_auth=True,
+                 help_section=SECTION_AUTH,
+                 help_text="Log out from Telegram.")
+async def logout(evt: CommandEvent) -> Optional[Dict]:
     if await evt.sender.log_out():
         return await evt.reply("Logged out successfully.")
     return await evt.reply("Failed to log out.")

mautrix_telegram/commands/telegram/misc.py

@@ -0,0 +1,283 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Dict, List, Optional, Tuple
+import codecs
+import base64
+import re
+
+from telethon.errors import (InviteHashInvalidError, InviteHashExpiredError, OptionsTooMuchError,
+                             UserAlreadyParticipantError)
+from telethon.tl.patched import Message
+from telethon.tl.types import (User as TLUser, TypeUpdates, MessageMediaGame, MessageMediaPoll,
+                               TypePeer)
+from telethon.tl.types.messages import BotCallbackAnswer
+from telethon.tl.functions.messages import (ImportChatInviteRequest, CheckChatInviteRequest,
+                                            GetBotCallbackAnswerRequest, SendVoteRequest)
+from telethon.tl.functions.channels import JoinChannelRequest
+
+from ... import puppet as pu, portal as po
+from ...abstract_user import AbstractUser
+from ...db import Message as DBMessage
+from ...types import TelegramID
+from ...commands import command_handler, CommandEvent, SECTION_MISC, SECTION_CREATING_PORTALS
+
+
+@command_handler(help_section=SECTION_MISC,
+                 help_args="[_-r|--remote_] <_query_>",
+                 help_text="Search your contacts or the Telegram servers for users.")
+async def search(evt: CommandEvent) -> Optional[Dict]:
+    if len(evt.args) == 0:
+        return await evt.reply("**Usage:** `$cmdprefix+sp search [-r|--remote] <query>`")
+
+    force_remote = False
+    if evt.args[0] in {"-r", "--remote"}:
+        force_remote = True
+        evt.args.pop(0)
+
+    query = " ".join(evt.args)
+    if force_remote and len(query) < 5:
+        return await evt.reply("Minimum length of query for remote search is 5 characters.")
+
+    results, remote = await evt.sender.search(query, force_remote)
+
+    if not results:
+        if len(query) < 5 and remote:
+            return await evt.reply("No local results. "
+                                   "Minimum length of remote query is 5 characters.")
+        return await evt.reply("No results 3:")
+
+    reply = []  # type: List[str]
+    if remote:
+        reply += ["**Results from Telegram server:**", ""]
+    else:
+        reply += ["**Results in contacts:**", ""]
+    reply += [(f"* [{puppet.displayname}](https://matrix.to/#/{puppet.mxid}): "
+               f"{puppet.id} ({similarity}% match)")
+              for puppet, similarity in results]
+
+    # TODO somehow show remote channel results when joining by alias is possible?
+
+    return await evt.reply("\n".join(reply))
+
+
+@command_handler(help_section=SECTION_CREATING_PORTALS,
+                 help_args="<_identifier_>",
+                 help_text="Open a private chat with the given Telegram user. The identifier is "
+                           "either the internal user ID, the username or the phone number. "
+                           "**N.B.** The phone numbers you start chats with must already be in "
+                           "your contacts.")
+async def pm(evt: CommandEvent) -> Optional[Dict]:
+    if len(evt.args) == 0:
+        return await evt.reply("**Usage:** `$cmdprefix+sp pm <user identifier>`")
+
+    try:
+        user = await evt.sender.client.get_entity(evt.args[0])
+    except ValueError:
+        return await evt.reply("Invalid user identifier or user not found.")
+
+    if not user:
+        return await evt.reply("User not found.")
+    elif not isinstance(user, TLUser):
+        return await evt.reply("That doesn't seem to be a user.")
+    portal = po.Portal.get_by_entity(user, evt.sender.tgid)
+    await portal.create_matrix_room(evt.sender, user, [evt.sender.mxid])
+    return await evt.reply("Created private chat room with "
+                           f"{pu.Puppet.get_displayname(user, False)}")
+
+
+async def _join(evt: CommandEvent, arg: str) -> Tuple[Optional[TypeUpdates], Optional[Dict]]:
+    if arg.startswith("joinchat/"):
+        invite_hash = arg[len("joinchat/"):]
+        try:
+            await evt.sender.client(CheckChatInviteRequest(invite_hash))
+        except InviteHashInvalidError:
+            return None, await evt.reply("Invalid invite link.")
+        except InviteHashExpiredError:
+            return None, await evt.reply("Invite link expired.")
+        try:
+            return (await evt.sender.client(ImportChatInviteRequest(invite_hash))), None
+        except UserAlreadyParticipantError:
+            return None, await evt.reply("You are already in that chat.")
+    else:
+        channel = await evt.sender.client.get_entity(arg)
+        if not channel:
+            return None, await evt.reply("Channel/supergroup not found.")
+        return await evt.sender.client(JoinChannelRequest(channel)), None
+
+
+@command_handler(help_section=SECTION_CREATING_PORTALS,
+                 help_args="<_link_>",
+                 help_text="Join a chat with an invite link.")
+async def join(evt: CommandEvent) -> Optional[Dict]:
+    if len(evt.args) == 0:
+        return await evt.reply("**Usage:** `$cmdprefix+sp join <invite link>`")
+
+    regex = re.compile(r"(?:https?://)?t(?:elegram)?\.(?:dog|me)(?:joinchat/)?/(.+)")
+    arg = regex.match(evt.args[0])
+    if not arg:
+        return await evt.reply("That doesn't look like a Telegram invite link.")
+
+    updates, _ = await _join(evt, arg.group(1))
+    if not updates:
+        return None
+
+    for chat in updates.chats:
+        portal = po.Portal.get_by_entity(chat)
+        if portal.mxid:
+            await portal.invite_to_matrix([evt.sender.mxid])
+            return await evt.reply(f"Invited you to portal of {portal.title}")
+        else:
+            await evt.reply(f"Creating room for {chat.title}... This might take a while.")
+            await portal.create_matrix_room(evt.sender, chat, [evt.sender.mxid])
+            return await evt.reply(f"Created room for {portal.title}")
+    return None
+
+
+@command_handler(help_section=SECTION_MISC,
+                 help_args="[`chats`|`contacts`|`me`]",
+                 help_text="Synchronize your chat portals, contacts and/or own info.")
+async def sync(evt: CommandEvent) -> Optional[Dict]:
+    if len(evt.args) > 0:
+        sync_only = evt.args[0]
+        if sync_only not in ("chats", "contacts", "me"):
+            return await evt.reply("**Usage:** `$cmdprefix+sp sync [chats|contacts|me]`")
+    else:
+        sync_only = None
+
+    if not sync_only or sync_only == "chats":
+        await evt.sender.sync_dialogs(synchronous_create=True)
+    if not sync_only or sync_only == "contacts":
+        await evt.sender.sync_contacts()
+    if not sync_only or sync_only == "me":
+        await evt.sender.update_info()
+    return await evt.reply("Synchronization complete.")
+
+
+PEER_TYPE_CHAT = b"g"
+
+
+class MessageIDError(ValueError):
+    def __init__(self, message: str) -> None:
+        super().__init__(message)
+        self.message = message
+
+
+async def _parse_encoded_msgid(user: AbstractUser, enc_id: str, type_name: str
+                               ) -> Tuple[TypePeer, Message]:
+    try:
+        enc_id += (4 - len(enc_id) % 4) * "="
+        enc_id = base64.b64decode(enc_id)
+        peer_type, enc_id = bytes([enc_id[0]]), enc_id[1:]
+        tgid = TelegramID(int(codecs.encode(enc_id[0:5], "hex_codec"), 16))
+        msg_id = TelegramID(int(codecs.encode(enc_id[5:10], "hex_codec"), 16))
+        space = None
+        if peer_type == PEER_TYPE_CHAT:
+            space = TelegramID(int(codecs.encode(enc_id[10:15], "hex_codec"), 16))
+    except ValueError as e:
+        raise MessageIDError(f"Invalid {type_name} ID (format)") from e
+
+    if peer_type == PEER_TYPE_CHAT:
+        orig_msg = DBMessage.get_one_by_tgid(msg_id, space)
+        if not orig_msg:
+            raise MessageIDError(f"Invalid {type_name} ID (original message not found in db)")
+        new_msg = DBMessage.get_by_mxid(orig_msg.mxid, orig_msg.mx_room, user.tgid)
+        if not new_msg:
+            raise MessageIDError(f"Invalid {type_name} ID (your copy of message not found in db)")
+        msg_id = new_msg.tgid
+    try:
+        peer = await user.client.get_input_entity(tgid)
+    except ValueError as e:
+        raise MessageIDError(f"Invalid {type_name} ID (chat not found)") from e
+
+    msg = await user.client.get_messages(entity=peer, ids=msg_id)
+    if not msg:
+        raise MessageIDError(f"Invalid {type_name} ID (message not found)")
+    return peer, msg
+
+
+@command_handler(help_section=SECTION_MISC,
+                 help_args="<_play ID_>",
+                 help_text="Play a Telegram game.")
+async def play(evt: CommandEvent) -> Optional[Dict]:
+    if len(evt.args) < 1:
+        return await evt.reply("**Usage:** `$cmdprefix+sp play <play ID>`")
+    elif not await evt.sender.is_logged_in():
+        return await evt.reply("You must be logged in with a real account to play games.")
+    elif evt.sender.is_bot:
+        return await evt.reply("Bots can't play games :(")
+
+    try:
+        peer, msg = await _parse_encoded_msgid(evt.sender, evt.args[0], type_name="play")
+    except MessageIDError as e:
+        return await evt.reply(e.message)
+
+    if not isinstance(msg.media, MessageMediaGame):
+        return await evt.reply("Invalid play ID (message doesn't look like a game)")
+
+    game = await evt.sender.client(GetBotCallbackAnswerRequest(peer=peer, msg_id=msg.id, game=True))
+    if not isinstance(game, BotCallbackAnswer):
+        return await evt.reply("Game request response invalid")
+
+    await evt.reply(f"Click [here]({game.url}) to play {msg.media.game.title}:\n\n"
+                    f"{msg.media.game.description}")
+
+
+@command_handler(help_section=SECTION_MISC,
+                 help_args="<_poll ID_> <_choice number_>",
+                 help_text="Vote in a Telegram poll.")
+async def vote(evt: CommandEvent) -> Optional[Dict]:
+    if len(evt.args) < 1:
+        return await evt.reply("**Usage:** `$cmdprefix+sp vote <poll ID> <choice number>`")
+    elif not await evt.sender.is_logged_in():
+        return await evt.reply("You must be logged in with a real account to vote in polls.")
+    elif evt.sender.is_bot:
+        return await evt.reply("Bots can't vote in polls :(")
+
+    try:
+        peer, msg = await _parse_encoded_msgid(evt.sender, evt.args[0], type_name="poll")
+    except MessageIDError as e:
+        return await evt.reply(e.message)
+
+    if not isinstance(msg.media, MessageMediaPoll):
+        return await evt.reply("Invalid poll ID (message doesn't look like a poll)")
+
+    options = []
+    for option in evt.args[1:]:
+        try:
+            if len(option) > 10:
+                raise ValueError("option index too long")
+            option_index = int(option) - 1
+        except ValueError:
+            option_index = None
+        if option_index is None:
+            return await evt.reply(f"Invalid option number \"{option}\"",
+                                   render_markdown=False, allow_html=False)
+        elif option_index < 0:
+            return await evt.reply(f"Invalid option number {option}. "
+                                   f"Option numbers must be positive.")
+        elif option_index >= len(msg.media.poll.answers):
+            return await evt.reply(f"Invalid option number {option}. "
+                                   f"The poll only has {len(msg.media.poll.answers)} options.")
+        options.append(msg.media.poll.answers[option_index].option)
+    options = [msg.media.poll.answers[int(option) - 1].option
+               for option in evt.args[1:]]
+    try:
+        resp = await evt.sender.client(SendVoteRequest(peer=peer, msg_id=msg.id, options=options))
+    except OptionsTooMuchError:
+        return await evt.reply("You passed too many options.")
+    # TODO use response
+    return await evt.mark_read()

mautrix_telegram/config.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -14,71 +14,81 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Any, Dict, Optional, Tuple
 from ruamel.yaml import YAML
 from ruamel.yaml.comments import CommentedMap
 import random
 import string
 
-yaml = YAML()
+yaml = YAML()  # type: YAML
 yaml.indent(4)
 
 
 class DictWithRecursion:
-    def __init__(self, data=None):
-        self._data = data or CommentedMap()
+    def __init__(self, data: Optional[CommentedMap] = None) -> None:
+        self._data = data or CommentedMap()  # type: CommentedMap
 
-    def _recursive_get(self, data, key, default_value):
-        if '.' in key:
-            key, next_key = key.split('.', 1)
+    @staticmethod
+    def _parse_key(key: str) -> Tuple[str, Optional[str]]:
+        if '.' not in key:
+            return key, None
+        key, next_key = key.split('.', 1)
+        if len(key) > 0 and key[0] == "[":
+            end_index = next_key.index("]")
+            key = key[1:] + "." + next_key[:end_index]
+            next_key = next_key[end_index + 2:] if len(next_key) > end_index + 1 else None
+        return key, next_key
+
+    def _recursive_get(self, data: CommentedMap, key: str, default_value: Any) -> Any:
+        key, next_key = self._parse_key(key)
+        if next_key is not None:
             next_data = data.get(key, CommentedMap())
             return self._recursive_get(next_data, next_key, default_value)
         return data.get(key, default_value)
 
-    def get(self, key, default_value, allow_recursion=True):
+    def get(self, key: str, default_value: Any, allow_recursion: bool = True) -> Any:
         if allow_recursion and '.' in key:
             return self._recursive_get(self._data, key, default_value)
         return self._data.get(key, default_value)
 
-    def __getitem__(self, key):
+    def __getitem__(self, key: str) -> Any:
         return self.get(key, None)
 
-    def __contains__(self, key):
+    def __contains__(self, key: str) -> bool:
         return self[key] is not None
 
-    def _recursive_set(self, data, key, value):
-        if '.' in key:
-            key, next_key = key.split('.', 1)
+    def _recursive_set(self, data: CommentedMap, key: str, value: Any) -> None:
+        key, next_key = self._parse_key(key)
+        if next_key is not None:
             if key not in data:
                 data[key] = CommentedMap()
             next_data = data.get(key, CommentedMap())
-            self._recursive_set(next_data, next_key, value)
-            return
+            return self._recursive_set(next_data, next_key, value)
         data[key] = value
 
-    def set(self, key, value, allow_recursion=True):
+    def set(self, key: str, value: Any, allow_recursion: bool = True) -> None:
         if allow_recursion and '.' in key:
             self._recursive_set(self._data, key, value)
             return
         self._data[key] = value
 
-    def __setitem__(self, key, value):
+    def __setitem__(self, key: str, value: Any) -> None:
         self.set(key, value)
 
-    def _recursive_del(self, data, key):
-        if '.' in key:
-            key, next_key = key.split('.', 1)
+    def _recursive_del(self, data: CommentedMap, key: str) -> None:
+        key, next_key = self._parse_key(key)
+        if next_key is not None:
             if key not in data:
                 return
             next_data = data[key]
-            self._recursive_del(next_data, next_key)
-            return
+            return self._recursive_del(next_data, next_key)
         try:
             del data[key]
             del data.ca.items[key]
         except KeyError:
             pass
 
-    def delete(self, key, allow_recursion=True):
+    def delete(self, key: str, allow_recursion: bool = True) -> None:
         if allow_recursion and '.' in key:
             self._recursive_del(self._data, key)
             return
@@ -88,23 +98,31 @@ class DictWithRecursion:
         except KeyError:
             pass
 
-    def __delitem__(self, key):
+    def __delitem__(self, key: str) -> None:
         self.delete(key)
 
 
 class Config(DictWithRecursion):
-    def __init__(self, path, registration_path, base_path):
+    def __init__(self, path: str, registration_path: str, base_path: str,
+                 overrides: Dict[str, Any] = None) -> None:
         super().__init__()
-        self.path = path
-        self.registration_path = registration_path
-        self.base_path = base_path
-        self._registration = None
+        self.path = path  # type: str
+        self.registration_path = registration_path  # type: str
+        self.base_path = base_path  # type: str
+        self._registration = None  # type: Optional[Dict]
+        self._overrides = overrides or {}  # type: Dict[str, Any]
 
-    def load(self):
+    def __getitem__(self, key: str) -> Any:
+        try:
+            return self._overrides[f"MAUTRIX_TELEGRAM_{key.replace('.', '_').upper()}"]
+        except KeyError:
+            return super().__getitem__(key)
+
+    def load(self) -> None:
         with open(self.path, 'r') as stream:
             self._data = yaml.load(stream)
 
-    def load_base(self):
+    def load_base(self) -> Optional[DictWithRecursion]:
         try:
             with open(self.base_path, 'r') as stream:
                 return DictWithRecursion(yaml.load(stream))
@@ -112,7 +130,7 @@ class Config(DictWithRecursion):
             pass
         return None
 
-    def save(self):
+    def save(self) -> None:
         with open(self.path, 'w') as stream:
             yaml.dump(self._data, stream)
         if self._registration and self.registration_path:
@@ -120,32 +138,39 @@ class Config(DictWithRecursion):
                 yaml.dump(self._registration, stream)
 
     @staticmethod
-    def _new_token():
+    def _new_token() -> str:
         return "".join(random.choice(string.ascii_lowercase + string.digits) for _ in range(64))
 
-    def update(self):
+    def update(self) -> None:
         base = self.load_base()
         if not base:
             return
 
-        def copy(from_path, to_path=None):
+        def copy(from_path, to_path=None) -> None:
             if from_path in self:
                 base[to_path or from_path] = self[from_path]
 
-        def copy_dict(from_path, to_path=None):
+        def copy_dict(from_path, to_path=None, override_existing_map=True) -> None:
             if from_path in self:
                 to_path = to_path or from_path
-                base[to_path] = CommentedMap()
+                if override_existing_map or to_path not in base:
+                    base[to_path] = CommentedMap()
                 for key, value in self[from_path].items():
                     base[to_path][key] = value
 
         copy("homeserver.address")
-        copy("homeserver.verify_ssl")
         copy("homeserver.domain")
+        copy("homeserver.verify_ssl")
 
-        copy("appservice.protocol")
+        if "appservice.protocol" in self and "appservice.address" not in self:
+            protocol, hostname, port = (self["appservice.protocol"], self["appservice.hostname"],
+                                        self["appservice.port"])
+            base["appservice.address"] = f"{protocol}://{hostname}:{port}"
+        else:
+            copy("appservice.address")
         copy("appservice.hostname")
         copy("appservice.port")
+        copy("appservice.max_body_size")
 
         copy("appservice.database")
 
@@ -153,33 +178,67 @@ class Config(DictWithRecursion):
         copy("appservice.public.prefix")
         copy("appservice.public.external")
 
-        copy("appservice.debug")
+        copy("appservice.provisioning.enabled")
+        copy("appservice.provisioning.prefix")
+        copy("appservice.provisioning.shared_secret")
+        if base["appservice.provisioning.shared_secret"] == "generate":
+            base["appservice.provisioning.shared_secret"] = self._new_token()
 
         copy("appservice.id")
         copy("appservice.bot_username")
         copy("appservice.bot_displayname")
+        copy("appservice.bot_avatar")
+
+        copy("appservice.community_id")
 
         copy("appservice.as_token")
         copy("appservice.hs_token")
 
+        copy("metrics.enabled")
+        copy("metrics.listen_port")
+
         copy("bridge.username_template")
         copy("bridge.alias_template")
         copy("bridge.displayname_template")
 
         copy("bridge.displayname_preference")
 
-        copy("bridge.edits_as_replies")
-        copy("bridge.highlight_edits")
-        copy("bridge.bridge_notices")
-        copy("bridge.bot_messages_as_notices")
         copy("bridge.max_initial_member_sync")
+        copy("bridge.sync_channel_members")
+        copy("bridge.skip_deleted_members")
+        copy("bridge.startup_sync")
+        copy("bridge.sync_dialog_limit")
         copy("bridge.max_telegram_delete")
+        copy("bridge.sync_matrix_state")
         copy("bridge.allow_matrix_login")
-        copy("bridge.inline_images")
         copy("bridge.plaintext_highlights")
         copy("bridge.public_portals")
-        copy("bridge.native_stickers")
         copy("bridge.catch_up")
+        copy("bridge.sync_with_custom_puppets")
+        copy("bridge.telegram_link_preview")
+        copy("bridge.inline_images")
+        copy("bridge.image_as_file_size")
+        copy("bridge.max_document_size")
+
+        copy("bridge.bot_messages_as_notices")
+        if isinstance(self["bridge.bridge_notices"], bool):
+            base["bridge.bridge_notices"] = {
+                "default": self["bridge.bridge_notices"],
+                "exceptions": ["@importantbot:example.com"],
+            }
+        else:
+            copy("bridge.bridge_notices")
+
+        copy("bridge.deduplication.pre_db_check")
+        copy("bridge.deduplication.cache_queue_length")
+
+        if "bridge.message_formats.m_text" in self:
+            del self["bridge.message_formats"]
+        copy_dict("bridge.message_formats", override_existing_map=False)
+
+        copy("bridge.state_event_formats.join")
+        copy("bridge.state_event_formats.leave")
+        copy("bridge.state_event_formats.name_change")
 
         copy("bridge.filter.mode")
         copy("bridge.filter.list")
@@ -205,22 +264,57 @@ class Config(DictWithRecursion):
             copy("bridge.relaybot.authless_portals")
             copy("bridge.relaybot.whitelist_group_admins")
             copy("bridge.relaybot.whitelist")
+            copy("bridge.relaybot.ignore_own_incoming_events")
 
         copy("telegram.api_id")
         copy("telegram.api_hash")
         copy("telegram.bot_token")
 
+        copy("telegram.connection.timeout")
+        copy("telegram.connection.retries")
+        copy("telegram.connection.retry_delay")
+        copy("telegram.connection.flood_sleep_threshold")
+        copy("telegram.connection.request_retries")
+
+        copy("telegram.device_info.device_model")
+        copy("telegram.device_info.system_version")
+        copy("telegram.device_info.app_version")
+        copy("telegram.device_info.lang_code")
+        copy("telegram.device_info.system_lang_code")
+
+        copy("telegram.server.enabled")
+        copy("telegram.server.dc")
+        copy("telegram.server.ip")
+        copy("telegram.server.port")
+
+        copy("telegram.proxy.type")
+        copy("telegram.proxy.address")
+        copy("telegram.proxy.port")
+        copy("telegram.proxy.rdns")
+        copy("telegram.proxy.username")
+        copy("telegram.proxy.password")
+
+        if "appservice.debug" in self and "logging" not in self:
+            level = "DEBUG" if self["appservice.debug"] else "INFO"
+            base["logging.root.level"] = level
+            base["logging.loggers.mau.level"] = level
+            base["logging.loggers.telethon.level"] = level
+        else:
+            copy("logging")
+
         self._data = base._data
         self.save()
 
-    def _get_permissions(self, key):
+    def _get_permissions(self, key: str) -> Tuple[bool, bool, bool, bool, bool, bool]:
         level = self["bridge.permissions"].get(key, "")
         admin = level == "admin"
-        whitelisted = level == "full" or admin
-        relaybot = level == "relaybot" or whitelisted
-        return relaybot, whitelisted, admin
+        matrix_puppeting = level == "full" or admin
+        puppeting = level == "puppeting" or matrix_puppeting
+        user = level == "user" or puppeting
+        relaybot = level == "relaybot" or user
+        return relaybot, user, puppeting, matrix_puppeting, admin, level
 
-    def get_permissions(self, mxid):
+    def get_permissions(self, mxid: str) -> Tuple[bool, bool, bool, bool, bool, bool]:
         permissions = self["bridge.permissions"] or {}
         if mxid in permissions:
             return self._get_permissions(mxid)
@@ -231,7 +325,7 @@ class Config(DictWithRecursion):
 
         return self._get_permissions("*")
 
-    def generate_registration(self):
+    def generate_registration(self) -> None:
         homeserver = self["homeserver.domain"]
 
         username_format = self.get("bridge.username_template", "telegram_{userid}") \
@@ -242,10 +336,8 @@ class Config(DictWithRecursion):
         self.set("appservice.as_token", self._new_token())
         self.set("appservice.hs_token", self._new_token())
 
-        url = (f"{self['appservice.protocol']}://"
-               f"{self['appservice.hostname']}:{self['appservice.port']}")
         self._registration = {
-            "id": self.get("appservice.id", "telegram"),
+            "id": self["appservice.id"] or "telegram",
             "as_token": self["appservice.as_token"],
             "hs_token": self["appservice.hs_token"],
             "namespaces": {
@@ -258,7 +350,10 @@ class Config(DictWithRecursion):
                     "regex": f"#{alias_format}:{homeserver}"
                 }]
             },
-            "url": url,
+            "url": self["appservice.address"],
             "sender_localpart": self["appservice.bot_username"],
             "rate_limited": False
         }
+        if self["appservice.community_id"]:
+            self._registration["namespaces"]["users"][0]["group_id"] \
+                = self["appservice.community_id"]

mautrix_telegram/context.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -14,21 +14,32 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Optional, Tuple, TYPE_CHECKING
+
+if TYPE_CHECKING:
+    import asyncio
+
+    from alchemysession import AlchemySessionContainer
+    from mautrix_appservice import AppService
+
+    from .web import PublicBridgeWebsite, ProvisioningAPI
+    from .config import Config
+    from .bot import Bot
+    from .matrix import MatrixHandler
 
 
 class Context:
-    def __init__(self, az, db, config, loop, bot, mx, telethon_session_container):
-        self.az = az
-        self.db = db
-        self.config = config
-        self.loop = loop
-        self.bot = bot
-        self.mx = mx
-        self.telethon_session_container = telethon_session_container
+    def __init__(self, az: 'AppService', config: 'Config', loop: 'asyncio.AbstractEventLoop',
+                 session_container: 'AlchemySessionContainer', bot: Optional['Bot']) -> None:
+        self.az = az  # type: AppService
+        self.config = config  # type: Config
+        self.loop = loop  # type: asyncio.AbstractEventLoop
+        self.bot = bot  # type: Optional[Bot]
+        self.mx = None  # type: Optional[MatrixHandler]
+        self.session_container = session_container  # type: AlchemySessionContainer
+        self.public_website = None  # type: Optional[PublicBridgeWebsite]
+        self.provisioning_api = None  # type: Optional[ProvisioningAPI]
 
-    def __iter__(self):
-        yield self.az
-        yield self.db
-        yield self.config
-        yield self.loop
-        yield self.bot
+    @property
+    def core(self) -> Tuple['AppService', 'Config', 'asyncio.AbstractEventLoop', Optional['Bot']]:
+        return self.az, self.config, self.loop, self.bot

mautrix_telegram/db.py

@@ -1,132 +0,0 @@
-# -*- coding: future_fstrings -*-
-# mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from sqlalchemy import (Column, UniqueConstraint, ForeignKey, ForeignKeyConstraint, Integer,
-                        BigInteger, String, Boolean)
-from sqlalchemy.orm import relationship
-
-from .base import Base
-
-
-class Portal(Base):
-    query = None
-    __tablename__ = "portal"
-
-    # Telegram chat information
-    tgid = Column(Integer, primary_key=True)
-    tg_receiver = Column(Integer, primary_key=True)
-    peer_type = Column(String)
-    megagroup = Column(Boolean)
-
-    # Matrix portal information
-    mxid = Column(String, unique=True, nullable=True)
-
-    # Telegram chat metadata
-    username = Column(String, nullable=True)
-    title = Column(String, nullable=True)
-    about = Column(String, nullable=True)
-    photo_id = Column(String, nullable=True)
-
-
-class Message(Base):
-    query = None
-    __tablename__ = "message"
-
-    mxid = Column(String)
-    mx_room = Column(String)
-    tgid = Column(Integer, primary_key=True)
-    tg_space = Column(Integer, primary_key=True)
-
-    __table_args__ = (UniqueConstraint("mxid", "mx_room", "tg_space", name="_mx_id_room"),)
-
-
-class UserPortal(Base):
-    query = None
-    __tablename__ = "user_portal"
-
-    user = Column(Integer, ForeignKey("user.tgid"), primary_key=True)
-    portal = Column(Integer, primary_key=True)
-    portal_receiver = Column(Integer, primary_key=True)
-
-    __table_args__ = (ForeignKeyConstraint(("portal", "portal_receiver"),
-                                           ("portal.tgid", "portal.tg_receiver")),)
-
-
-class User(Base):
-    query = None
-    __tablename__ = "user"
-
-    mxid = Column(String, primary_key=True)
-    tgid = Column(Integer, nullable=True)
-    tg_username = Column(String, nullable=True)
-    saved_contacts = Column(Integer, default=0)
-    contacts = relationship("Contact", uselist=True,
-                            cascade="save-update, merge, delete, delete-orphan")
-    portals = relationship("Portal", secondary="user_portal")
-
-
-class Contact(Base):
-    query = None
-    __tablename__ = "contact"
-
-    user = Column(Integer, ForeignKey("user.tgid"), primary_key=True)
-    contact = Column(Integer, ForeignKey("puppet.id"), primary_key=True)
-
-
-class Puppet(Base):
-    query = None
-    __tablename__ = "puppet"
-
-    id = Column(Integer, primary_key=True)
-    displayname = Column(String, nullable=True)
-    displayname_source = Column(Integer, nullable=True)
-    username = Column(String, nullable=True)
-    photo_id = Column(String, nullable=True)
-    is_bot = Column(Boolean, nullable=True)
-
-
-# Fucking Telegram not telling bots what chats they are in 3:<
-class BotChat(Base):
-    query = None
-    __tablename__ = "bot_chat"
-    id = Column(Integer, primary_key=True)
-    type = Column(String, nullable=False)
-
-
-class TelegramFile(Base):
-    query = None
-    __tablename__ = "telegram_file"
-
-    id = Column(String, primary_key=True)
-    mxc = Column(String)
-    mime_type = Column(String)
-    was_converted = Column(Boolean)
-    timestamp = Column(BigInteger)
-    size = Column(Integer, nullable=True)
-    width = Column(Integer, nullable=True)
-    height = Column(Integer, nullable=True)
-    thumbnail_id = Column("thumbnail", String, ForeignKey("telegram_file.id"), nullable=True)
-    thumbnail = relationship("TelegramFile", uselist=False)
-
-
-def init(db_session):
-    Portal.query = db_session.query_property()
-    Message.query = db_session.query_property()
-    UserPortal.query = db_session.query_property()
-    User.query = db_session.query_property()
-    Puppet.query = db_session.query_property()
-    BotChat.query = db_session.query_property()
-    TelegramFile.query = db_session.query_property()

mautrix_telegram/db/__init__.py

@@ -0,0 +1,33 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from .base import Base
+from .bot_chat import BotChat
+from .message import Message
+from .portal import Portal
+from .puppet import Puppet
+from .room_state import RoomState
+from .telegram_file import TelegramFile
+from .user import User, UserPortal, Contact
+from .user_profile import UserProfile
+
+
+def init(db_engine) -> None:
+    for table in (Portal, Message, User, Contact, UserPortal, Puppet, TelegramFile, UserProfile,
+                  RoomState, BotChat):
+        table.db = db_engine
+        table.t = table.__table__
+        table.c = table.t.c

mautrix_telegram/db/base.py

@@ -0,0 +1,58 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from abc import abstractmethod
+
+from sqlalchemy import Table
+from sqlalchemy.engine.base import Engine
+from sqlalchemy.engine.result import RowProxy
+from sqlalchemy.sql.base import ImmutableColumnCollection
+from sqlalchemy.ext.declarative import declarative_base
+
+
+class BaseBase:
+    db = None  # type: Engine
+    t = None  # type: Table
+    __table__ = None  # type: Table
+    c = None  # type: ImmutableColumnCollection
+
+    @classmethod
+    @abstractmethod
+    def _one_or_none(cls, rows: RowProxy):
+        pass
+
+    @classmethod
+    def _select_one_or_none(cls, *args):
+        return cls._one_or_none(cls.db.execute(cls.t.select().where(*args)))
+
+    @property
+    @abstractmethod
+    def _edit_identity(self):
+        pass
+
+    def update(self, **values) -> None:
+        with self.db.begin() as conn:
+            conn.execute(self.t.update()
+                         .where(self._edit_identity)
+                         .values(**values))
+        for key, value in values.items():
+            setattr(self, key, value)
+
+    def delete(self) -> None:
+        with self.db.begin() as conn:
+            conn.execute(self.t.delete().where(self._edit_identity))
+
+Base = declarative_base(cls=BaseBase)

mautrix_telegram/db/base.pyi

@@ -0,0 +1,26 @@
+from abc import abstractmethod
+
+from sqlalchemy import Table
+from sqlalchemy.engine.base import Engine
+from sqlalchemy.engine.result import RowProxy
+from sqlalchemy.sql.base import ImmutableColumnCollection
+from sqlalchemy.ext.declarative import declarative_base
+
+class Base(declarative_base):
+	db: Engine
+	t: Table
+	__table__: Table
+	c: ImmutableColumnCollection
+
+    @classmethod
+    @abstractmethod
+    def _one_or_none(cls, rows: RowProxy): ...
+
+	@classmethod
+	def _select_one_or_none(cls, *args): ...
+
+    def _edit_identity(self): ...
+
+    def update(self, **values) -> None: ...
+
+    def delete(self) -> None: ...

mautrix_telegram/db/bot_chat.py

@@ -0,0 +1,45 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Iterable
+
+from sqlalchemy import Column, Integer, String
+
+from ..types import TelegramID
+from .base import Base
+
+
+# Fucking Telegram not telling bots what chats they are in 3:<
+class BotChat(Base):
+    __tablename__ = "bot_chat"
+    id = Column(Integer, primary_key=True)  # type: TelegramID
+    type = Column(String, nullable=False)
+
+    @classmethod
+    def delete(cls, chat_id: TelegramID) -> None:
+        with cls.db.begin() as conn:
+            conn.execute(cls.t.delete().where(cls.c.id == chat_id))
+
+    @classmethod
+    def all(cls) -> Iterable['BotChat']:
+        rows = cls.db.execute(cls.t.select())
+        for row in rows:
+            chat_id, chat_type = row
+            yield cls(id=chat_id, type=chat_type)
+
+    def insert(self) -> None:
+        with self.db.begin() as conn:
+            conn.execute(self.t.insert().values(id=self.id, type=self.type))

mautrix_telegram/db/message.py

@@ -0,0 +1,113 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from sqlalchemy import Column, UniqueConstraint, Integer, String, and_, func, desc, select
+from sqlalchemy.engine.result import RowProxy
+from typing import Optional, List
+
+from ..types import MatrixRoomID, MatrixEventID, TelegramID
+from .base import Base
+
+
+class Message(Base):
+    __tablename__ = "message"
+
+    mxid = Column(String)  # type: MatrixEventID
+    mx_room = Column(String)  # type: MatrixRoomID
+    tgid = Column(Integer, primary_key=True)  # type: TelegramID
+    tg_space = Column(Integer, primary_key=True)  # type: TelegramID
+    edit_index = Column(Integer, primary_key=True)  # type: int
+
+    __table_args__ = (UniqueConstraint("mxid", "mx_room", "tg_space", name="_mx_id_room"),)
+
+    @classmethod
+    def _one_or_none(cls, rows: RowProxy) -> Optional['Message']:
+        try:
+            mxid, mx_room, tgid, tg_space, edit_index = next(rows)
+            return cls(mxid=mxid, mx_room=mx_room, tgid=tgid, tg_space=tg_space,
+                       edit_index=edit_index)
+        except StopIteration:
+            return None
+
+    @staticmethod
+    def _all(rows: RowProxy) -> List['Message']:
+        return [Message(mxid=row[0], mx_room=row[1], tgid=row[2], tg_space=row[3],
+                        edit_index=row[4])
+                for row in rows]
+
+    @classmethod
+    def get_all_by_tgid(cls, tgid: TelegramID, tg_space: TelegramID) -> List['Message']:
+        return cls._all(cls.db.execute(cls.t.select().where(and_(cls.c.tgid == tgid,
+                                                                 cls.c.tg_space == tg_space))))
+
+    @classmethod
+    def get_one_by_tgid(cls, tgid: TelegramID, tg_space: TelegramID, edit_index: int = 0
+                        ) -> Optional['Message']:
+        query = cls.t.select()
+        if edit_index < 0:
+            query = (query
+                     .where(and_(cls.c.tgid == tgid, cls.c.tg_space == tg_space))
+                     .order_by(desc(cls.c.edit_index))
+                     .limit(1)
+                     .offset(-edit_index - 1))
+        else:
+            query = query.where(and_(cls.c.tgid == tgid, cls.c.tg_space == tg_space,
+                                     cls.c.edit_index == edit_index))
+        return cls._one_or_none(cls.db.execute(query))
+
+    @classmethod
+    def count_spaces_by_mxid(cls, mxid: MatrixEventID, mx_room: MatrixRoomID) -> int:
+        rows = cls.db.execute(select([func.count(cls.c.tg_space)])
+                              .where(and_(cls.c.mxid == mxid, cls.c.mx_room == mx_room)))
+        try:
+            count, = next(rows)
+            return count
+        except StopIteration:
+            return 0
+
+    @classmethod
+    def get_by_mxid(cls, mxid: MatrixEventID, mx_room: MatrixRoomID, tg_space: TelegramID
+                    ) -> Optional['Message']:
+        return cls._select_one_or_none(and_(cls.c.mxid == mxid,
+                                            cls.c.mx_room == mx_room,
+                                            cls.c.tg_space == tg_space))
+
+    @classmethod
+    def update_by_tgid(cls, s_tgid: TelegramID, s_tg_space: TelegramID, s_edit_index: int,
+                       **values) -> None:
+        with cls.db.begin() as conn:
+            conn.execute(cls.t.update()
+                         .where(and_(cls.c.tgid == s_tgid, cls.c.tg_space == s_tg_space,
+                                     cls.c.edit_index == s_edit_index))
+                         .values(**values))
+
+    @classmethod
+    def update_by_mxid(cls, s_mxid: MatrixEventID, s_mx_room: MatrixRoomID, **values) -> None:
+        with cls.db.begin() as conn:
+            conn.execute(cls.t.update()
+                         .where(and_(cls.c.mxid == s_mxid, cls.c.mx_room == s_mx_room))
+                         .values(**values))
+
+    @property
+    def _edit_identity(self):
+        return and_(self.c.tgid == self.tgid, self.c.tg_space == self.tg_space,
+                    self.c.edit_index == self.edit_index)
+
+    def insert(self) -> None:
+        with self.db.begin() as conn:
+            conn.execute(self.t.insert().values(mxid=self.mxid, mx_room=self.mx_room,
+                                                tgid=self.tgid, tg_space=self.tg_space,
+                                                edit_index=self.edit_index))

mautrix_telegram/db/portal.py

@@ -0,0 +1,81 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from sqlalchemy import Column, Integer, String, Boolean, Text, and_
+from sqlalchemy.engine.result import RowProxy
+from typing import Optional
+
+from ..types import MatrixRoomID, TelegramID
+from .base import Base
+
+
+class Portal(Base):
+    __tablename__ = "portal"
+
+    # Telegram chat information
+    tgid = Column(Integer, primary_key=True)  # type: TelegramID
+    tg_receiver = Column(Integer, primary_key=True)  # type: TelegramID
+    peer_type = Column(String, nullable=False)
+    megagroup = Column(Boolean)
+
+    # Matrix portal information
+    mxid = Column(String, unique=True, nullable=True)  # type: Optional[MatrixRoomID]
+
+    config = Column(Text, nullable=True)
+
+    # Telegram chat metadata
+    username = Column(String, nullable=True)
+    title = Column(String, nullable=True)
+    about = Column(String, nullable=True)
+    photo_id = Column(String, nullable=True)
+
+    @classmethod
+    def scan(cls, row) -> Optional['Portal']:
+        (tgid, tg_receiver, peer_type, megagroup, mxid, config, username, title, about,
+         photo_id) = row
+        return cls(tgid=tgid, tg_receiver=tg_receiver, peer_type=peer_type, megagroup=megagroup,
+                   mxid=mxid, config=config, username=username, title=title, about=about,
+                   photo_id=photo_id)
+
+    @classmethod
+    def _one_or_none(cls, rows: RowProxy) -> Optional['Portal']:
+        try:
+            return cls.scan(next(rows))
+        except StopIteration:
+            return None
+
+    @classmethod
+    def get_by_tgid(cls, tgid: TelegramID, tg_receiver: TelegramID) -> Optional['Portal']:
+        return cls._select_one_or_none(and_(cls.c.tgid == tgid, cls.c.tg_receiver == tg_receiver))
+
+    @classmethod
+    def get_by_mxid(cls, mxid: MatrixRoomID) -> Optional['Portal']:
+        return cls._select_one_or_none(cls.c.mxid == mxid)
+
+    @classmethod
+    def get_by_username(cls, username: str) -> Optional['Portal']:
+        return cls._select_one_or_none(cls.c.username == username)
+
+    @property
+    def _edit_identity(self):
+        return and_(self.c.tgid == self.tgid, self.c.tg_receiver == self.tg_receiver)
+
+    def insert(self) -> None:
+        with self.db.begin() as conn:
+            conn.execute(self.t.insert().values(
+                tgid=self.tgid, tg_receiver=self.tg_receiver, peer_type=self.peer_type,
+                megagroup=self.megagroup, mxid=self.mxid, config=self.config,
+                username=self.username, title=self.title, about=self.about, photo_id=self.photo_id))

mautrix_telegram/db/puppet.py

@@ -0,0 +1,88 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from sqlalchemy import Column, Integer, String, Boolean
+from sqlalchemy.engine.result import RowProxy
+from sqlalchemy.sql import expression
+from typing import Optional, Iterable
+
+from ..types import MatrixUserID, MatrixRoomID, TelegramID
+from .base import Base
+
+
+class Puppet(Base):
+    __tablename__ = "puppet"
+
+    id = Column(Integer, primary_key=True)  # type: TelegramID
+    custom_mxid = Column(String, nullable=True)  # type: Optional[MatrixUserID]
+    access_token = Column(String, nullable=True)
+    displayname = Column(String, nullable=True)
+    displayname_source = Column(Integer, nullable=True)  # type: Optional[TelegramID]
+    username = Column(String, nullable=True)
+    photo_id = Column(String, nullable=True)
+    is_bot = Column(Boolean, nullable=True)
+    matrix_registered = Column(Boolean, nullable=False, server_default=expression.false())
+    disable_updates = Column(Boolean, nullable=False, server_default=expression.false())
+
+    @classmethod
+    def scan(cls, row) -> Optional['Puppet']:
+        (id, custom_mxid, access_token, displayname, displayname_source, username, photo_id,
+         is_bot, matrix_registered, disable_updates) = row
+        return cls(id=id, custom_mxid=custom_mxid, access_token=access_token,
+                   displayname=displayname, displayname_source=displayname_source,
+                   username=username, photo_id=photo_id, is_bot=is_bot,
+                   matrix_registered=matrix_registered, disable_updates=disable_updates)
+
+    @classmethod
+    def _one_or_none(cls, rows: RowProxy) -> Optional['Puppet']:
+        try:
+            return cls.scan(next(rows))
+        except StopIteration:
+            return None
+
+    @classmethod
+    def all_with_custom_mxid(cls) -> Iterable['Puppet']:
+        rows = cls.db.execute(cls.t.select().where(cls.c.custom_mxid != None))
+        for row in rows:
+            yield cls.scan(row)
+
+    @classmethod
+    def get_by_tgid(cls, tgid: TelegramID) -> Optional['Puppet']:
+        return cls._select_one_or_none(cls.c.id == tgid)
+
+    @classmethod
+    def get_by_custom_mxid(cls, mxid: MatrixUserID) -> Optional['Puppet']:
+        return cls._select_one_or_none(cls.c.custom_mxid == mxid)
+
+    @classmethod
+    def get_by_username(cls, username: str) -> Optional['Puppet']:
+        return cls._select_one_or_none(cls.c.username == username)
+
+    @classmethod
+    def get_by_displayname(cls, displayname: str) -> Optional['Puppet']:
+        return cls._select_one_or_none(cls.c.displayname == displayname)
+
+    @property
+    def _edit_identity(self):
+        return self.c.id == self.id
+
+    def insert(self) -> None:
+        with self.db.begin() as conn:
+            conn.execute(self.t.insert().values(
+                id=self.id, custom_mxid=self.custom_mxid, access_token=self.access_token,
+                displayname=self.displayname, displayname_source=self.displayname_source,
+                username=self.username, photo_id=self.photo_id, is_bot=self.is_bot,
+                matrix_registered=self.matrix_registered, disable_updates=self.disable_updates))

mautrix_telegram/db/room_state.py

@@ -0,0 +1,62 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from sqlalchemy import Column, String, Text
+from typing import Dict, Optional
+import json
+
+from ..types import MatrixRoomID
+from .base import Base
+
+
+class RoomState(Base):
+    __tablename__ = "mx_room_state"
+
+    room_id = Column(String, primary_key=True)  # type: MatrixRoomID
+    power_levels = Column("power_levels", Text, nullable=True)  # type: Optional[Dict]
+
+    @property
+    def _power_levels_text(self) -> Optional[str]:
+        return json.dumps(self.power_levels) if self.power_levels else None
+
+    @property
+    def has_power_levels(self) -> bool:
+        return bool(self.power_levels)
+
+    @classmethod
+    def get(cls, room_id: MatrixRoomID) -> Optional['RoomState']:
+        rows = cls.db.execute(cls.t.select().where(cls.c.room_id == room_id))
+        try:
+            room_id, power_levels_text = next(rows)
+            return cls(room_id=room_id, power_levels=(json.loads(power_levels_text)
+                                                      if power_levels_text else None))
+        except StopIteration:
+            return None
+
+    def update(self) -> None:
+        with self.db.begin() as conn:
+            conn.execute(self.t.update()
+                         .where(self.c.room_id == self.room_id)
+                         .values(power_levels=self._power_levels_text))
+
+    @property
+    def _edit_identity(self):
+        return self.c.room_id == self.room_id
+
+    def insert(self) -> None:
+        with self.db.begin() as conn:
+            conn.execute(self.t.insert().values(room_id=self.room_id,
+                                                power_levels=self._power_levels_text))

mautrix_telegram/db/telegram_file.py

@@ -0,0 +1,56 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from sqlalchemy import Column, ForeignKey, Integer, BigInteger, String, Boolean
+from typing import Optional
+
+from .base import Base
+
+
+class TelegramFile(Base):
+    __tablename__ = "telegram_file"
+
+    id = Column(String, primary_key=True)
+    mxc = Column(String)
+    mime_type = Column(String)
+    was_converted = Column(Boolean)
+    timestamp = Column(BigInteger)
+    size = Column(Integer, nullable=True)
+    width = Column(Integer, nullable=True)
+    height = Column(Integer, nullable=True)
+    thumbnail_id = Column("thumbnail", String, ForeignKey("telegram_file.id"), nullable=True)
+    thumbnail = None  # type: Optional[TelegramFile]
+
+    @classmethod
+    def get(cls, loc_id: str) -> Optional['TelegramFile']:
+        rows = cls.db.execute(cls.t.select().where(cls.c.id == loc_id))
+        try:
+            loc_id, mxc, mime, conv, ts, s, w, h, thumb_id = next(rows)
+            thumb = None
+            if thumb_id:
+                thumb = cls.get(thumb_id)
+            return cls(id=loc_id, mxc=mxc, mime_type=mime, was_converted=conv, timestamp=ts,
+                       size=s, width=w, height=h, thumbnail_id=thumb_id, thumbnail=thumb)
+        except StopIteration:
+            return None
+
+    def insert(self) -> None:
+        with self.db.begin() as conn:
+            conn.execute(self.t.insert().values(
+                id=self.id, mxc=self.mxc, mime_type=self.mime_type,
+                was_converted=self.was_converted, timestamp=self.timestamp, size=self.size,
+                width=self.width, height=self.height,
+                thumbnail=self.thumbnail.id if self.thumbnail else self.thumbnail_id))

mautrix_telegram/db/user.py

@@ -0,0 +1,130 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from sqlalchemy import Column, ForeignKey, ForeignKeyConstraint, Integer, String
+from sqlalchemy.engine.result import RowProxy
+from typing import Optional, Iterable, Tuple
+
+from ..types import MatrixUserID, TelegramID
+from .base import Base
+
+
+class User(Base):
+    __tablename__ = "user"
+
+    mxid = Column(String, primary_key=True)  # type: MatrixUserID
+    tgid = Column(Integer, nullable=True, unique=True)  # type: Optional[TelegramID]
+    tg_username = Column(String, nullable=True)
+    tg_phone = Column(String, nullable=True)
+    saved_contacts = Column(Integer, default=0, nullable=False)
+
+    @classmethod
+    def _one_or_none(cls, rows: RowProxy) -> Optional['User']:
+        try:
+            mxid, tgid, tg_username, tg_phone, saved_contacts = next(rows)
+            return cls(mxid=mxid, tgid=tgid, tg_username=tg_username, tg_phone=tg_phone,
+                       saved_contacts=saved_contacts)
+        except StopIteration:
+            return None
+
+    @classmethod
+    def all(cls) -> Iterable['User']:
+        rows = cls.db.execute(cls.t.select())
+        for row in rows:
+            mxid, tgid, tg_username, tg_phone, saved_contacts = row
+            yield cls(mxid=mxid, tgid=tgid, tg_username=tg_username, tg_phone=tg_phone,
+                      saved_contacts=saved_contacts)
+
+    @classmethod
+    def get_by_tgid(cls, tgid: TelegramID) -> Optional['User']:
+        return cls._select_one_or_none(cls.c.tgid == tgid)
+
+    @classmethod
+    def get_by_mxid(cls, mxid: MatrixUserID) -> Optional['User']:
+        return cls._select_one_or_none(cls.c.mxid == mxid)
+
+    @classmethod
+    def get_by_username(cls, username: str) -> Optional['User']:
+        return cls._select_one_or_none(cls.c.tg_username == username)
+
+    @property
+    def _edit_identity(self):
+        return self.c.mxid == self.mxid
+
+    def insert(self) -> None:
+        with self.db.begin() as conn:
+            conn.execute(self.t.insert().values(
+                mxid=self.mxid, tgid=self.tgid, tg_username=self.tg_username,
+                tg_phone=self.tg_phone, saved_contacts=self.saved_contacts))
+
+    @property
+    def contacts(self) -> Iterable[TelegramID]:
+        rows = self.db.execute(Contact.t.select().where(Contact.c.user == self.tgid))
+        for row in rows:
+            user, contact = row
+            yield contact
+
+    @contacts.setter
+    def contacts(self, puppets: Iterable[TelegramID]) -> None:
+        with self.db.begin() as conn:
+            conn.execute(Contact.t.delete().where(Contact.c.user == self.tgid))
+            insert_puppets = [{"user": self.tgid, "contact": tgid} for tgid in puppets]
+            if insert_puppets:
+                conn.execute(Contact.t.insert(), insert_puppets)
+
+    @property
+    def portals(self) -> Iterable[Tuple[TelegramID, TelegramID]]:
+        rows = self.db.execute(UserPortal.t.select().where(UserPortal.c.user == self.tgid))
+        for row in rows:
+            user, portal, portal_receiver = row
+            yield (portal, portal_receiver)
+
+    @portals.setter
+    def portals(self, portals: Iterable[Tuple[TelegramID, TelegramID]]) -> None:
+        with self.db.begin() as conn:
+            conn.execute(UserPortal.t.delete().where(UserPortal.c.user == self.tgid))
+            insert_portals = [{
+                "user": self.tgid,
+                "portal": tgid,
+                "portal_receiver": tg_receiver
+            } for tgid, tg_receiver in portals]
+            if insert_portals:
+                conn.execute(UserPortal.t.insert(), insert_portals)
+
+    def delete(self) -> None:
+        super().delete()
+        self.portals = []
+        self.contacts = []
+
+
+class UserPortal(Base):
+    __tablename__ = "user_portal"
+
+    user = Column(Integer, ForeignKey("user.tgid", onupdate="CASCADE", ondelete="CASCADE"),
+                  primary_key=True)  # type: TelegramID
+    portal = Column(Integer, primary_key=True)  # type: TelegramID
+    portal_receiver = Column(Integer, primary_key=True)  # type: TelegramID
+
+    __table_args__ = (ForeignKeyConstraint(("portal", "portal_receiver"),
+                                           ("portal.tgid", "portal.tg_receiver"),
+                                           onupdate="CASCADE", ondelete="CASCADE"),)
+
+
+class Contact(Base):
+    __tablename__ = "contact"
+
+    user = Column(Integer, ForeignKey("user.tgid"), primary_key=True)  # type: TelegramID
+    contact = Column(Integer, ForeignKey("puppet.id"), primary_key=True)  # type: TelegramID

mautrix_telegram/db/user_profile.py

@@ -0,0 +1,69 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from sqlalchemy import Column, String, and_
+from typing import Dict, Optional
+
+from ..types import MatrixUserID, MatrixRoomID
+from .base import Base
+
+
+class UserProfile(Base):
+    __tablename__ = "mx_user_profile"
+
+    room_id = Column(String, primary_key=True)  # type: MatrixRoomID
+    user_id = Column(String, primary_key=True)  # type: MatrixUserID
+    membership = Column(String, nullable=False, default="leave")
+    displayname = Column(String, nullable=True)
+    avatar_url = Column(String, nullable=True)
+
+    def dict(self) -> Dict[str, str]:
+        return {
+            "membership": self.membership,
+            "displayname": self.displayname,
+            "avatar_url": self.avatar_url,
+        }
+
+    @classmethod
+    def get(cls, room_id: MatrixRoomID, user_id: MatrixUserID) -> Optional['UserProfile']:
+        rows = cls.db.execute(
+            cls.t.select().where(and_(cls.c.room_id == room_id, cls.c.user_id == user_id)))
+        try:
+            room_id, user_id, membership, displayname, avatar_url = next(rows)
+            return cls(room_id=room_id, user_id=user_id, membership=membership,
+                       displayname=displayname, avatar_url=avatar_url)
+        except StopIteration:
+            return None
+
+    @classmethod
+    def delete_all(cls, room_id: MatrixRoomID) -> None:
+        with cls.db.begin() as conn:
+            conn.execute(cls.t.delete().where(cls.c.room_id == room_id))
+
+    def update(self) -> None:
+        super().update(membership=self.membership, displayname=self.displayname,
+                       avatar_url=self.avatar_url)
+
+    @property
+    def _edit_identity(self):
+        return and_(self.c.room_id == self.room_id, self.c.user_id == self.user_id)
+
+    def insert(self) -> None:
+        with self.db.begin() as conn:
+            conn.execute(self.t.insert().values(room_id=self.room_id, user_id=self.user_id,
+                                                membership=self.membership,
+                                                displayname=self.displayname,
+                                                avatar_url=self.avatar_url))

mautrix_telegram/formatter/__init__.py

@@ -1,9 +1,8 @@
 from .from_matrix import (matrix_reply_to_telegram, matrix_to_telegram, matrix_text_to_telegram,
                           init_mx)
-from .from_telegram import (telegram_reply_to_matrix, telegram_to_matrix, init_tg)
-from ..context import Context
+from .from_telegram import telegram_reply_to_matrix, telegram_to_matrix
+from .. import context as c
 
 
-def init(context: Context):
+def init(context: c.Context) -> None:
     init_mx(context)
-    init_tg(context)

mautrix_telegram/formatter/from_matrix.py

@@ -1,339 +0,0 @@
-# -*- coding: future_fstrings -*-
-# mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from html import unescape
-from html.parser import HTMLParser
-from collections import deque
-from typing import Optional, List, Tuple, Type, Callable, Dict, Any
-import math
-import re
-import logging
-
-from telethon.tl.types import (MessageEntityMention,
-                               InputMessageEntityMentionName, MessageEntityEmail,
-                               MessageEntityUrl, MessageEntityTextUrl, MessageEntityBold,
-                               MessageEntityItalic, MessageEntityCode, MessageEntityPre,
-                               MessageEntityBotCommand, InputUser, TypeMessageEntity)
-
-from ..context import Context
-from .. import user as u, puppet as pu, portal as po
-from ..db import Message as DBMessage
-from .util import (add_surrogates, remove_surrogates, trim_reply_fallback_html,
-                   trim_reply_fallback_text, html_to_unicode)
-
-log = logging.getLogger("mau.fmt.mx")
-should_bridge_plaintext_highlights = False
-
-
-class MatrixParser(HTMLParser):
-    mention_regex = re.compile("https://matrix.to/#/(@.+:.+)")
-    room_regex = re.compile("https://matrix.to/#/(#.+:.+)")
-    block_tags = ("br", "p", "pre", "blockquote",
-                  "ol", "ul", "li",
-                  "h1", "h2", "h3", "h4", "h5", "h6",
-                  "div", "hr", "table")
-
-    def __init__(self):
-        super().__init__()
-        self.text = ""
-        self.entities = []
-        self._building_entities = {}
-        self._list_counter = 0
-        self._open_tags = deque()
-        self._open_tags_meta = deque()
-        self._line_is_new = True
-        self._list_entry_is_new = False
-
-    def _parse_url(self, url: str, args: Dict[str, Any]
-                   ) -> Tuple[Optional[Type[TypeMessageEntity]], Optional[str]]:
-        mention = self.mention_regex.match(url)
-        if mention:
-            mxid = mention.group(1)
-            user = (pu.Puppet.get_by_mxid(mxid)
-                    or u.User.get_by_mxid(mxid, create=False))
-            if not user:
-                return None, None
-            if user.username:
-                return MessageEntityMention, f"@{user.username}"
-            elif user.tgid:
-                args["user_id"] = InputUser(user.tgid, 0)
-                return InputMessageEntityMentionName, user.displayname or None
-            else:
-                return None, None
-
-        room = self.room_regex.match(url)
-        if room:
-            username = po.Portal.get_username_from_mx_alias(room.group(1))
-            portal = po.Portal.find_by_username(username)
-            if portal and portal.username:
-                return MessageEntityMention, f"@{portal.username}"
-
-        if url.startswith("mailto:"):
-            return MessageEntityEmail, url[len("mailto:"):]
-        elif self.get_starttag_text() == url:
-            return MessageEntityUrl, url
-        else:
-            args["url"] = url
-            return MessageEntityTextUrl, None
-
-    def handle_starttag(self, tag: str, attrs: List[Tuple[str, str]]):
-        self._open_tags.appendleft(tag)
-        self._open_tags_meta.appendleft(0)
-
-        attrs = dict(attrs)
-        entity_type = None
-        args = {}
-        if tag in ("strong", "b"):
-            entity_type = MessageEntityBold
-        elif tag in ("em", "i"):
-            entity_type = MessageEntityItalic
-        elif tag == "code":
-            try:
-                pre = self._building_entities["pre"]
-                try:
-                    # Pre tag and language found, add language to MessageEntityPre
-                    pre.language = attrs["class"][len("language-"):]
-                except KeyError:
-                    # Pre tag found, but language not found, keep pre as-is
-                    pass
-            except KeyError:
-                # No pre tag found, this is inline code
-                entity_type = MessageEntityCode
-        elif tag == "pre":
-            entity_type = MessageEntityPre
-            args["language"] = ""
-        elif tag == "command":
-            entity_type = MessageEntityBotCommand
-        elif tag == "li":
-            self._list_entry_is_new = True
-        elif tag == "a":
-            try:
-                url = attrs["href"]
-            except KeyError:
-                return
-            entity_type, url = self._parse_url(url, args)
-            self._open_tags_meta.popleft()
-            self._open_tags_meta.appendleft(url)
-
-        if tag in self.block_tags and ("blockquote" not in self._open_tags or tag == "br"):
-            self._newline()
-
-        if entity_type and tag not in self._building_entities:
-            offset = len(self.text)
-            self._building_entities[tag] = entity_type(offset=offset, length=0, **args)
-
-    @property
-    def _list_indent(self) -> int:
-        indent = 0
-        first_skipped = False
-        for index, tag in enumerate(self._open_tags):
-            if not first_skipped and tag in ("ol", "ul"):
-                # The first list level isn't indented, so skip it.
-                first_skipped = True
-                continue
-            if tag == "ol":
-                n = self._open_tags_meta[index]
-                extra_length_for_long_index = (int(math.log(n, 10)) - 1) * 3
-                indent += 4 + extra_length_for_long_index
-            elif tag == "ul":
-                indent += 3
-        return indent
-
-    def _newline(self, allow_multi: bool = False):
-        if self._line_is_new and not allow_multi:
-            return
-        self.text += "\n"
-        self._line_is_new = True
-        for entity in self._building_entities.values():
-            entity.length += 1
-
-    def _handle_special_previous_tags(self, text: str) -> str:
-        if "pre" not in self._open_tags and "code" not in self._open_tags:
-            text = text.replace("\n", "")
-        else:
-            text = text.strip()
-
-        previous_tag = self._open_tags[0] if len(self._open_tags) > 0 else ""
-        if previous_tag == "a":
-            url = self._open_tags_meta[0]
-            if url:
-                text = url
-        elif previous_tag == "command":
-            text = f"/{text}"
-        return text
-
-    def _html_to_unicode(self, text: str) -> str:
-        strikethrough, underline = "del" in self._open_tags, "u" in self._open_tags
-        if strikethrough and underline:
-            text = html_to_unicode(text, "\u0336\u0332")
-        elif strikethrough:
-            text = html_to_unicode(text, "\u0336")
-        elif underline:
-            text = html_to_unicode(text, "\u0332")
-        return text
-
-    def _handle_tags_for_data(self, text: str) -> Tuple[str, int]:
-        extra_offset = 0
-        list_entry_handled_once = False
-        # In order to maintain order of things like blockquotes in lists or lists in blockquotes,
-        # we can't just have ifs/elses and we need to actually loop through the open tags in order.
-        for index, tag in enumerate(self._open_tags):
-            if tag == "blockquote" and self._line_is_new:
-                text = f"> {text}"
-                extra_offset += 2
-            elif tag == "li" and not list_entry_handled_once:
-                list_type_index = index + 1
-                list_type = self._open_tags[list_type_index]
-                indent = self._list_indent * " " if self._line_is_new else ""
-                if list_type == "ol":
-                    n = self._open_tags_meta[list_type_index]
-                    if self._list_entry_is_new:
-                        n += 1
-                        self._open_tags_meta[list_type_index] = n
-                        prefix = f"{n}. "
-                    else:
-                        prefix = int(math.log(n, 10)) * 3 * " " + 4 * " "
-                else:
-                    prefix = "* " if self._list_entry_is_new else 3 * " "
-                if not self._list_entry_is_new and not self._line_is_new:
-                    prefix = ""
-                extra_offset += len(indent) + len(prefix)
-                text = indent + prefix + text
-                self._list_entry_is_new = False
-                list_entry_handled_once = True
-        return text, extra_offset
-
-    def _extend_entities_in_construction(self, text: str, extra_offset: int):
-        for tag, entity in self._building_entities.items():
-            entity.length += len(text) - extra_offset
-            entity.offset += extra_offset
-
-    def handle_data(self, text: str):
-        text = unescape(text)
-        text = self._handle_special_previous_tags(text)
-        text = self._html_to_unicode(text)
-        text, extra_offset = self._handle_tags_for_data(text)
-        self._extend_entities_in_construction(text, extra_offset)
-        self._line_is_new = False
-        self.text += text
-
-    def handle_endtag(self, tag: str):
-        try:
-            self._open_tags.popleft()
-            self._open_tags_meta.popleft()
-        except IndexError:
-            pass
-
-        entity = self._building_entities.pop(tag, None)
-        if entity:
-            self.entities.append(entity)
-
-        if tag in self.block_tags and tag != "br" and "blockquote" not in self._open_tags:
-            self._newline(allow_multi=tag == "br")
-
-
-command_regex = re.compile(r"^!([A-Za-z0-9@]+)")
-not_command_regex = re.compile(r"^\\(![A-Za-z0-9@]+)")
-plain_mention_regex = None
-
-
-def plain_mention_to_html(match):
-    puppet = pu.Puppet.find_by_displayname(match.group(2))
-    if puppet:
-        return (f"{match.group(1)}"
-                f"<a href='https://matrix.to/#/{puppet.mxid}'>"
-                f"{puppet.displayname}"
-                "</a>")
-    return "".join(match.groups())
-
-
-def matrix_to_telegram(html: str) -> Tuple[str, List[TypeMessageEntity]]:
-    try:
-        parser = MatrixParser()
-        html = command_regex.sub(r"<command>\1</command>", html)
-        html = not_command_regex.sub(r"\1", html)
-        if should_bridge_plaintext_highlights:
-            html = plain_mention_regex.sub(plain_mention_to_html, html)
-        parser.feed(add_surrogates(html))
-        return remove_surrogates(parser.text.strip()), parser.entities
-    except Exception:
-        log.exception("Failed to convert Matrix format:\nhtml=%s", html)
-
-
-def matrix_reply_to_telegram(content: dict, tg_space: int, room_id: Optional[str] = None
-                             ) -> Optional[int]:
-    try:
-        reply = content["m.relates_to"]["m.in_reply_to"]
-        room_id = room_id or reply["room_id"]
-        event_id = reply["event_id"]
-
-        try:
-            if content["format"] == "org.matrix.custom.html":
-                content["formatted_body"] = trim_reply_fallback_html(content["formatted_body"])
-        except KeyError:
-            pass
-        content["body"] = trim_reply_fallback_text(content["body"])
-
-        message = DBMessage.query.filter(DBMessage.mxid == event_id,
-                                         DBMessage.tg_space == tg_space,
-                                         DBMessage.mx_room == room_id).one_or_none()
-        if message:
-            return message.tgid
-    except KeyError:
-        pass
-    return None
-
-
-def matrix_text_to_telegram(text: str) -> Tuple[str, List[TypeMessageEntity]]:
-    text = command_regex.sub(r"/\1", text)
-    text = not_command_regex.sub(r"\1", text)
-    if should_bridge_plaintext_highlights:
-        entities, pmr_replacer = plain_mention_to_text()
-        text = plain_mention_regex.sub(pmr_replacer, text)
-    else:
-        entities = []
-    return text, entities
-
-
-def plain_mention_to_text() -> Tuple[List[TypeMessageEntity], Callable[[str], str]]:
-    entities = []
-
-    def replacer(match):
-        puppet = pu.Puppet.find_by_displayname(match.group(2))
-        if puppet:
-            offset = match.start()
-            length = match.end() - offset
-            if puppet.username:
-                entity = MessageEntityMention(offset, length)
-                text = f"@{puppet.username}"
-            else:
-                entity = InputMessageEntityMentionName(offset, length,
-                                                       user_id=InputUser(puppet.tgid, 0))
-                text = puppet.displayname
-            entities.append(entity)
-            return text
-        return "".join(match.groups())
-
-    return entities, replacer
-
-
-def init_mx(context: Context):
-    global plain_mention_regex, should_bridge_plaintext_highlights
-    config = context.config
-    dn_template = config.get("bridge.displayname_template", "{displayname} (Telegram)")
-    dn_template = re.escape(dn_template).replace(re.escape("{displayname}"), "[^>]+")
-    plain_mention_regex = re.compile(f"(\s|^)({dn_template})")
-    should_bridge_plaintext_highlights = config["bridge.plaintext_highlights"] or False

mautrix_telegram/formatter/from_matrix/__init__.py

@@ -0,0 +1,154 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Optional, List, Tuple, Callable, Pattern, Match, TYPE_CHECKING, Dict, Any
+import re
+import logging
+
+from telethon.tl.types import (MessageEntityMention, MessageEntityMentionName, MessageEntityItalic,
+                               TypeMessageEntity)
+
+from ... import puppet as pu
+from ...types import TelegramID, MatrixRoomID
+from ...db import Message as DBMessage
+from ..util import (add_surrogates, remove_surrogates, trim_reply_fallback_html,
+                    trim_reply_fallback_text)
+from .parser import ParsedMessage, parse_html
+
+if TYPE_CHECKING:
+    from ...context import Context
+
+log = logging.getLogger("mau.fmt.mx")  # type: logging.Logger
+should_bridge_plaintext_highlights = False  # type: bool
+
+command_regex = re.compile(r"^!([A-Za-z0-9@]+)")  # type: Pattern
+not_command_regex = re.compile(r"^\\(![A-Za-z0-9@]+)")  # type: Pattern
+plain_mention_regex = None  # type: Optional[Pattern]
+
+
+def plain_mention_to_html(match: Match) -> str:
+    puppet = pu.Puppet.find_by_displayname(match.group(2))
+    if puppet:
+        return (f"{match.group(1)}"
+                f"<a href='https://matrix.to/#/{puppet.mxid}'>"
+                f"{puppet.displayname}"
+                "</a>")
+    return "".join(match.groups())
+
+
+def cut_long_message(message: str, entities: List[TypeMessageEntity]) -> ParsedMessage:
+    if len(message) > 4096:
+        message = message[0:4082] + " [message cut]"
+        new_entities = []
+        for entity in entities:
+            if entity.offset > 4082:
+                continue
+            if entity.offset + entity.length > 4082:
+                entity.length = 4082 - entity.offset
+            new_entities.append(entity)
+        new_entities.append(MessageEntityItalic(4082, len(" [message cut]")))
+        entities = new_entities
+    return message, entities
+
+
+class FormatError(Exception):
+    pass
+
+
+def matrix_to_telegram(html: str) -> ParsedMessage:
+    try:
+        html = command_regex.sub(r"<command>\1</command>", html)
+        html = html.replace("\t", " " * 4)
+        html = not_command_regex.sub(r"\1", html)
+        if should_bridge_plaintext_highlights:
+            html = plain_mention_regex.sub(plain_mention_to_html, html)
+
+        text, entities = parse_html(add_surrogates(html))
+        text = remove_surrogates(text.strip())
+        text, entities = cut_long_message(text, entities)
+
+        return text, entities
+    except Exception as e:
+        raise FormatError(f"Failed to convert Matrix format: {html}") from e
+
+
+def matrix_reply_to_telegram(content: Dict[str, Any], tg_space: TelegramID,
+                             room_id: Optional[MatrixRoomID] = None) -> Optional[TelegramID]:
+    relates_to = content.get("m.relates_to", None) or {}
+    if not relates_to:
+        return None
+    reply = (relates_to if relates_to.get("rel_type", None) == "m.reference"
+             else relates_to.get("m.in_reply_to", None) or {})
+    if not reply:
+        return None
+    room_id = room_id or reply.get("room_id", None)
+    event_id = reply.get("event_id", None)
+    if not event_id:
+        return
+
+    try:
+        if content["format"] == "org.matrix.custom.html":
+            content["formatted_body"] = trim_reply_fallback_html(content["formatted_body"])
+    except KeyError:
+        pass
+    content["body"] = trim_reply_fallback_text(content["body"])
+
+    message = DBMessage.get_by_mxid(event_id, room_id, tg_space)
+    if message:
+        return message.tgid
+    return None
+
+
+def matrix_text_to_telegram(text: str) -> ParsedMessage:
+    text = command_regex.sub(r"/\1", text)
+    text = text.replace("\t", " " * 4)
+    text = not_command_regex.sub(r"\1", text)
+    if should_bridge_plaintext_highlights:
+        entities, pmr_replacer = plain_mention_to_text()
+        text = plain_mention_regex.sub(pmr_replacer, text)
+    else:
+        entities = []
+    return text, entities
+
+
+def plain_mention_to_text() -> Tuple[List[TypeMessageEntity], Callable[[str], str]]:
+    entities = []
+
+    def replacer(match) -> str:
+        puppet = pu.Puppet.find_by_displayname(match.group(2))
+        if puppet:
+            offset = match.start()
+            length = match.end() - offset
+            if puppet.username:
+                entity = MessageEntityMention(offset, length)
+                text = f"@{puppet.username}"
+            else:
+                entity = MessageEntityMentionName(offset, length, user_id=puppet.tgid)
+                text = puppet.displayname
+            entities.append(entity)
+            return text
+        return "".join(match.groups())
+
+    return entities, replacer
+
+
+def init_mx(context: "Context") -> None:
+    global plain_mention_regex, should_bridge_plaintext_highlights
+    config = context.config
+    dn_template = config.get("bridge.displayname_template", "{displayname} (Telegram)")
+    dn_template = re.escape(dn_template).replace(re.escape("{displayname}"), "[^>]+")
+    plain_mention_regex = re.compile(f"^({dn_template})")
+    should_bridge_plaintext_highlights = config["bridge.plaintext_highlights"] or False

mautrix_telegram/formatter/from_matrix/html_reader.py

@@ -0,0 +1,66 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Dict, List, Tuple
+
+from html.parser import HTMLParser
+
+
+class HTMLNode(list):
+    def __init__(self, tag: str, attrs: List[Tuple[str, str]]):
+        super().__init__()
+        self.tag = tag  # type: str
+        self.text = ""  # type: str
+        self.tail = ""  # type: str
+        self.attrib = dict(attrs)  # type: Dict[str, str]
+
+
+class NodeifyingParser(HTMLParser):
+    # From https://www.w3.org/TR/html5/syntax.html#writing-html-documents-elements
+    void_tags = ("area", "base", "br", "col", "command", "embed", "hr", "img", "input", "link",
+                 "meta", "param", "source", "track", "wbr")
+
+    def __init__(self):
+        super().__init__()
+        self.stack = [HTMLNode("html", [])]  # type: List[HTMLNode]
+
+    def handle_starttag(self, tag, attrs):
+        node = HTMLNode(tag, attrs)
+        self.stack[-1].append(node)
+        if tag not in self.void_tags:
+            self.stack.append(node)
+
+    def handle_startendtag(self, tag, attrs):
+        self.stack[-1].append(HTMLNode(tag, attrs))
+
+    def handle_endtag(self, tag):
+        if tag == self.stack[-1].tag:
+            self.stack.pop()
+
+    def handle_data(self, data):
+        if len(self.stack[-1]) > 0:
+            self.stack[-1][-1].tail += data
+        else:
+            self.stack[-1].text += data
+
+    def error(self, message):
+        pass
+
+
+def read_html(data: str) -> HTMLNode:
+    parser = NodeifyingParser()
+    parser.feed(data)
+    return parser.stack[0]

mautrix_telegram/formatter/from_matrix/html_reader.pyi

@@ -0,0 +1,11 @@
+from typing import Dict, List
+
+
+class HTMLNode(List['HTMLNode']):
+    tag: str
+    text: str
+    tail: str
+    attrib: Dict[str, str]
+
+
+def read_html(data: str) -> HTMLNode: ...

mautrix_telegram/formatter/from_matrix/parser.py

@@ -0,0 +1,253 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import List, Tuple, Pattern
+import re
+
+from telethon.tl.types import (MessageEntityMention as Mention, MessageEntityBotCommand as Command,
+                               MessageEntityMentionName as MentionName, MessageEntityUrl as URL,
+                               MessageEntityEmail as Email, MessageEntityTextUrl as TextURL,
+                               MessageEntityBold as Bold, MessageEntityItalic as Italic,
+                               MessageEntityCode as Code, MessageEntityPre as Pre,
+                               MessageEntityStrike as Strike, MessageEntityUnderline as Underline,
+                               MessageEntityBlockquote as Blockquote, TypeMessageEntity)
+
+from ... import user as u, puppet as pu, portal as po
+from ...types import MatrixUserID
+from .telegram_message import TelegramMessage, Entity, offset_length_multiply
+
+from .html_reader import HTMLNode, read_html
+
+ParsedMessage = Tuple[str, List[TypeMessageEntity]]
+
+
+def parse_html(input_html: str) -> ParsedMessage:
+    return MatrixParser.parse(input_html)
+
+
+class RecursionContext:
+    def __init__(self, strip_linebreaks: bool = True, ul_depth: int = 0):
+        self.strip_linebreaks = strip_linebreaks  # type: bool
+        self.ul_depth = ul_depth  # type: int
+        self._inited = True  # type: bool
+
+    def __setattr__(self, key, value):
+        if getattr(self, "_inited", False) is True:
+            raise TypeError("'RecursionContext' object is immutable")
+        super(RecursionContext, self).__setattr__(key, value)
+
+    def enter_list(self) -> 'RecursionContext':
+        return RecursionContext(strip_linebreaks=self.strip_linebreaks, ul_depth=self.ul_depth + 1)
+
+    def enter_code_block(self) -> 'RecursionContext':
+        return RecursionContext(strip_linebreaks=False, ul_depth=self.ul_depth)
+
+
+class MatrixParser:
+    mention_regex = re.compile("https://matrix.to/#/(@.+:.+)")  # type: Pattern
+    room_regex = re.compile("https://matrix.to/#/(#.+:.+)")  # type: Pattern
+    block_tags = ("p", "pre", "blockquote",
+                  "ol", "ul", "li",
+                  "h1", "h2", "h3", "h4", "h5", "h6",
+                  "div", "hr", "table")  # type: Tuple[str, ...]
+    list_bullets = ("●", "○", "■", "‣")  # type: Tuple[str, ...]
+
+    @classmethod
+    def list_bullet(cls, depth: int) -> str:
+        return cls.list_bullets[(depth - 1) % len(cls.list_bullets)] + " "
+
+    @classmethod
+    def list_to_tmessage(cls, node: HTMLNode, ctx: RecursionContext) -> TelegramMessage:
+        ordered = node.tag == "ol"
+        tagged_children = cls.node_to_tagged_tmessages(node, ctx)
+        counter = 1
+        indent_length = 0
+        if ordered:
+            try:
+                counter = int(node.attrib.get("start", "1"))
+            except ValueError:
+                counter = 1
+
+            longest_index = counter - 1 + len(tagged_children)
+            indent_length = len(str(longest_index))
+        indent = (indent_length + 4) * " "
+        children = []  # type: List[TelegramMessage]
+        for child, tag in tagged_children:
+            if tag != "li":
+                continue
+
+            if ordered:
+                prefix = f"{counter}. "
+                counter += 1
+            else:
+                prefix = cls.list_bullet(ctx.ul_depth)
+            child = child.prepend(prefix)
+            parts = child.split("\n")
+            parts = parts[:1] + [part.prepend(indent) for part in parts[1:]]
+            child = TelegramMessage.join(parts, "\n")
+            children.append(child)
+        return TelegramMessage.join(children, "\n")
+
+    @classmethod
+    def header_to_tmessage(cls, node: HTMLNode, ctx: RecursionContext) -> TelegramMessage:
+        children = cls.node_to_tmessages(node, ctx)
+        length = int(node.tag[1])
+        prefix = "#" * length + " "
+        return TelegramMessage.join(children, "").prepend(prefix).format(Bold)
+
+    @classmethod
+    def basic_format_to_tmessage(cls, node: HTMLNode, ctx: RecursionContext) -> TelegramMessage:
+        msg = cls.tag_aware_parse_node(node, ctx)
+        if node.tag in ("b", "strong"):
+            msg.format(Bold)
+        elif node.tag in ("i", "em"):
+            msg.format(Italic)
+        elif node.tag in ("s", "strike", "del"):
+            msg.format(Strike)
+        elif node.tag in ("u", "ins"):
+            msg.format(Underline)
+        elif node == "blockquote":
+            msg.format(Blockquote)
+        elif node.tag == "command":
+            msg.format(Command)
+
+        return msg
+
+    @classmethod
+    def link_to_tstring(cls, node: HTMLNode, ctx: RecursionContext) -> TelegramMessage:
+        msg = cls.tag_aware_parse_node(node, ctx)
+        href = node.attrib.get("href", "")
+        if not href:
+            return msg
+
+        if href.startswith("mailto:"):
+            return TelegramMessage(href[len("mailto:"):]).format(Email)
+
+        mention = cls.mention_regex.match(href)
+        if mention:
+            mxid = MatrixUserID(mention.group(1))
+            user = (pu.Puppet.get_by_mxid(mxid)
+                    or u.User.get_by_mxid(mxid, create=False))
+            if not user:
+                return msg
+            if user.username:
+                return TelegramMessage(f"@{user.username}").format(Mention)
+            elif user.tgid:
+                displayname = user.plain_displayname or msg.text
+                return TelegramMessage(displayname).format(MentionName, user_id=user.tgid)
+            return msg
+
+        room = cls.room_regex.match(href)
+        if room:
+            username = po.Portal.get_username_from_mx_alias(room.group(1))
+            portal = po.Portal.find_by_username(username)
+            if portal and portal.username:
+                return TelegramMessage(f"@{portal.username}").format(Mention)
+
+        return (msg.format(URL)
+                if msg.text == href
+                else msg.format(TextURL, url=href))
+
+    @classmethod
+    def blockquote_to_tmessage(cls, node: HTMLNode, ctx: RecursionContext) -> TelegramMessage:
+        msg = cls.tag_aware_parse_node(node, ctx)
+        children = msg.trim().split("\n")
+        children = [child.prepend("> ") for child in children]
+        return TelegramMessage.join(children, "\n")
+
+    @classmethod
+    def node_to_tmessage(cls, node: HTMLNode, ctx: RecursionContext) -> TelegramMessage:
+        if node.tag == "mx-reply":
+            return TelegramMessage("")
+        elif node.tag == "ol":
+            return cls.list_to_tmessage(node, ctx)
+        elif node.tag == "ul":
+            return cls.list_to_tmessage(node, ctx.enter_list())
+        elif node.tag in ("h1", "h2", "h3", "h4", "h5", "h6"):
+            return cls.header_to_tmessage(node, ctx)
+        elif node.tag == "br":
+            return TelegramMessage("\n")
+        elif node.tag in ("b", "strong", "i", "em", "s", "del", "u", "ins", "command"):
+            return cls.basic_format_to_tmessage(node, ctx)
+        elif node.tag == "blockquote":
+            # Telegram already has blockquote entities in the protocol schema, but it strips them
+            # server-side and none of the official clients support them.
+            # TODO once Telegram changes that, use the above if block for blockquotes too.
+            return cls.blockquote_to_tmessage(node, ctx)
+        elif node.tag == "a":
+            return cls.link_to_tstring(node, ctx)
+        elif node.tag == "p":
+            return cls.tag_aware_parse_node(node, ctx).append("\n")
+        elif node.tag == "pre":
+            lang = ""
+            try:
+                if node[0].tag == "code":
+                    node = node[0]
+                    lang = node.attrib["class"][len("language-"):]
+            except (IndexError, KeyError):
+                pass
+            return cls.parse_node(node, ctx.enter_code_block()).format(Pre, language=lang)
+        elif node.tag == "code":
+            return cls.parse_node(node, ctx.enter_code_block()).format(Code)
+        return cls.tag_aware_parse_node(node, ctx)
+
+    @staticmethod
+    def text_to_tmessage(text: str, ctx: RecursionContext) -> TelegramMessage:
+        if ctx.strip_linebreaks:
+            text = text.replace("\n", "")
+        return TelegramMessage(text)
+
+    @classmethod
+    def node_to_tagged_tmessages(cls, node: HTMLNode, ctx: RecursionContext
+                                 ) -> List[Tuple[TelegramMessage, str]]:
+        output = []
+
+        if node.text:
+            output.append((cls.text_to_tmessage(node.text, ctx), "text"))
+        for child in node:
+            output.append((cls.node_to_tmessage(child, ctx), child.tag))
+            if child.tail:
+                output.append((cls.text_to_tmessage(child.tail, ctx), "text"))
+        return output
+
+    @classmethod
+    def node_to_tmessages(cls, node: HTMLNode, ctx: RecursionContext
+                          ) -> List[TelegramMessage]:
+        return [msg for (msg, tag) in cls.node_to_tagged_tmessages(node, ctx)]
+
+    @classmethod
+    def tag_aware_parse_node(cls, node: HTMLNode, ctx: RecursionContext
+                             ) -> TelegramMessage:
+        msgs = cls.node_to_tagged_tmessages(node, ctx)
+        output = TelegramMessage()
+        prev_was_block = False
+        for msg, tag in msgs:
+            if tag in cls.block_tags:
+                msg = msg.append("\n")
+                if not prev_was_block:
+                    msg = msg.prepend("\n")
+                prev_was_block = True
+            output = output.append(msg)
+        return output.trim()
+
+    @classmethod
+    def parse_node(cls, node: HTMLNode, ctx: RecursionContext) -> TelegramMessage:
+        return TelegramMessage.join(cls.node_to_tmessages(node, ctx))
+
+    @classmethod
+    def parse(cls, data: str) -> ParsedMessage:
+        msg = cls.node_to_tmessage(read_html(f"<body>{data}</body>"), RecursionContext())
+        return msg.text, msg.entities

mautrix_telegram/formatter/from_matrix/telegram_message.py

@@ -0,0 +1,158 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Callable, List, Optional, Sequence, Type, Union
+
+from telethon.tl.types import (MessageEntityMentionName as MentionName,
+                               MessageEntityTextUrl as TextURL, MessageEntityPre as Pre,
+                               TypeMessageEntity, InputMessageEntityMentionName as InputMentionName)
+
+
+class Entity:
+    @staticmethod
+    def copy(entity: TypeMessageEntity) -> Optional[TypeMessageEntity]:
+        if not entity:
+            return None
+        kwargs = {
+            "offset": entity.offset,
+            "length": entity.length,
+        }
+        if isinstance(entity, Pre):
+            kwargs["language"] = entity.language
+        elif isinstance(entity, TextURL):
+            kwargs["url"] = entity.url
+        elif isinstance(entity, (MentionName, InputMentionName)):
+            kwargs["user_id"] = entity.user_id
+        return entity.__class__(**kwargs)
+
+    @classmethod
+    def adjust(cls, entity: Union[TypeMessageEntity, List[TypeMessageEntity]],
+               func: Callable[[TypeMessageEntity], None]
+               ) -> Union[Optional[TypeMessageEntity], List[TypeMessageEntity]]:
+        if isinstance(entity, list):
+            return [Entity.adjust(element, func) for element in entity if entity]
+        elif not entity:
+            return None
+        entity = cls.copy(entity)
+        func(entity)
+        if entity.offset < 0:
+            entity.length += entity.offset
+            entity.offset = 0
+        return entity
+
+
+def offset_diff(amount: int) -> Callable[[TypeMessageEntity], None]:
+    def func(entity: TypeMessageEntity) -> None:
+        entity.offset += amount
+
+    return func
+
+
+def offset_length_multiply(amount: int) -> Callable[[TypeMessageEntity], None]:
+    def func(entity: TypeMessageEntity) -> None:
+        entity.offset *= amount
+        entity.length *= amount
+
+    return func
+
+
+class TelegramMessage:
+    def __init__(self, text: str = "", entities: Optional[List[TypeMessageEntity]] = None) -> None:
+        self.text = text  # type: str
+        self.entities = entities or []  # type: List[TypeMessageEntity]
+
+    def offset_entities(self, offset: int) -> 'TelegramMessage':
+        def apply_offset(entity: TypeMessageEntity, inner_offset: int
+                         ) -> Optional[TypeMessageEntity]:
+            entity = Entity.copy(entity)
+            entity.offset += inner_offset
+            if entity.offset < 0:
+                entity.offset = 0
+            elif entity.offset > len(self.text):
+                return None
+            elif entity.offset + entity.length > len(self.text):
+                entity.length = len(self.text) - entity.offset
+            return entity
+
+        self.entities = [apply_offset(entity, offset) for entity in self.entities if entity]
+        self.entities = [x for x in self.entities if x is not None]
+        return self
+
+    def append(self, *args: Union[str, 'TelegramMessage']) -> 'TelegramMessage':
+        for msg in args:
+            if isinstance(msg, str):
+                msg = TelegramMessage(text=msg)
+            self.entities += Entity.adjust(msg.entities, offset_diff(len(self.text)))
+            self.text += msg.text
+        return self
+
+    def prepend(self, *args: Union[str, 'TelegramMessage']) -> 'TelegramMessage':
+        for msg in args:
+            if isinstance(msg, str):
+                msg = TelegramMessage(text=msg)
+            self.entities = msg.entities + Entity.adjust(self.entities, offset_diff(len(msg.text)))
+            self.text = msg.text + self.text
+        return self
+
+    def format(self, entity_type: Type[TypeMessageEntity], offset: int = None, length: int = None,
+               **kwargs) -> 'TelegramMessage':
+        self.entities.append(entity_type(offset=offset or 0,
+                                         length=length if length is not None else len(self.text),
+                                         **kwargs))
+        return self
+
+    def concat(self, *args: Union[str, 'TelegramMessage']) -> 'TelegramMessage':
+        return TelegramMessage().append(self, *args)
+
+    def trim(self) -> 'TelegramMessage':
+        orig_len = len(self.text)
+        self.text = self.text.lstrip()
+        diff = orig_len - len(self.text)
+        self.text = self.text.rstrip()
+        self.offset_entities(-diff)
+        return self
+
+    def split(self, separator, max_items: int = 0) -> List['TelegramMessage']:
+        text_parts = self.text.split(separator, max_items - 1)
+        output = []  # type: List[TelegramMessage]
+
+        offset = 0
+        for part in text_parts:
+            msg = TelegramMessage(part)
+            for entity in self.entities:
+                start_in_range = len(part) > entity.offset - offset >= 0
+                end_in_range = len(part) >= entity.offset - offset + entity.length > 0
+                if start_in_range and end_in_range:
+                    msg.entities.append(Entity.adjust(entity, offset_diff(-offset)))
+            output.append(msg)
+
+            offset += len(part)
+            offset += len(separator)
+
+        return output
+
+    @staticmethod
+    def join(items: Sequence[Union[str, 'TelegramMessage']],
+             separator: str = " ") -> 'TelegramMessage':
+        main = TelegramMessage()
+        for msg in items:
+            if isinstance(msg, str):
+                msg = TelegramMessage(text=msg)
+            main.entities += Entity.adjust(msg.entities, offset_diff(len(main.text)))
+            main.text += msg.text + separator
+        if len(separator) > 0:
+            main.text = main.text[:-len(separator)]
+        return main

mautrix_telegram/formatter/from_telegram.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -14,47 +14,49 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Dict, List, Optional, Tuple, TYPE_CHECKING
 from html import escape
-from typing import Optional, List, Tuple
-
-try:
-    from lxml.html.diff import htmldiff
-except ImportError:
-    htmldiff = None  # type: function
 import logging
 import re
 
-from telethon.tl.types import (MessageEntityMention, MessageEntityMentionName,
-                               MessageEntityEmail, MessageEntityUrl, MessageEntityTextUrl,
-                               MessageEntityBold, MessageEntityItalic, MessageEntityCode,
-                               MessageEntityPre, MessageEntityBotCommand, Message, PeerChannel,
-                               MessageEntityHashtag, TypeMessageEntity, MessageFwdHeader, PeerUser)
+from telethon.tl.types import (MessageEntityMention, MessageEntityMentionName, MessageEntityUrl,
+                               MessageEntityEmail, MessageEntityTextUrl, MessageEntityBold,
+                               MessageEntityItalic, MessageEntityCode, MessageEntityPre,
+                               MessageEntityBotCommand, MessageEntityHashtag, MessageEntityCashtag,
+                               MessageEntityPhone, TypeMessageEntity, Message, PeerChannel,
+                               MessageEntityBlockquote, MessageEntityStrike, MessageFwdHeader,
+                               MessageEntityUnderline, PeerUser)
 
 from mautrix_appservice import MatrixRequestError
 from mautrix_appservice.intent_api import IntentAPI
 
 from .. import user as u, puppet as pu, portal as po
-from ..context import Context
+from ..types import TelegramID
 from ..db import Message as DBMessage
 from .util import (add_surrogates, remove_surrogates, trim_reply_fallback_html,
-                   trim_reply_fallback_text, unicode_to_html)
+                   trim_reply_fallback_text)
 
-log = logging.getLogger("mau.fmt.tg")
-should_highlight_edits = False
+if TYPE_CHECKING:
+    from ..abstract_user import AbstractUser
+
+log = logging.getLogger("mau.fmt.tg")  # type: logging.Logger
 
 
-def telegram_reply_to_matrix(evt: Message, source: u.User) -> dict:
+def telegram_reply_to_matrix(evt: Message, source: 'AbstractUser') -> Dict:
     if evt.reply_to_msg_id:
         space = (evt.to_id.channel_id
                  if isinstance(evt, Message) and isinstance(evt.to_id, PeerChannel)
                  else source.tgid)
-        msg = DBMessage.query.get((evt.reply_to_msg_id, space))
+        msg = DBMessage.get_one_by_tgid(evt.reply_to_msg_id, space)
         if msg:
             return {
                 "m.in_reply_to": {
                     "event_id": msg.mxid,
                     "room_id": msg.mx_room,
-                }
+                },
+                "rel_type": "m.reference",
+                "event_id": msg.mxid,
+                "room_id": msg.mx_room,
             }
     return {}
 
@@ -65,21 +67,37 @@ async def _add_forward_header(source, text: str, html: Optional[str],
         html = escape(text)
     fwd_from_html, fwd_from_text = None, None
     if fwd_from.from_id:
-        user = u.User.get_by_tgid(fwd_from.from_id)
+        user = u.User.get_by_tgid(TelegramID(fwd_from.from_id))
         if user:
             fwd_from_text = user.displayname or user.mxid
-            fwd_from_html = f"<a href='https://matrix.to/#/{user.mxid}'>{fwd_from_text}</a>"
+            fwd_from_html = (f"<a href='https://matrix.to/#/{user.mxid}'>"
+                             f"{escape(fwd_from_text)}</a>")
 
         if not fwd_from_text:
-            puppet = pu.Puppet.get(fwd_from.from_id, create=False)
+            puppet = pu.Puppet.get(TelegramID(fwd_from.from_id), create=False)
             if puppet and puppet.displayname:
                 fwd_from_text = puppet.displayname or puppet.mxid
-                fwd_from_html = f"<a href='https://matrix.to/#/{puppet.mxid}'>{fwd_from_text}</a>"
+                fwd_from_html = (f"<a href='https://matrix.to/#/{puppet.mxid}'>"
+                                 f"{escape(fwd_from_text)}</a>")
 
         if not fwd_from_text:
             user = await source.client.get_entity(PeerUser(fwd_from.from_id))
             if user:
-                fwd_from_text = pu.Puppet.get_displayname(user, format=False)
+                fwd_from_text = pu.Puppet.get_displayname(user, False)
+                fwd_from_html = f"<b>{escape(fwd_from_text)}</b>"
+    else:
+        portal = po.Portal.get_by_tgid(TelegramID(fwd_from.channel_id))
+        if portal:
+            fwd_from_text = portal.title
+            if portal.alias:
+                fwd_from_html = (f"<a href='https://matrix.to/#/{portal.alias}'>"
+                                 f"{escape(fwd_from_text)}</a>")
+            else:
+                fwd_from_html = f"<b>{escape(fwd_from_text)}</b>"
+        else:
+            channel = await source.client.get_entity(PeerChannel(fwd_from.channel_id))
+            if channel:
+                fwd_from_text = channel.title
                 fwd_from_html = f"<b>{fwd_from_text}</b>"
 
     if not fwd_from_text:
@@ -96,31 +114,19 @@ async def _add_forward_header(source, text: str, html: Optional[str],
     return text, html
 
 
-def highlight_edits(new_html: str, old_html: str) -> str:
-    # Don't include `Edit:` text in diff.
-    if old_html.startswith("<u>Edit:</u> "):
-        old_html = old_html[len("<u>Edit:</u> "):]
-
-    # Generate diff with lxml
-    new_html = htmldiff(old_html, new_html)
-
-    # Replace <ins> with <u> since Riot doesn't allow <ins>
-    new_html = new_html.replace("<ins>", "<u>").replace("</ins>", "</u>")
-    # Remove <del>s since we just want to hide deletions.
-    new_html = re.sub("<del>.+?</del>", "", new_html)
-    return new_html
-
-
-async def _add_reply_header(source: u.User, text: str, html: str, evt: Message, relates_to: dict,
-                            main_intent: IntentAPI, is_edit: bool) -> Tuple[str, str]:
+async def _add_reply_header(source: "AbstractUser", text: str, html: str, evt: Message,
+                            relates_to: Dict, main_intent: IntentAPI) -> Tuple[str, str]:
     space = (evt.to_id.channel_id
              if isinstance(evt, Message) and isinstance(evt.to_id, PeerChannel)
              else source.tgid)
 
-    msg = DBMessage.query.get((evt.reply_to_msg_id, space))
+    msg = DBMessage.get_one_by_tgid(evt.reply_to_msg_id, space)
     if not msg:
         return text, html
 
+    relates_to["rel_type"] = "m.reference"
+    relates_to["event_id"] = msg.mxid
+    relates_to["room_id"] = msg.mx_room
     relates_to["m.in_reply_to"] = {
         "event_id": msg.mxid,
         "room_id": msg.mx_room,
@@ -139,24 +145,17 @@ async def _add_reply_header(source: u.User, text: str, html: str, evt: Message,
 
         puppet = pu.Puppet.get_by_mxid(r_sender, create=False)
         r_displayname = puppet.displayname if puppet else r_sender
-        r_sender_link = f"<a href='https://matrix.to/#/{r_sender}'>{r_displayname}</a>"
-
-        if is_edit and should_highlight_edits:
-            html = highlight_edits(html or escape(text), r_html_body)
-    except (ValueError, KeyError, MatrixRequestError) as e:
+        r_sender_link = f"<a href='https://matrix.to/#/{r_sender}'>{escape(r_displayname)}</a>"
+    except (ValueError, KeyError, MatrixRequestError):
         r_sender_link = "unknown user"
         r_displayname = "unknown user"
         r_text_body = "Failed to fetch message"
         r_html_body = "<em>Failed to fetch message</em>"
 
-    if is_edit:
-        html = f"<u>Edit:</u> {html or escape(text)}"
-        text = f"Edit: {text}"
-
-    r_keyword = "In reply to" if not is_edit else "Edit to"
-    r_msg_link = f"<a href='https://matrix.to/#/{msg.mx_room}/{msg.mxid}'>{r_keyword}</a>"
-    html = (f"<mx-reply><blockquote>{r_msg_link} {r_sender_link}\n{r_html_body}</blockquote></mx-reply>"
-            + (html or escape(text)))
+    r_msg_link = f"<a href='https://matrix.to/#/{msg.mx_room}/{msg.mxid}'>In reply to</a>"
+    html = (
+        f"<mx-reply><blockquote>{r_msg_link} {r_sender_link}\n{r_html_body}</blockquote></mx-reply>"
+        + (html or escape(text)))
 
     lines = r_text_body.strip().split("\n")
     text_with_quote = f"> <{r_displayname}> {lines.pop(0)}"
@@ -168,12 +167,16 @@ async def _add_reply_header(source: u.User, text: str, html: str, evt: Message,
     return text_with_quote, html
 
 
-async def telegram_to_matrix(evt: Message, source: u.User, main_intent: Optional[IntentAPI] = None,
-                             is_edit: bool = False, prefix_text: Optional[str] = None,
-                             prefix_html: Optional[str] = None) -> Tuple[str, str, dict]:
-    text = add_surrogates(evt.message)
-    html = _telegram_entities_to_matrix_catch(text, evt.entities) if evt.entities else None
-    relates_to = {}
+async def telegram_to_matrix(evt: Message, source: "AbstractUser",
+                             main_intent: Optional[IntentAPI] = None,
+                             prefix_text: Optional[str] = None, prefix_html: Optional[str] = None,
+                             override_text: str = None,
+                             override_entities: List[TypeMessageEntity] = None,
+                             no_reply_fallback: bool = False) -> Tuple[str, str, Dict]:
+    text = add_surrogates(override_text or evt.message)
+    entities = override_entities or evt.entities
+    html = _telegram_entities_to_matrix_catch(text, entities) if entities else None
+    relates_to = {}  # type: Dict
 
     if prefix_html:
         html = prefix_html + (html or escape(text))
@@ -183,9 +186,8 @@ async def telegram_to_matrix(evt: Message, source: u.User, main_intent: Optional
     if evt.fwd_from:
         text, html = await _add_forward_header(source, text, html, evt.fwd_from)
 
-    if evt.reply_to_msg_id:
-        text, html = await _add_reply_header(source, text, html, evt, relates_to, main_intent,
-                                             is_edit)
+    if evt.reply_to_msg_id and not no_reply_fallback:
+        text, html = await _add_reply_header(source, text, html, evt, relates_to, main_intent)
 
     if isinstance(evt, Message) and evt.post and evt.post_author:
         if not html:
@@ -193,9 +195,6 @@ async def telegram_to_matrix(evt: Message, source: u.User, main_intent: Optional
         text += f"\n- {evt.post_author}"
         html += f"<br/><i>- <u>{evt.post_author}</u></i>"
 
-    html = unicode_to_html(text, html, "\u0336", "del")
-    html = unicode_to_html(text, html, "\u0332", "u")
-
     if html:
         html = html.replace("\n", "<br/>")
 
@@ -210,48 +209,65 @@ def _telegram_entities_to_matrix_catch(text: str, entities: List[TypeMessageEnti
                       "message=%s\n"
                       "entities=%s",
                       text, entities)
+    return "[failed conversion in _telegram_entities_to_matrix]"
 
 
-def _telegram_entities_to_matrix(text: str, entities: List[TypeMessageEntity]) -> str:
+def _telegram_entities_to_matrix(text: str, entities: List[TypeMessageEntity],
+                                 offset: int = 0, length: int = None) -> str:
     if not entities:
-        return text
+        return escape(text)
+    if length is None:
+        length = len(text)
     html = []
     last_offset = 0
-    for entity in entities:
-        if entity.offset > last_offset:
-            html.append(escape(text[last_offset:entity.offset]))
-        elif entity.offset < last_offset:
+    for i, entity in enumerate(entities):
+        if entity.offset > offset + length:
+            break
+        relative_offset = entity.offset - offset
+        if relative_offset > last_offset:
+            html.append(escape(text[last_offset:relative_offset]))
+        elif relative_offset < last_offset:
             continue
 
         skip_entity = False
-        entity_text = escape(text[entity.offset:entity.offset + entity.length])
+        entity_text = _telegram_entities_to_matrix(
+            text=text[relative_offset:relative_offset + entity.length],
+            entities=entities[i + 1:], offset=entity.offset, length=entity.length)
         entity_type = type(entity)
 
         if entity_type == MessageEntityBold:
             html.append(f"<strong>{entity_text}</strong>")
         elif entity_type == MessageEntityItalic:
             html.append(f"<em>{entity_text}</em>")
+        elif entity_type == MessageEntityUnderline:
+            html.append(f"<u>{entity_text}</u>")
+        elif entity_type == MessageEntityStrike:
+            html.append(f"<del>{entity_text}</del>")
+        elif entity_type == MessageEntityBlockquote:
+            html.append(f"<blockquote>{entity_text}</blockquote>")
         elif entity_type == MessageEntityCode:
-            html.append(f"<code>{entity_text}</code>")
+            html.append(f"<pre><code>{entity_text}</code></pre>"
+                        if "\n" in entity_text
+                        else f"<code>{entity_text}</code>")
         elif entity_type == MessageEntityPre:
             skip_entity = _parse_pre(html, entity_text, entity.language)
         elif entity_type == MessageEntityMention:
             skip_entity = _parse_mention(html, entity_text)
         elif entity_type == MessageEntityMentionName:
-            skip_entity = _parse_name_mention(html, entity_text, entity.user_id)
+            skip_entity = _parse_name_mention(html, entity_text, TelegramID(entity.user_id))
         elif entity_type == MessageEntityEmail:
             html.append(f"<a href='mailto:{entity_text}'>{entity_text}</a>")
-        elif entity_type in {MessageEntityTextUrl, MessageEntityUrl}:
+        elif entity_type in (MessageEntityTextUrl, MessageEntityUrl):
             skip_entity = _parse_url(html, entity_text,
                                      entity.url if entity_type == MessageEntityTextUrl else None)
         elif entity_type == MessageEntityBotCommand:
             html.append(f"<font color='blue'>!{entity_text[1:]}</font>")
-        elif entity_type == MessageEntityHashtag:
+        elif entity_type in (MessageEntityHashtag, MessageEntityCashtag, MessageEntityPhone):
             html.append(f"<font color='blue'>{entity_text}</font>")
         else:
             skip_entity = True
-        last_offset = entity.offset + (0 if skip_entity else entity.length)
-    html.append(text[last_offset:])
+        last_offset = relative_offset + (0 if skip_entity else entity.length)
+    html.append(escape(text[last_offset:]))
 
     return "".join(html)
 
@@ -283,7 +299,7 @@ def _parse_mention(html: List[str], entity_text: str) -> bool:
     return False
 
 
-def _parse_name_mention(html: List[str], entity_text: str, user_id: int) -> bool:
+def _parse_name_mention(html: List[str], entity_text: str, user_id: TelegramID) -> bool:
     user = u.User.get_by_tgid(user_id)
     if user:
         mxid = user.mxid
@@ -308,19 +324,14 @@ def _parse_url(html: List[str], entity_text: str, url: str) -> bool:
 
     message_link_match = message_link_regex.match(url)
     if message_link_match:
-        group, msgid = message_link_match.groups()
-        msgid = int(msgid)
+        group, msgid_str = message_link_match.groups()
+        msgid = int(msgid_str)
 
         portal = po.Portal.find_by_username(group)
         if portal:
-            message = DBMessage.query.get((msgid, portal.tgid))
+            message = DBMessage.get_one_by_tgid(TelegramID(msgid), portal.tgid)
             if message:
                 url = f"https://matrix.to/#/{portal.mxid}/{message.mxid}"
 
     html.append(f"<a href='{url}'>{entity_text}</a>")
     return False
-
-
-def init_tg(context: Context):
-    global should_highlight_edits
-    should_highlight_edits = htmldiff and context.config["bridge.highlight_edits"]

mautrix_telegram/formatter/util.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -14,15 +14,15 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Optional, Pattern
 from html import escape
-from typing import Optional
 import struct
 import re
 
 
 # add_surrogates and remove_surrogates are unicode surrogate utility functions from Telethon.
 # Licensed under the MIT license.
-# https://github.com/LonamiWebs/Telethon/blob/master/telethon/extensions/markdown.py
+# https://github.com/LonamiWebs/Telethon/blob/7cce7aa3e4c6c7019a55530391b1761d33e5a04e/telethon/helpers.py
 def add_surrogates(text: Optional[str]) -> Optional[str]:
     if text is None:
         return None
@@ -36,6 +36,9 @@ def remove_surrogates(text: Optional[str]) -> Optional[str]:
     return text.encode("utf-16", "surrogatepass").decode("utf-16")
 
 
+# trim_reply_fallback_text, html_reply_fallback_regex and trim_reply_fallback_html are Matrix
+# reply fallback utility functions.
+# You may copy and use them under any OSI-approved license.
 def trim_reply_fallback_text(text: str) -> str:
     if not text.startswith("> ") or "\n" not in text:
         return text
@@ -47,40 +50,8 @@ def trim_reply_fallback_text(text: str) -> str:
 
 html_reply_fallback_regex = re.compile("^<mx-reply>"
                                        r"[\s\S]+?"
-                                       "</mx-reply>")
+                                       "</mx-reply>")  # type: Pattern
 
 
 def trim_reply_fallback_html(html: str) -> str:
     return html_reply_fallback_regex.sub("", html)
-
-
-def unicode_to_html(text: str, html: str, ctrl: str, tag: str) -> str:
-    if ctrl not in text:
-        return html
-    if not html:
-        html = escape(text)
-    tag_start = f"<{tag}>"
-    tag_end = f"</{tag}>"
-    characters = html.split(ctrl)
-    html = ""
-    in_tag = False
-    for char in characters:
-        if not in_tag:
-            if len(char) > 1:
-                html += char[0:-1]
-                char = char[-1]
-            html += tag_start
-            in_tag = True
-            html += char
-        else:
-            if len(char) > 1:
-                html += tag_end
-                in_tag = False
-            html += char
-    if in_tag:
-        html += tag_end
-    return html
-
-
-def html_to_unicode(text: str, ctrl: str) -> str:
-    return ctrl.join(text) + ctrl

mautrix_telegram/matrix.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -14,177 +14,253 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Dict, List, Match, Optional, Set, Tuple, TYPE_CHECKING
 import logging
+import asyncio
+import time
+import re
 
-from mautrix_appservice import MatrixRequestError
+from mautrix_appservice import MatrixRequestError, IntentError
 
-from .user import User
-from .portal import Portal
-from .puppet import Puppet
-from .commands import CommandHandler
+from .types import MatrixEvent, MatrixEventID, MatrixRoomID, MatrixUserID
+from . import user as u, portal as po, puppet as pu, commands as com
+
+if TYPE_CHECKING:
+    from .context import Context
+
+try:
+    from prometheus_client import Histogram
+
+    EVENT_TIME = Histogram("matrix_event", "Time spent processing Matrix events",
+                           ["event_type"])
+except ImportError:
+    Histogram = None
+    EVENT_TIME = None
 
 
 class MatrixHandler:
-    log = logging.getLogger("mau.mx")
+    log = logging.getLogger("mau.mx")  # type: logging.Logger
 
-    def __init__(self, context):
-        self.az, self.db, self.config, _, self.tgbot = context
-        self.commands = CommandHandler(context)
+    def __init__(self, context: 'Context') -> None:
+        self.az, self.config, _, self.tgbot = context.core
+        self.commands = com.CommandProcessor(context)  # type: com.CommandProcessor
+        self.previously_typing = []  # type: List[MatrixUserID]
 
         self.az.matrix_event_handler(self.handle_event)
 
-    async def init_as_bot(self):
-        await self.az.intent.set_display_name(
-            self.config.get("appservice.bot_displayname", "Telegram bridge bot"))
+    async def init_as_bot(self) -> None:
+        displayname = self.config["appservice.bot_displayname"]
+        if displayname:
+            try:
+                await self.az.intent.set_display_name(
+                    displayname if displayname != "remove" else "")
+            except asyncio.TimeoutError:
+                self.log.exception("TimeoutError when trying to set displayname")
 
-    async def handle_puppet_invite(self, room, puppet, inviter):
-        self.log.debug(f"{inviter} invited puppet for {puppet.tgid} to {room}")
-        if not inviter.logged_in:
-            await puppet.intent.error_and_leave(
-                room, text="Please log in before inviting Telegram puppets.")
+        avatar = self.config["appservice.bot_avatar"]
+        if avatar:
+            try:
+                await self.az.intent.set_avatar(avatar if avatar != "remove" else "")
+            except asyncio.TimeoutError:
+                self.log.exception("TimeoutError when trying to set avatar")
+
+    async def handle_puppet_invite(self, room_id: MatrixRoomID, puppet: pu.Puppet, inviter: u.User
+                                   ) -> None:
+        intent = puppet.default_mxid_intent
+        self.log.debug(f"{inviter} invited puppet for {puppet.tgid} to {room_id}")
+        if not await inviter.is_logged_in():
+            await intent.error_and_leave(
+                room_id, text="Please log in before inviting Telegram puppets.")
             return
-        portal = Portal.get_by_mxid(room)
+        portal = po.Portal.get_by_mxid(room_id)
         if portal:
             if portal.peer_type == "user":
-                await puppet.intent.error_and_leave(
-                    room, text="You can not invite additional users to private chats.")
+                await intent.error_and_leave(
+                    room_id, text="You can not invite additional users to private chats.")
                 return
             await portal.invite_telegram(inviter, puppet)
-            await puppet.intent.join_room(room)
+            await intent.join_room(room_id)
             return
         try:
-            members = await self.az.intent.get_room_members(room)
+            members = await self.az.intent.get_room_members(room_id)
         except MatrixRequestError:
             members = []
         if self.az.bot_mxid not in members:
             if len(members) > 1:
-                await puppet.intent.error_and_leave(room, text=None, html=(
+                await intent.error_and_leave(room_id, text=None, html=(
                     f"Please invite "
                     f"<a href='https://matrix.to/#/{self.az.bot_mxid}'>the bridge bot</a> "
                     f"first if you want to create a Telegram chat."))
                 return
 
-            await puppet.intent.join_room(room)
-            portal = Portal.get_by_tgid(puppet.tgid, inviter.tgid, "user")
+            await intent.join_room(room_id)
+            portal = po.Portal.get_by_tgid(puppet.tgid, inviter.tgid, "user")
+            # TODO: if portal is None:
             if portal.mxid:
                 try:
-                    await puppet.intent.invite(portal.mxid, inviter.mxid)
-                    await puppet.intent.send_notice(room, text=None, html=(
+                    await intent.invite(portal.mxid, inviter.mxid)
+                    await intent.send_notice(room_id, text=None, html=(
                         "You already have a private chat with me: "
                         f"<a href='https://matrix.to/#/{portal.mxid}'>"
                         "Link to room"
                         "</a>"))
-                    await puppet.intent.leave_room(room)
+                    await intent.leave_room(room_id)
                     return
                 except MatrixRequestError:
                     pass
-            portal.mxid = room
+            portal.mxid = room_id
             portal.save()
             inviter.register_portal(portal)
-            await puppet.intent.send_notice(room, "Portal to private chat created.")
+            await intent.send_notice(room_id, "Portal to private chat created.")
         else:
-            await puppet.intent.join_room(room)
-            await puppet.intent.send_notice(room, "This puppet will remain inactive until a "
-                                                  "Telegram chat is created for this room.")
+            await intent.join_room(room_id)
+            await intent.send_notice(room_id, "This puppet will remain inactive until a "
+                                              "Telegram chat is created for this room.")
 
-    async def handle_invite(self, room, user, inviter):
-        self.log.debug(f"{inviter} invited {user} to {room}")
-        inviter = await User.get_by_mxid(inviter).ensure_started()
-        if user == self.az.bot_mxid:
-            await self.az.intent.join_room(room)
-            if not inviter.whitelisted:
-                await self.az.intent.send_notice(
-                    room, text=None,
-                    html="You are not whitelisted to use this bridge.<br/><br/>"
-                         "If you are the owner of this bridge, see the "
-                         "<code>bridge.permissions</code> section in your config file.")
-                await self.az.intent.leave_room(room)
-            return
+    async def accept_bot_invite(self, room_id: MatrixRoomID, inviter: u.User) -> None:
+        tries = 0
+        while tries < 5:
+            try:
+                await self.az.intent.join_room(room_id)
+                break
+            except (IntentError, MatrixRequestError):
+                tries += 1
+                wait_for_seconds = (tries + 1) * 10
+                if tries < 5:
+                    self.log.exception(f"Failed to join room {room_id} with bridge bot, "
+                                       f"retrying in {wait_for_seconds} seconds...")
+                    await asyncio.sleep(wait_for_seconds)
+                else:
+                    self.log.exception("Failed to join room {room}, giving up.")
+                    return
+
+        if not inviter.whitelisted:
+            await self.az.intent.send_notice(
+                room_id,
+                text="You are not whitelisted to use this bridge.\n\n"
+                     "If you are the owner of this bridge, see the "
+                     "`bridge.permissions` section in your config file.",
+                html="<p>You are not whitelisted to use this bridge.</p>"
+                     "<p>If you are the owner of this bridge, see the "
+                     "<code>bridge.permissions</code> section in your config file.</p>")
+            await self.az.intent.leave_room(room_id)
+
+        try:
+            is_management = len(await self.az.intent.get_room_members(room_id)) == 2
+        except MatrixRequestError:
+            is_management = False
+        cmd_prefix = self.commands.command_prefix
+        text = html = "Hello, I'm a Telegram bridge bot. "
+        if is_management and inviter.puppet_whitelisted and not await inviter.is_logged_in():
+            text += f"Use `{cmd_prefix} help` for help or `{cmd_prefix} login` to log in."
+            html += (f"Use <code>{cmd_prefix} help</code> for help"
+                     f" or <code>{cmd_prefix} login</code> to log in.")
+            pass
+        else:
+            text += f"Use `{cmd_prefix} help` for help."
+            html += f"Use <code>{cmd_prefix} help</code> for help."
+        await self.az.intent.send_notice(room_id, text=text, html=html)
+
+    async def handle_invite(self, room_id: MatrixRoomID, user_id: MatrixUserID,
+                            inviter_mxid: MatrixUserID) -> None:
+        self.log.debug(f"{inviter_mxid} invited {user_id} to {room_id}")
+        inviter = u.User.get_by_mxid(inviter_mxid)
+        if inviter is None:
+            self.log.exception("Failed to find user with Matrix ID {inviter_mxid}")
+        await inviter.ensure_started()
+        if user_id == self.az.bot_mxid:
+            return await self.accept_bot_invite(room_id, inviter)
         elif not inviter.whitelisted:
             return
 
-        puppet = Puppet.get_by_mxid(user)
+        puppet = pu.Puppet.get_by_mxid(user_id)
         if puppet:
-            await self.handle_puppet_invite(room, puppet, inviter)
+            await self.handle_puppet_invite(room_id, puppet, inviter)
             return
 
-        user = User.get_by_mxid(user, create=False)
+        user = u.User.get_by_mxid(user_id, create=False)
         if not user:
             return
         await user.ensure_started()
-        portal = Portal.get_by_mxid(room)
-        if user and user.has_full_access and portal:
+        portal = po.Portal.get_by_mxid(room_id)
+        if user and await user.has_full_access(allow_bot=True) and portal:
             await portal.invite_telegram(inviter, user)
             return
 
         # The rest can probably be ignored
 
-    async def handle_join(self, room, user, event_id):
-        user = await User.get_by_mxid(user).ensure_started()
+    async def handle_join(self, room_id: MatrixRoomID, user_id: MatrixUserID,
+                          event_id: MatrixEventID) -> None:
+        user = await u.User.get_by_mxid(user_id).ensure_started()
 
-        portal = Portal.get_by_mxid(room)
+        portal = po.Portal.get_by_mxid(room_id)
         if not portal:
             return
 
         if not user.relaybot_whitelisted:
-            await portal.main_intent.kick(room, user.mxid,
+            await portal.main_intent.kick(room_id, user.mxid,
                                           "You are not whitelisted on this Telegram bridge.")
             return
-        elif not user.logged_in and not portal.has_bot:
-            await portal.main_intent.kick(room, user.mxid,
+        elif not await user.is_logged_in() and not portal.has_bot:
+            await portal.main_intent.kick(room_id, user.mxid,
                                           "This chat does not have a bot relaying "
                                           "messages for unauthenticated users.")
             return
 
-        self.log.debug(f"{user} joined {room}")
-        if user.logged_in or portal.has_bot:
+        self.log.debug(f"{user} joined {room_id}")
+        if await user.is_logged_in() or portal.has_bot:
             await portal.join_matrix(user, event_id)
 
-    async def handle_part(self, room, user, sender, event_id):
-        self.log.debug(f"{user} left {room}")
+    async def handle_part(self, room_id: MatrixRoomID, user_id: MatrixUserID,
+                          sender_mxid: MatrixUserID, event_id: MatrixEventID) -> None:
+        self.log.debug(f"{user_id} left {room_id}")
 
-        sender = User.get_by_mxid(sender, create=False)
+        sender = u.User.get_by_mxid(sender_mxid, create=False)
         if not sender:
             return
         await sender.ensure_started()
 
-        portal = Portal.get_by_mxid(room)
+        portal = po.Portal.get_by_mxid(room_id)
         if not portal:
             return
 
-        puppet = Puppet.get_by_mxid(user)
-        if sender and puppet:
-            await portal.leave_matrix(puppet, sender)
+        puppet = pu.Puppet.get_by_mxid(user_id)
+        if puppet:
+            if sender:
+                await portal.kick_matrix(puppet, sender)
+            return
 
-        user = User.get_by_mxid(user, create=False)
+        user = u.User.get_by_mxid(user_id, create=False)
         if not user:
             return
         await user.ensure_started()
-        if user.logged_in or portal.has_bot:
+        if await user.is_logged_in() or portal.has_bot:
             await portal.leave_matrix(user, sender, event_id)
 
-    def is_command(self, message):
+    def is_command(self, message: Dict) -> Tuple[bool, str]:
         text = message.get("body", "")
         prefix = self.config["bridge.command_prefix"]
         is_command = text.startswith(prefix)
         if is_command:
-            text = text[len(prefix) + 1:]
+            text = text[len(prefix) + 1:].lstrip()
         return is_command, text
 
-    async def handle_message(self, room, sender, message, event_id):
-        self.log.debug(f"{sender} sent {message} to ${room}")
-
+    async def handle_message(self, room: MatrixRoomID, sender_id: MatrixUserID, message: Dict,
+                             event_id: MatrixEventID) -> None:
         is_command, text = self.is_command(message)
-        sender = await User.get_by_mxid(sender).ensure_started()
+        sender = await u.User.get_by_mxid(sender_id).ensure_started()
         if not sender.relaybot_whitelisted:
+            self.log.debug(f"Ignoring message \"{message}\" from {sender} to {room}:"
+                           " User is not whitelisted.")
             return
+        self.log.debug(f"Received Matrix event \"{message}\" from {sender} in {room}")
 
-        portal = Portal.get_by_mxid(room)
-        if not is_command and portal and (sender.logged_in or portal.has_bot):
+        portal = po.Portal.get_by_mxid(room)
+        if not is_command and portal and (await sender.is_logged_in() or portal.has_bot):
             await portal.handle_matrix_message(sender, message, event_id)
             return
 
-        if not sender.whitelisted or message["msgtype"] != "m.text":
+        if not sender.whitelisted or message.get("msgtype", "m.unknown") != "m.text":
             return
 
         try:
@@ -201,90 +277,201 @@ class MatrixHandler:
                 # Not enough values to unpack, i.e. no arguments
                 command = text
                 args = []
-            await self.commands.handle(room, sender, command, args, is_management,
+            await self.commands.handle(room, event_id, sender, command, args, is_management,
                                        is_portal=portal is not None)
 
-    async def handle_redaction(self, room, sender, event_id):
-        sender = await User.get_by_mxid(sender).ensure_started()
+    @staticmethod
+    async def handle_redaction(room_id: MatrixRoomID, sender_mxid: MatrixUserID,
+                               event_id: MatrixEventID) -> None:
+        sender = await u.User.get_by_mxid(sender_mxid).ensure_started()
         if not sender.relaybot_whitelisted:
             return
 
-        portal = Portal.get_by_mxid(room)
+        portal = po.Portal.get_by_mxid(room_id)
         if not portal:
             return
 
         await portal.handle_matrix_deletion(sender, event_id)
 
-    async def handle_power_levels(self, room, sender, new, old):
-        portal = Portal.get_by_mxid(room)
-        sender = await User.get_by_mxid(sender).ensure_started()
-        if sender.has_full_access and portal:
+    @staticmethod
+    async def handle_power_levels(room_id: MatrixRoomID, sender_mxid: MatrixUserID,
+                                  new: Dict, old: Dict) -> None:
+        portal = po.Portal.get_by_mxid(room_id)
+        sender = await u.User.get_by_mxid(sender_mxid).ensure_started()
+        if await sender.has_full_access(allow_bot=True) and portal:
             await portal.handle_matrix_power_levels(sender, new["users"], old["users"])
 
-    async def handle_room_meta(self, type, room, sender, content):
-        portal = Portal.get_by_mxid(room)
-        sender = await User.get_by_mxid(sender).ensure_started()
-        if sender.has_full_access and portal:
+    @staticmethod
+    async def handle_room_meta(evt_type: str, room_id: MatrixRoomID, sender_mxid: MatrixUserID,
+                               content: dict) -> None:
+        portal = po.Portal.get_by_mxid(room_id)
+        sender = await u.User.get_by_mxid(sender_mxid).ensure_started()
+        if await sender.has_full_access(allow_bot=True) and portal:
             handler, content_key = {
                 "m.room.name": (portal.handle_matrix_title, "name"),
                 "m.room.topic": (portal.handle_matrix_about, "topic"),
                 "m.room.avatar": (portal.handle_matrix_avatar, "url"),
-            }[type]
+            }[evt_type]
             if content_key not in content:
                 return
             await handler(sender, content[content_key])
 
-    async def handle_room_pin(self, room, sender, new_events, old_events):
-        portal = Portal.get_by_mxid(room)
-        sender = await User.get_by_mxid(sender).ensure_started()
-        if sender.has_full_access and portal:
+    @staticmethod
+    async def handle_room_pin(room_id: MatrixRoomID, sender_mxid: MatrixUserID,
+                              new_events: Set[str], old_events: Set[str]) -> None:
+        portal = po.Portal.get_by_mxid(room_id)
+        sender = await u.User.get_by_mxid(sender_mxid).ensure_started()
+        if await sender.has_full_access(allow_bot=True) and portal:
             events = new_events - old_events
             if len(events) > 0:
                 # New event pinned, set that as pinned in Telegram.
-                await portal.handle_matrix_pin(sender, events.pop())
+                await portal.handle_matrix_pin(sender, MatrixEventID(events.pop()))
             elif len(new_events) == 0:
                 # All pinned events removed, remove pinned event in Telegram.
                 await portal.handle_matrix_pin(sender, None)
 
-    def filter_matrix_event(self, event):
-        return (event["sender"] == self.az.bot_mxid
-                or Puppet.get_id_from_mxid(event["sender"]) is not None)
+    @staticmethod
+    async def handle_room_upgrade(room_id: MatrixRoomID, new_room_id: MatrixRoomID) -> None:
+        portal = po.Portal.get_by_mxid(room_id)
+        if portal:
+            await portal.handle_matrix_upgrade(new_room_id)
 
-    async def handle_event(self, evt):
+    @staticmethod
+    async def handle_name_change(room_id: MatrixRoomID, user_id: MatrixUserID, displayname: str,
+                                 prev_displayname: str, event_id: MatrixEventID) -> None:
+        portal = po.Portal.get_by_mxid(room_id)
+        if not portal or not portal.has_bot:
+            return
+
+        user = await u.User.get_by_mxid(user_id).ensure_started()
+        if await user.needs_relaybot(portal):
+            await portal.name_change_matrix(user, displayname, prev_displayname, event_id)
+
+    @staticmethod
+    def parse_read_receipts(content: Dict) -> Dict[MatrixUserID, MatrixEventID]:
+        return {user_id: event_id
+                for event_id, receipts in content.items()
+                for user_id in receipts.get("m.read", {})}
+
+    @staticmethod
+    async def handle_read_receipts(room_id: MatrixRoomID,
+                                   receipts: Dict[MatrixUserID, MatrixEventID]) -> None:
+        portal = po.Portal.get_by_mxid(room_id)
+        if not portal:
+            return
+
+        for user_id, event_id in receipts.items():
+            user = await u.User.get_by_mxid(user_id).ensure_started()
+            if not await user.is_logged_in():
+                continue
+            await portal.mark_read(user, event_id)
+
+    @staticmethod
+    async def handle_presence(user_id: MatrixUserID, presence: str) -> None:
+        user = await u.User.get_by_mxid(user_id).ensure_started()
+        if not await user.is_logged_in():
+            return
+        await user.set_presence(presence == "online")
+
+    async def handle_typing(self, room_id: MatrixRoomID, now_typing: List[MatrixUserID]) -> None:
+        portal = po.Portal.get_by_mxid(room_id)
+        if not portal:
+            return
+
+        for user_id in set(self.previously_typing + now_typing):
+            is_typing = user_id in now_typing
+            was_typing = user_id in self.previously_typing
+            if is_typing and was_typing:
+                continue
+
+            user = await u.User.get_by_mxid(user_id).ensure_started()
+            if not await user.is_logged_in():
+                continue
+
+            await portal.set_typing(user, is_typing)
+
+        self.previously_typing = now_typing
+
+    def filter_matrix_event(self, event: MatrixEvent) -> bool:
+        sender = event.get("sender", None)
+        if not sender:
+            return False
+        return (sender == self.az.bot_mxid
+                or pu.Puppet.get_id_from_mxid(sender) is not None)
+
+    async def try_handle_ephemeral_event(self, evt: MatrixEvent) -> None:
+        try:
+            await self.handle_ephemeral_event(evt)
+        except Exception:
+            self.log.exception("Error handling manually received Matrix event")
+
+    async def handle_ephemeral_event(self, evt: MatrixEvent) -> None:
+        evt_type = evt.get("type", "m.unknown")  # type: str
+        room_id = evt.get("room_id", None)  # type: Optional[MatrixRoomID]
+        sender = evt.get("sender", None)  # type: Optional[MatrixUserID]
+        content = evt.get("content", {})  # type: Dict
+        if evt_type == "m.receipt":
+            await self.handle_read_receipts(room_id, self.parse_read_receipts(content))
+        elif evt_type == "m.presence":
+            await self.handle_presence(sender, content.get("presence", "offline"))
+        elif evt_type == "m.typing":
+            await self.handle_typing(room_id, content.get("user_ids", []))
+
+    async def handle_event(self, evt: MatrixEvent) -> None:
         if self.filter_matrix_event(evt):
             return
+        start_time = time.time()
         self.log.debug("Received event: %s", evt)
-        type = evt["type"]
-        content = evt.get("content", {})
-        if type == "m.room.member":
-            prev_content = evt.get("unsigned", {}).get("prev_content", {})
-            membership = content.get("membership", "")
-            prev_membership = prev_content.get("membership", "leave")
-            if membership == prev_membership:
-                # TODO handle displayname/avatar changes
-                pass
-            elif membership == "invite":
-                await self.handle_invite(evt["room_id"], evt["state_key"], evt["sender"])
-            elif prev_membership == "join" and membership == "leave":
-                await self.handle_part(evt["room_id"], evt["state_key"], evt["sender"],
-                                       evt["event_id"])
-            elif membership == "join":
-                await self.handle_join(evt["room_id"], evt["state_key"], evt["event_id"])
-        elif type in ("m.room.message", "m.sticker"):
-            if type != "m.room.message":
-                content["msgtype"] = type
-            await self.handle_message(evt["room_id"], evt["sender"], content, evt["event_id"])
-        elif type == "m.room.redaction":
-            await self.handle_redaction(evt["room_id"], evt["sender"], evt["redacts"])
-        elif type == "m.room.power_levels":
-            await self.handle_power_levels(evt["room_id"], evt["sender"], evt["content"],
-                                           evt["prev_content"])
-        elif type in ("m.room.name", "m.room.avatar", "m.room.topic"):
-            await self.handle_room_meta(type, evt["room_id"], evt["sender"], evt["content"])
-        elif type == "m.room.pinned_events":
-            new_events = set(evt["content"]["pinned"])
-            try:
-                old_events = set(evt["unsigned"]["prev_content"]["pinned"])
-            except KeyError:
-                old_events = set()
-            await self.handle_room_pin(evt["room_id"], evt["sender"], new_events, old_events)
+        evt_type = evt.get("type", "m.unknown")  # type: str
+        room_id = evt.get("room_id", None)  # type: Optional[MatrixRoomID]
+        event_id = evt.get("event_id", None)  # type: Optional[MatrixEventID]
+        sender = evt.get("sender", None)  # type: Optional[MatrixUserID]
+        state_key = evt.get("state_key", None)
+        content = evt.get("content", {})  # type: Dict
+        if state_key is not None:
+            if evt_type == "m.room.member":
+                prev_content = evt.get("unsigned", {}).get("prev_content", {})  # type: Dict
+                membership = content.get("membership", "")  # type: str
+                prev_membership = prev_content.get("membership", "leave")  # type: str
+                if membership == prev_membership:
+                    match = re.compile("@(.+):(.+)").match(state_key)  # type: Match
+                    mxid = match.group(0)  # type: str
+                    displayname = content.get("displayname", None) or mxid  # type: str
+                    prev_displayname = prev_content.get("displayname", None) or mxid  # type: str
+                    if displayname != prev_displayname:
+                        await self.handle_name_change(room_id, state_key, displayname,
+                                                      prev_displayname, event_id)
+                elif membership == "invite":
+                    await self.handle_invite(room_id, state_key, sender)
+                elif prev_membership == "join" and membership == "leave":
+                    await self.handle_part(room_id, state_key, sender, event_id)
+                elif membership == "join":
+                    await self.handle_join(room_id, state_key, event_id)
+            elif evt_type == "m.room.power_levels":
+                prev_content = evt.get("unsigned", {}).get("prev_content", {})
+                await self.handle_power_levels(room_id, sender, evt["content"], prev_content)
+            elif evt_type in ("m.room.name", "m.room.avatar", "m.room.topic"):
+                await self.handle_room_meta(evt_type, room_id, sender, evt["content"])
+            elif evt_type == "m.room.pinned_events":
+                new_events = set(evt["content"]["pinned"])
+                try:
+                    old_events = set(evt["unsigned"]["prev_content"]["pinned"])
+                except KeyError:
+                    old_events = set()
+                await self.handle_room_pin(room_id, sender, new_events, old_events)
+            elif evt_type == "m.room.tombstone":
+                await self.handle_room_upgrade(room_id, evt["content"]["replacement_room"])
+            else:
+                return
+        else:
+            if evt_type in ("m.room.message", "m.sticker"):
+                if evt_type != "m.room.message":
+                    content["msgtype"] = evt_type
+                await self.handle_message(room_id, sender, content, event_id)
+            elif evt_type == "m.room.redaction":
+                await self.handle_redaction(room_id, sender, evt["redacts"])
+            else:
+                return
+
+        if EVENT_TIME:
+            EVENT_TIME.labels(event_type=evt_type).observe(time.time() - start_time)

mautrix_telegram/public/__init__.py

@@ -1,166 +0,0 @@
-# -*- coding: future_fstrings -*-
-# mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
-#
-# This program is free software: you can redistribute it and/or modify
-# it under the terms of the GNU Affero General Public License as published by
-# the Free Software Foundation, either version 3 of the License, or
-# (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU Affero General Public License for more details.
-#
-# You should have received a copy of the GNU Affero General Public License
-# along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from aiohttp import web
-from mako.template import Template
-import asyncio
-import pkg_resources
-import logging
-
-from telethon.errors import *
-
-from ..user import User
-from ..commands.auth import enter_password
-from ..util import format_duration
-
-
-class PublicBridgeWebsite:
-    log = logging.getLogger("mau.public")
-
-    def __init__(self, loop):
-        self.loop = loop
-
-        self.login = Template(
-            pkg_resources.resource_string("mautrix_telegram", "public/login.html.mako"))
-
-        self.app = web.Application(loop=loop)
-        self.app.router.add_route("GET", "/login", self.get_login)
-        self.app.router.add_route("POST", "/login", self.post_login)
-        self.app.router.add_static("/",
-                                   pkg_resources.resource_filename("mautrix_telegram", "public/"))
-
-    async def get_login(self, request):
-        user = (User.get_by_mxid(request.rel_url.query["mxid"], create=False)
-                if "mxid" in request.rel_url.query else None)
-        if not user:
-            return self.render_login(
-                mxid=request.rel_url.query["mxid"] if "mxid" in request.rel_url.query else None,
-                state="request")
-        elif not user.whitelisted:
-            return self.render_login(mxid=user.mxid, error="You are not whitelisted.", status=403)
-        await user.ensure_started()
-        if not user.logged_in:
-            return self.render_login(mxid=user.mxid, state="request")
-
-        return self.render_login(mxid=user.mxid, username=user.username)
-
-    def render_login(self, status=200, username="", state="", error="", message="", mxid=""):
-        return web.Response(status=status, content_type="text/html",
-                            text=self.login.render(username=username, state=state, error=error,
-                                                   message=message, mxid=mxid))
-
-    async def post_login_phone(self, user, phone):
-        try:
-            await user.client.sign_in(phone or "+123")
-            return self.render_login(mxid=user.mxid, state="code", status=200,
-                                     message="Code requested successfully.")
-        except PhoneNumberInvalidError:
-            return self.render_login(mxid=user.mxid, state="request", status=400,
-                                     error="Invalid phone number.")
-        except PhoneNumberUnoccupiedError:
-            return self.render_login(mxid=user.mxid, state="request", status=404,
-                                     error="That phone number has not been registered.")
-        except PhoneNumberFloodError:
-            return self.render_login(
-                mxid=user.mxid, state="request", status=429,
-                error="Your phone number has been temporarily blocked for flooding. "
-                      "The ban is usually applied for around a day.")
-        except FloodWaitError as e:
-            return self.render_login(
-                mxid=user.mxid, state="request", status=429,
-                error="Your phone number has been temporarily blocked for flooding. "
-                      f"Please wait for {format_duration(e.seconds)} before trying again.")
-        except PhoneNumberBannedError:
-            return self.render_login(mxid=user.mxid, state="request", status=401,
-                                     error="Your phone number is banned from Telegram.")
-        except PhoneNumberAppSignupForbiddenError:
-            return self.render_login(mxid=user.mxid, state="request", status=401,
-                                     error="You have disabled 3rd party apps on your account.")
-        except Exception:
-            self.log.exception("Error requesting phone code")
-            return self.render_login(mxid=user.mxid, state="request", status=500,
-                                     error="Internal server error while requesting code.")
-
-    async def post_login_code(self, user, code, password_in_data):
-        try:
-            user_info = await user.client.sign_in(code=code)
-            asyncio.ensure_future(user.post_login(user_info), loop=self.loop)
-            if user.command_status and user.command_status["action"] == "Login":
-                user.command_status = None
-            return self.render_login(mxid=user.mxid, state="logged-in", status=200,
-                                     username=user_info.username)
-        except PhoneCodeInvalidError:
-            return self.render_login(mxid=user.mxid, state="code", status=403,
-                                     error="Incorrect phone code.")
-        except PhoneCodeExpiredError:
-            return self.render_login(mxid=user.mxid, state="code", status=403,
-                                     error="Phone code expired.")
-        except SessionPasswordNeededError:
-            if not password_in_data:
-                if user.command_status and user.command_status["action"] == "Login":
-                    user.command_status = {
-                        "next": enter_password,
-                        "action": "Login (password entry)",
-                    }
-                return self.render_login(
-                    mxid=user.mxid, state="password", status=200,
-                    message="Code accepted, but you have 2-factor authentication is enabled.")
-            return None
-        except Exception:
-            self.log.exception("Error sending phone code")
-            return self.render_login(mxid=user.mxid, state="code", status=500,
-                                     error="Internal server error while sending code.")
-
-    async def post_login_password(self, user, password):
-        try:
-            user_info = await user.client.sign_in(password=password)
-            asyncio.ensure_future(user.post_login(user_info), loop=self.loop)
-            if user.command_status and user.command_status["action"] == "Login (password entry)":
-                user.command_status = None
-            return self.render_login(mxid=user.mxid, state="logged-in", status=200,
-                                     username=user_info.username)
-        except (PasswordHashInvalidError, PasswordEmptyError):
-            return self.render_login(mxid=user.mxid, state="password", status=400,
-                                     error="Incorrect password.")
-        except Exception:
-            self.log.exception("Error sending password")
-            return self.render_login(mxid=user.mxid, state="password", status=500,
-                                     error="Internal server error while sending password.")
-
-    async def post_login(self, request):
-        data = await request.post()
-        if "mxid" not in data:
-            return self.render_login(error="Please enter your Matrix ID.", status=400)
-
-        user = await User.get_by_mxid(data["mxid"]).ensure_started(even_if_no_session=True)
-        if not user.whitelisted:
-            return self.render_login(mxid=user.mxid, error="You are not whitelisted.", status=403)
-        elif user.logged_in:
-            return self.render_login(mxid=user.mxid, username=user.username)
-
-        if "phone" in data:
-            return await self.post_login_phone(user, data["phone"])
-        elif "code" in data:
-            resp = await self.post_login_code(user, data["code"],
-                                              password_in_data="password" in data)
-            if resp or "password" not in data:
-                return resp
-        elif "password" not in data:
-            return self.render_login(error="No data given.", status=400)
-
-        if "password" in data:
-            return await self.post_login_password(user, data["password"])
-        return self.render_login(error="This should never happen.", status=500)

mautrix_telegram/puppet.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -14,84 +14,309 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Awaitable, Any, Dict, List, Iterable, Optional, Pattern, Union, TYPE_CHECKING
 from difflib import SequenceMatcher
-import re
+from enum import Enum
+from aiohttp import ServerDisconnectedError
+import asyncio
 import logging
+import re
 
-from telethon.tl.types import UserProfilePhoto
-from telethon.errors.rpc_error_list import LocationInvalidError
+from telethon.tl.types import (UserProfilePhoto, User, UpdateUserName, PeerUser, TypeInputPeer,
+                               InputPeerPhotoFileLocation, UserProfilePhotoEmpty)
+from mautrix_appservice import AppService, IntentAPI, IntentError, MatrixRequestError
 
+from .types import MatrixUserID, TelegramID
 from .db import Puppet as DBPuppet
 from . import util
 
-config = None
+if TYPE_CHECKING:
+    from .matrix import MatrixHandler
+    from .config import Config
+    from .context import Context
+    from .abstract_user import AbstractUser
+
+PuppetError = Enum('PuppetError', 'Success OnlyLoginSelf InvalidAccessToken')
+
+config = None  # type: Config
 
 
 class Puppet:
-    log = logging.getLogger("mau.puppet")
-    db = None
-    az = None
-    mxid_regex = None
-    username_template = None
-    hs_domain = None
-    cache = {}
+    log = logging.getLogger("mau.puppet")  # type: logging.Logger
+    az = None  # type: AppService
+    mx = None  # type: MatrixHandler
+    loop = None  # type: asyncio.AbstractEventLoop
+    mxid_regex = None  # type: Pattern
+    username_template = None  # type: str
+    hs_domain = None  # type: str
+    cache = {}  # type: Dict[TelegramID, Puppet]
+    by_custom_mxid = {}  # type: Dict[str, Puppet]
 
-    def __init__(self, id=None, username=None, displayname=None, displayname_source=None,
-                 photo_id=None, is_bot=None, db_instance=None):
-        self.id = id
-        self.mxid = self.get_mxid_from_id(self.id)
+    def __init__(self,
+                 id: TelegramID,
+                 access_token: Optional[str] = None,
+                 custom_mxid: Optional[MatrixUserID] = None,
+                 username: Optional[str] = None,
+                 displayname: Optional[str] = None,
+                 displayname_source: Optional[TelegramID] = None,
+                 photo_id: Optional[str] = None,
+                 is_bot: bool = False,
+                 is_registered: bool = False,
+                 disable_updates: bool = False,
+                 db_instance: Optional[DBPuppet] = None) -> None:
+        self.id = id  # type: TelegramID
+        self.access_token = access_token  # type: Optional[str]
+        self.custom_mxid = custom_mxid  # type: Optional[MatrixUserID]
+        self.default_mxid = self.get_mxid_from_id(self.id)  # type: MatrixUserID
 
-        self.username = username
-        self.displayname = displayname
-        self.displayname_source = displayname_source
-        self.photo_id = photo_id
-        self.is_bot = is_bot
-        self._db_instance = db_instance
+        self.username = username  # type: Optional[str]
+        self.displayname = displayname  # type: Optional[str]
+        self.displayname_source = displayname_source  # type: Optional[TelegramID]
+        self.photo_id = photo_id  # type: Optional[str]
+        self.is_bot = is_bot  # type: bool
+        self.is_registered = is_registered  # type: bool
+        self.disable_updates = disable_updates  # type: bool
+        self._db_instance = db_instance  # type: Optional[DBPuppet]
 
-        self.intent = self.az.intent.user(self.mxid)
-        self.logged_in = True
+        self.default_mxid_intent = self.az.intent.user(self.default_mxid)
+        self.intent = self._fresh_intent()  # type: IntentAPI
+        self.sync_task = None  # type: Optional[asyncio.Future]
 
         self.cache[id] = self
+        if self.custom_mxid:
+            self.by_custom_mxid[self.custom_mxid] = self
 
     @property
-    def tgid(self):
+    def mxid(self) -> MatrixUserID:
+        return self.custom_mxid or self.default_mxid
+
+    @property
+    def tgid(self) -> TelegramID:
         return self.id
 
     @property
-    def db_instance(self):
+    def is_real_user(self) -> bool:
+        """ Is True when the puppet is a real Matrix user. """
+        return bool(self.custom_mxid and self.access_token)
+
+    @staticmethod
+    async def is_logged_in() -> bool:
+        """ Is True if the puppet is logged in. """
+        return True
+
+    @property
+    def plain_displayname(self) -> str:
+        tpl = config["bridge.displayname_template"]
+        if tpl == "{displayname}":
+            # Template has no extra stuff, no need to parse.
+            return self.displayname
+        regex = re.compile("^" + re.escape(tpl).replace(re.escape("{displayname}"), "(.+?)") + "$")
+        match = regex.match(self.displayname)
+        return match.group(1) or self.displayname
+
+    def get_input_entity(self, user: 'AbstractUser') -> Awaitable[TypeInputPeer]:
+        return user.client.get_input_entity(PeerUser(user_id=self.tgid))
+
+    # region Custom puppet management
+    def _fresh_intent(self) -> IntentAPI:
+        return (self.az.intent.user(self.custom_mxid, self.access_token)
+                if self.is_real_user else self.default_mxid_intent)
+
+    async def switch_mxid(self, access_token: Optional[str],
+                          mxid: Optional[MatrixUserID]) -> PuppetError:
+        prev_mxid = self.custom_mxid
+        self.custom_mxid = mxid
+        self.access_token = access_token
+        self.intent = self._fresh_intent()
+
+        err = await self.init_custom_mxid()
+        if err != PuppetError.Success:
+            return err
+
+        try:
+            del self.by_custom_mxid[prev_mxid]  # type: ignore
+        except KeyError:
+            pass
+        if self.mxid != self.default_mxid:
+            self.by_custom_mxid[self.mxid] = self
+            await self.leave_rooms_with_default_user()
+        self.save()
+        return PuppetError.Success
+
+    async def init_custom_mxid(self) -> PuppetError:
+        if not self.is_real_user:
+            return PuppetError.Success
+
+        mxid = await self.intent.whoami()
+        if not mxid or mxid != self.custom_mxid:
+            self.custom_mxid = None
+            self.access_token = None
+            self.intent = self._fresh_intent()
+            if mxid != self.custom_mxid:
+                return PuppetError.OnlyLoginSelf
+            return PuppetError.InvalidAccessToken
+        if config["bridge.sync_with_custom_puppets"]:
+            self.sync_task = asyncio.ensure_future(self.sync(), loop=self.loop)
+        return PuppetError.Success
+
+    async def leave_rooms_with_default_user(self) -> None:
+        for room_id in await self.default_mxid_intent.get_joined_rooms():
+            try:
+                await self.default_mxid_intent.leave_room(room_id)
+                await self.intent.ensure_joined(room_id)
+            except (IntentError, MatrixRequestError):
+                pass
+
+    def create_sync_filter(self) -> Awaitable[str]:
+        return self.intent.client.create_filter(self.custom_mxid, {
+            "room": {
+                "include_leave": False,
+                "state": {
+                    "types": []
+                },
+                "timeline": {
+                    "types": [],
+                },
+                "ephemeral": {
+                    "types": ["m.typing", "m.receipt"],
+                },
+                "account_data": {
+                    "types": []
+                }
+            },
+            "account_data": {
+                "types": [],
+            },
+            "presence": {
+                "types": ["m.presence"],
+                "senders": [self.custom_mxid],
+            },
+        })
+
+    def filter_events(self, events: List[Dict]) -> List:
+        new_events = []
+        for event in events:
+            evt_type = event.get("type", None)
+            event.setdefault("content", {})
+            if evt_type == "m.typing":
+                is_typing = self.custom_mxid in event["content"].get("user_ids", [])
+                event["content"]["user_ids"] = [self.custom_mxid] if is_typing else []
+            elif evt_type == "m.receipt":
+                val = None
+                evt = None
+                for event_id in event["content"]:
+                    try:
+                        val = event["content"][event_id]["m.read"][self.custom_mxid]
+                        evt = event_id
+                        break
+                    except KeyError:
+                        pass
+                if val and evt:
+                    event["content"] = {evt: {"m.read": {
+                        self.custom_mxid: val
+                    }}}
+                else:
+                    continue
+            new_events.append(event)
+        return new_events
+
+    def handle_sync(self, presence: List, ephemeral: Dict) -> None:
+        presence_events = [self.mx.try_handle_ephemeral_event(event) for event in presence]
+
+        for room_id, events in ephemeral.items():
+            for event in events:
+                event["room_id"] = room_id
+
+        ephemeral_events = [self.mx.try_handle_ephemeral_event(event)
+                            for events in ephemeral.values()
+                            for event in self.filter_events(events)]
+
+        events = ephemeral_events + presence_events  # List[Callable[[int], Awaitable[None]]]
+        coro = asyncio.gather(*events, loop=self.loop)
+        asyncio.ensure_future(coro, loop=self.loop)
+
+    async def sync(self) -> None:
+        try:
+            await self._sync()
+        except asyncio.CancelledError:
+            self.log.info("Syncing cancelled")
+        except Exception:
+            self.log.exception("Fatal error syncing")
+
+    async def _sync(self) -> None:
+        if not self.is_real_user:
+            self.log.warning("Called sync() for non-custom puppet.")
+            return
+        custom_mxid = self.custom_mxid
+        access_token_at_start = self.access_token
+        errors = 0
+        next_batch = None
+        filter_id = await self.create_sync_filter()
+        self.log.debug(f"Starting syncer for {custom_mxid} with sync filter {filter_id}.")
+        while access_token_at_start == self.access_token:
+            try:
+                sync_resp = await self.intent.client.sync(filter=filter_id, since=next_batch,
+                                                          set_presence="offline")  # type: Dict
+                errors = 0
+                if next_batch is not None:
+                    presence = sync_resp.get("presence", {}).get("events", [])  # type: List
+                    ephemeral = {room: data.get("ephemeral", {}).get("events", [])
+                                 for room, data
+                                 in sync_resp.get("rooms", {}).get("join", {}).items()
+                                 }  # type: Dict
+                    self.handle_sync(presence, ephemeral)
+                next_batch = sync_resp.get("next_batch", None)
+            except (MatrixRequestError, ServerDisconnectedError) as e:
+                wait = min(errors, 11) ** 2
+                self.log.warning(f"Syncer for {custom_mxid} errored: {e}. "
+                                 f"Waiting for {wait} seconds...")
+                errors += 1
+                await asyncio.sleep(wait)
+        self.log.debug(f"Syncer for custom puppet {custom_mxid} stopped.")
+
+    # endregion
+    # region DB conversion
+
+    @property
+    def db_instance(self) -> DBPuppet:
         if not self._db_instance:
             self._db_instance = self.new_db_instance()
         return self._db_instance
 
-    def new_db_instance(self):
-        return DBPuppet(id=self.id, username=self.username, displayname=self.displayname,
+    def new_db_instance(self) -> DBPuppet:
+        return DBPuppet(id=self.id, access_token=self.access_token, custom_mxid=self.custom_mxid,
+                        username=self.username, displayname=self.displayname,
                         displayname_source=self.displayname_source, photo_id=self.photo_id,
-                        is_bot=self.is_bot)
+                        is_bot=self.is_bot, matrix_registered=self.is_registered,
+                        disable_updates=self.disable_updates)
 
     @classmethod
-    def from_db(cls, db_puppet):
-        return Puppet(db_puppet.id, db_puppet.username, db_puppet.displayname,
-                      db_puppet.displayname_source, db_puppet.photo_id, db_puppet.is_bot,
-                      db_instance=db_puppet)
+    def from_db(cls, db_puppet: DBPuppet) -> 'Puppet':
+        return Puppet(db_puppet.id, db_puppet.access_token, db_puppet.custom_mxid,
+                      db_puppet.username, db_puppet.displayname, db_puppet.displayname_source,
+                      db_puppet.photo_id, db_puppet.is_bot, db_puppet.matrix_registered,
+                      db_puppet.disable_updates, db_instance=db_puppet)
 
-    def save(self):
-        self.db_instance.username = self.username
-        self.db_instance.displayname = self.displayname
-        self.db_instance.displayname_source = self.displayname_source
-        self.db_instance.photo_id = self.photo_id
-        self.db_instance.is_bot = self.is_bot
-        self.db.commit()
+    def save(self) -> None:
+        self.db_instance.update(access_token=self.access_token, custom_mxid=self.custom_mxid,
+                                username=self.username, displayname=self.displayname,
+                                displayname_source=self.displayname_source, photo_id=self.photo_id,
+                                is_bot=self.is_bot, matrix_registered=self.is_registered,
+                                disable_updates=self.disable_updates)
 
-    def similarity(self, query):
+    # endregion
+    # region Info updating
+
+    def similarity(self, query: str) -> int:
         username_similarity = (SequenceMatcher(None, self.username, query).ratio()
                                if self.username else 0)
         displayname_similarity = (SequenceMatcher(None, self.displayname, query).ratio()
                                   if self.displayname else 0)
         similarity = max(username_similarity, displayname_similarity)
-        return round(similarity * 1000) / 10
+        return int(round(similarity * 100))
 
     @staticmethod
-    def get_displayname(info, format=True):
+    def get_displayname(info: User, enable_format: bool = True) -> str:
         data = {
             "phone number": info.phone if hasattr(info, "phone") else None,
             "username": info.username,
@@ -100,25 +325,26 @@ class Puppet:
             "first name": info.first_name,
             "last name": info.last_name,
         }
-        preferences = config.get("bridge.displayname_preference",
-                                 ["full name", "username", "phone"])
+        preferences = config["bridge.displayname_preference"]
         name = None
         for preference in preferences:
             name = data[preference]
             if name:
                 break
 
-        if info.deleted:
+        if isinstance(info, User) and info.deleted:
             name = f"Deleted account {info.id}"
         elif not name:
             name = info.id
 
-        if not format:
+        if not enable_format:
             return name
-        return config.get("bridge.displayname_template", "{displayname} (Telegram)").format(
+        return config["bridge.displayname_template"].format(
             displayname=name)
 
-    async def update_info(self, source, info):
+    async def update_info(self, source: 'AbstractUser', info: User) -> None:
+        if self.disable_updates:
+            return
         changed = False
         if self.username != info.username:
             self.username = info.username
@@ -126,77 +352,146 @@ class Puppet:
 
         changed = await self.update_displayname(source, info) or changed
         if isinstance(info.photo, UserProfilePhoto):
-            changed = await self.update_avatar(source, info.photo.photo_big) or changed
+            changed = await self.update_avatar(source, info.photo) or changed
 
         self.is_bot = info.bot
 
         if changed:
             self.save()
 
-    async def update_displayname(self, source, info):
-        ignore_source = (not source.is_relaybot
-                         and self.displayname_source is not None
-                         and self.displayname_source != source.tgid)
-        if ignore_source:
-            return
+    async def update_displayname(self, source: 'AbstractUser', info: Union[User, UpdateUserName]
+                                 ) -> bool:
+        if self.disable_updates:
+            return False
+        allow_source = (source.is_relaybot
+                        or self.displayname_source == source.tgid
+                        # No displayname source, so just trust anything
+                        or self.displayname_source is None
+                        # No phone -> not in contact list -> can't set custom name
+                        or (isinstance(info, User) and info.phone is None))
+        if not allow_source:
+            return False
+        elif isinstance(info, UpdateUserName):
+            info = await source.client.get_entity(PeerUser(self.tgid))
 
         displayname = self.get_displayname(info)
         if displayname != self.displayname:
-            await self.intent.set_display_name(displayname)
             self.displayname = displayname
             self.displayname_source = source.tgid
+            try:
+                await self.default_mxid_intent.set_display_name(displayname[:100])
+            except MatrixRequestError:
+                self.log.exception("Failed to set displayname")
+                self.displayname = ""
+                self.displayname_source = None
             return True
         elif source.is_relaybot or self.displayname_source is None:
             self.displayname_source = source.tgid
             return True
+        return False
 
-    async def update_avatar(self, source, photo):
-        photo_id = f"{photo.volume_id}-{photo.local_id}"
+    async def update_avatar(self, source: 'AbstractUser',
+                            photo: Union[UserProfilePhoto, UserProfilePhotoEmpty]) -> bool:
+        if self.disable_updates:
+            return False
+
+        if isinstance(photo, UserProfilePhotoEmpty):
+            photo_id = ""
+        else:
+            photo_id = str(photo.photo_id)
         if self.photo_id != photo_id:
-            file = await util.transfer_file_to_matrix(self.db, source.client, self.intent, photo)
+            if not photo_id:
+                self.photo_id = ""
+                try:
+                    await self.default_mxid_intent.set_avatar("")
+                except MatrixRequestError:
+                    self.log.exception("Failed to set avatar")
+                    self.photo_id = ""
+                return True
+
+            loc = InputPeerPhotoFileLocation(
+                peer=await self.get_input_entity(source),
+                local_id=photo.photo_big.local_id,
+                volume_id=photo.photo_big.volume_id,
+                big=True
+            )
+            file = await util.transfer_file_to_matrix(source.client, self.default_mxid_intent, loc)
             if file:
-                await self.intent.set_avatar(file.mxc)
                 self.photo_id = photo_id
+                try:
+                    await self.default_mxid_intent.set_avatar(file.mxc)
+                except MatrixRequestError:
+                    self.log.exception("Failed to set avatar")
+                    self.photo_id = ""
                 return True
         return False
 
+    # endregion
+    # region Getters
+
     @classmethod
-    def get(cls, id, create=True):
+    def get(cls, tgid: TelegramID, create: bool = True) -> Optional['Puppet']:
         try:
-            return cls.cache[id]
+            return cls.cache[tgid]
         except KeyError:
             pass
 
-        puppet = DBPuppet.query.get(id)
+        puppet = DBPuppet.get_by_tgid(tgid)
         if puppet:
             return cls.from_db(puppet)
 
         if create:
-            puppet = cls(id)
-            cls.db.add(puppet.db_instance)
-            cls.db.commit()
+            puppet = cls(tgid)
+            puppet.db_instance.insert()
             return puppet
 
         return None
 
     @classmethod
-    def get_by_mxid(cls, mxid, create=True):
+    def get_by_mxid(cls, mxid: MatrixUserID, create: bool = True) -> Optional['Puppet']:
         tgid = cls.get_id_from_mxid(mxid)
-        return cls.get(tgid, create) if tgid else None
+        if tgid:
+            return cls.get(tgid, create)
 
-    @classmethod
-    def get_id_from_mxid(cls, mxid):
-        match = cls.mxid_regex.match(mxid)
-        if match:
-            return int(match.group(1))
         return None
 
     @classmethod
-    def get_mxid_from_id(cls, id):
-        return f"@{cls.username_template.format(userid=id)}:{cls.hs_domain}"
+    def get_by_custom_mxid(cls, mxid: MatrixUserID) -> Optional['Puppet']:
+        if not mxid:
+            raise ValueError("Matrix ID can't be empty")
+
+        try:
+            return cls.by_custom_mxid[mxid]
+        except KeyError:
+            pass
+
+        puppet = DBPuppet.get_by_custom_mxid(mxid)
+        if puppet:
+            puppet = cls.from_db(puppet)
+            return puppet
+
+        return None
 
     @classmethod
-    def find_by_username(cls, username):
+    def all_with_custom_mxid(cls) -> Iterable['Puppet']:
+        return (cls.by_custom_mxid[puppet.mxid]
+                if puppet.custom_mxid in cls.by_custom_mxid
+                else cls.from_db(puppet)
+                for puppet in DBPuppet.all_with_custom_mxid())
+
+    @classmethod
+    def get_id_from_mxid(cls, mxid: MatrixUserID) -> Optional[TelegramID]:
+        match = cls.mxid_regex.match(mxid)
+        if match:
+            return TelegramID(int(match.group(1)))
+        return None
+
+    @classmethod
+    def get_mxid_from_id(cls, tgid: TelegramID) -> MatrixUserID:
+        return MatrixUserID(f"@{cls.username_template.format(userid=tgid)}:{cls.hs_domain}")
+
+    @classmethod
+    def find_by_username(cls, username: str) -> Optional['Puppet']:
         if not username:
             return None
 
@@ -204,14 +499,14 @@ class Puppet:
             if puppet.username and puppet.username.lower() == username.lower():
                 return puppet
 
-        puppet = DBPuppet.query.filter(DBPuppet.username == username).one_or_none()
-        if puppet:
-            return cls.from_db(puppet)
+        dbpuppet = DBPuppet.get_by_username(username)
+        if dbpuppet:
+            return cls.from_db(dbpuppet)
 
         return None
 
     @classmethod
-    def find_by_displayname(cls, displayname):
+    def find_by_displayname(cls, displayname: str) -> Optional['Puppet']:
         if not displayname:
             return None
 
@@ -219,17 +514,20 @@ class Puppet:
             if puppet.displayname and puppet.displayname == displayname:
                 return puppet
 
-        puppet = DBPuppet.query.filter(DBPuppet.displayname == displayname).one_or_none()
-        if puppet:
-            return cls.from_db(puppet)
+        dbpuppet = DBPuppet.get_by_displayname(displayname)
+        if dbpuppet:
+            return cls.from_db(dbpuppet)
 
         return None
+    # endregion
 
 
-def init(context):
+def init(context: 'Context') -> List[Awaitable[Any]]:  # [None, None, PuppetError]
     global config
-    Puppet.az, Puppet.db, config, _, _ = context
+    Puppet.az, config, Puppet.loop, _ = context.core
+    Puppet.mx = context.mx
     Puppet.username_template = config.get("bridge.username_template", "telegram_{userid}")
     Puppet.hs_domain = config["homeserver"]["domain"]
-    localpart = Puppet.username_template.format(userid="(.+)")
-    Puppet.mxid_regex = re.compile(f"@{localpart}:{Puppet.hs_domain}")
+    Puppet.mxid_regex = re.compile(
+        f"@{Puppet.username_template.format(userid='([0-9]+)')}:{Puppet.hs_domain}")
+    return [puppet.init_custom_mxid() for puppet in Puppet.all_with_custom_mxid()]

mautrix_telegram/scripts/dbms_migrate/__main__.py

@@ -0,0 +1,81 @@
+import argparse
+import sqlalchemy as sql
+from sqlalchemy import orm
+from sqlalchemy.ext.declarative import declarative_base
+
+from alchemysession import AlchemySessionContainer
+
+parser = argparse.ArgumentParser(description="mautrix-telegram dbms migration script",
+                                 prog="python -m mautrix_telegram.scripts.dbms_migrate")
+parser.add_argument("-f", "--from-url", type=str, required=True, metavar="<url>",
+                    help="the old database path")
+parser.add_argument("-t", "--to-url", type=str, required=True, metavar="<url>",
+                    help="the new database path")
+parser.add_argument("-v", "--verbose", action="store_true", help="Verbose logs while migrating")
+args = parser.parse_args()
+verbose = args.verbose or False
+
+
+def log(message, end="\n"):
+    if verbose:
+        print(message, end=end, flush=True)
+
+
+def connect(to):
+    import mautrix_telegram.db.base as base
+    base.Base = declarative_base(cls=base.BaseBase)
+    from mautrix_telegram.db import (Portal, Message, UserPortal, User, RoomState, UserProfile,
+                                     Contact, Puppet, BotChat, TelegramFile)
+    db_engine = sql.create_engine(to)
+    db_factory = orm.sessionmaker(bind=db_engine)
+    db_session = orm.scoped_session(db_factory)  # type: orm.Session
+    base.Base.metadata.bind = db_engine
+    session_container = AlchemySessionContainer(engine=db_engine, session=db_session,
+                                                table_base=base.Base, table_prefix="telethon_",
+                                                manage_tables=False)
+
+    return db_session, {
+        "Version": session_container.Version,
+        "Session": session_container.Session,
+        "Entity": session_container.Entity,
+        "SentFile": session_container.SentFile,
+        "UpdateState": session_container.UpdateState,
+        "Portal": Portal,
+        "Message": Message,
+        "Puppet": Puppet,
+        "User": User,
+        "UserPortal": UserPortal,
+        "RoomState": RoomState,
+        "UserProfile": UserProfile,
+        "Contact": Contact,
+        "BotChat": BotChat,
+        "TelegramFile": TelegramFile,
+    }
+
+log("Connecting to old database")
+session, tables = connect(args.from_url)
+
+data = {}
+for name, table in tables.items():
+    log("Reading table {name}...".format(name=name), end=" ")
+    data[name] = session.query(table).all()
+    log("Done!")
+
+log("Connecting to new database")
+session, tables = connect(args.to_url)
+
+for name, table in tables.items():
+    log("Writing table {name}".format(name=name), end="")
+    length = len(data[name])
+    n = 0
+    for row in data[name]:
+        session.merge(row)
+        n += 5
+        if n >= length:
+            log(".", end="")
+            n = 0
+    log(" Done!")
+
+log("Committing changes to database...", end=" ")
+session.commit()
+log("Done!")

mautrix_telegram/scripts/telematrix_import/__main__.py

@@ -0,0 +1,124 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Dict
+from sqlalchemy import orm
+import sqlalchemy as sql
+import argparse
+
+from mautrix_telegram.db import Base, Portal, Message, Puppet, BotChat
+from mautrix_telegram.config import Config
+
+from .models import ChatLink, TgUser, MatrixUser, Message as TMMessage, Base as TelematrixBase
+
+parser = argparse.ArgumentParser(
+    description="mautrix-telegram telematrix import script",
+    prog="python -m mautrix_telegram.scripts.telematrix_import")
+parser.add_argument("-c", "--config", type=str, default="config.yaml",
+                    metavar="<path>", help="the path to your mautrix-telegram config file")
+parser.add_argument("-b", "--bot-id", type=int, required=True,
+                    metavar="<id>", help="the telegram user ID of your relay bot")
+parser.add_argument("-t", "--telematrix-database", type=str, default="sqlite:///database.db",
+                    metavar="<url>", help="your telematrix database URL")
+args = parser.parse_args()
+
+config = Config(args.config, None, None)
+config.load()
+
+mxtg_db_engine = sql.create_engine(
+    config.get("appservice.database", "sqlite:///mautrix-telegram.db"))
+mxtg = orm.sessionmaker(bind=mxtg_db_engine)()
+Base.metadata.bind = mxtg_db_engine
+
+telematrix_db_engine = sql.create_engine(args.telematrix_database)
+telematrix = orm.sessionmaker(bind=telematrix_db_engine)()
+TelematrixBase.metadata.bind = telematrix_db_engine
+
+chat_links = telematrix.query(ChatLink).all()
+tg_users = telematrix.query(TgUser).all()
+mx_users = telematrix.query(MatrixUser).all()
+tm_messages = telematrix.query(TMMessage).all()
+
+telematrix.close()
+telematrix_db_engine.dispose()
+
+portals_by_tgid = {}  # type: Dict[int, Portal]
+portals_by_mxid = {}  # type: Dict[str, Portal]
+chats = {}  # type: Dict[int, BotChat]
+messages = {}  # type: Dict[str, Message]
+puppets = {}  # type: Dict[int, Puppet]
+
+for chat_link in chat_links:
+    if type(chat_link.tg_room) is str:
+        print("Expected tg_room to be a number, got a string. Ignoring %s" % chat_link.tg_room)
+        continue
+    if chat_link.tg_room >= 0:
+        print("Unexpected unprefixed telegram chat ID: %s, ignoring..." % chat_link.tg_room)
+        continue
+    tgid = str(chat_link.tg_room)
+    if tgid.startswith("-100"):
+        tgid = int(tgid[4:])
+        peer_type = "channel"
+        megagroup = True
+    else:
+        tgid = -chat_link.tg_room
+        peer_type = "chat"
+        megagroup = False
+
+    portal = Portal(tgid=tgid, tg_receiver=tgid, peer_type=peer_type, megagroup=megagroup,
+                    mxid=chat_link.matrix_room)
+    chats[tgid] = BotChat(id=tgid, type=peer_type)
+    if chat_link.tg_room in portals_by_tgid:
+        print(f"Warning: Ignoring bridge from {portal.tgid} to {portal.mxid} "
+              f"in favor of {portals_by_tgid[portal.tgid].mxid}")
+        continue
+    elif chat_link.matrix_room in portals_by_mxid:
+        print(f"Warning: Ignoring bridge from {portal.mxid} to {portal.tgid} "
+              f"in favor of {portals_by_mxid[portal.mxid].tgid}")
+        continue
+    portals_by_tgid[portal.tgid] = portal
+    portals_by_mxid[portal.mxid] = portal
+
+for tm_msg in tm_messages:
+    try:
+        portal = portals_by_tgid[tm_msg.tg_group_id]
+    except KeyError:
+        print(f"Found message entry {tm_msg.tg_message_id} in unlinked chat {tm_msg.tg_group_id},"
+              " ignoring...")
+        continue
+    if tm_msg.matrix_room_id != portal.mxid:
+        print(f"Found message entry {tm_msg.tg_message_id} with "
+              f"mismatching matrix room ID {tm_msg.matrix_room_id} (expected {portal.mxid})")
+        continue
+    tg_space = portal.tgid if portal.peer_type == "channel" else args.bot_id
+    message = Message(mxid=tm_msg.matrix_event_id, mx_room=tm_msg.matrix_room_id,
+                      tgid=tm_msg.tg_message_id, tg_space=tg_space)
+    messages[tm_msg.matrix_event_id] = message
+
+for user in tg_users:
+    puppets[user.tg_id] = Puppet(id=user.tg_id, displayname=user.name,
+                                 displayname_source=args.bot_id)
+
+for k, v in portals_by_tgid.items():
+    mxtg.add(v)
+for k, v in chats.items():
+    mxtg.add(v)
+for k, v in messages.items():
+    mxtg.add(v)
+for k, v in puppets.items():
+    mxtg.add(v)
+
+mxtg.commit()

mautrix_telegram/scripts/telematrix_import/models.py

@@ -0,0 +1,44 @@
+import sqlalchemy as sa
+from sqlalchemy.ext.declarative import declarative_base
+
+Base = declarative_base()
+
+
+class ChatLink(Base):
+    __tablename__ = "chat_link"
+
+    id = sa.Column(sa.Integer, primary_key=True)
+    matrix_room = sa.Column(sa.String)
+    tg_room = sa.Column(sa.BigInteger)
+    active = sa.Column(sa.Boolean)
+
+
+class TgUser(Base):
+    __tablename__ = "tg_user"
+
+    id = sa.Column(sa.Integer, primary_key=True)
+    tg_id = sa.Column(sa.BigInteger)
+    name = sa.Column(sa.String)
+    profile_pic_id = sa.Column(sa.String, nullable=True)
+
+
+class MatrixUser(Base):
+    __tablename__ = "matrix_user"
+
+    id = sa.Column(sa.Integer, primary_key=True)
+    matrix_id = sa.Column(sa.String)
+    name = sa.Column(sa.String)
+
+
+class Message(Base):
+    """Describes a message in a room bridged between Telegram and Matrix"""
+    __tablename__ = "message"
+
+    id = sa.Column(sa.Integer, primary_key=True)
+    tg_group_id = sa.Column(sa.BigInteger)
+    tg_message_id = sa.Column(sa.BigInteger)
+
+    matrix_room_id = sa.Column(sa.String)
+    matrix_event_id = sa.Column(sa.String)
+
+    displayname = sa.Column(sa.String)

mautrix_telegram/sqlstatestore.py

@@ -0,0 +1,119 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Dict, Tuple
+
+from mautrix_appservice import StateStore
+
+from .types import MatrixUserID, MatrixRoomID
+from . import puppet as pu
+from .db import RoomState, UserProfile
+
+
+class SQLStateStore(StateStore):
+    def __init__(self) -> None:
+        super().__init__()
+        self.profile_cache = {}  # type: Dict[Tuple[str, str], UserProfile]
+        self.room_state_cache = {}  # type: Dict[str, RoomState]
+
+    @staticmethod
+    def is_registered(user: MatrixUserID) -> bool:
+        puppet = pu.Puppet.get_by_mxid(user)
+        return puppet.is_registered if puppet else False
+
+    @staticmethod
+    def registered(user: MatrixUserID) -> None:
+        puppet = pu.Puppet.get_by_mxid(user)
+        if puppet:
+            puppet.is_registered = True
+            puppet.save()
+
+    def update_state(self, event: Dict) -> None:
+        event_type = event["type"]
+        if event_type == "m.room.power_levels":
+            self.set_power_levels(event["room_id"], event["content"])
+        elif event_type == "m.room.member":
+            self.set_member(event["room_id"], event["state_key"], event["content"])
+
+    def _get_user_profile(self, room_id: MatrixRoomID, user_id: MatrixUserID, create: bool = True
+                          ) -> UserProfile:
+        key = (room_id, user_id)
+        try:
+            return self.profile_cache[key]
+        except KeyError:
+            pass
+
+        profile = UserProfile.get(*key)
+        if profile:
+            self.profile_cache[key] = profile
+        elif create:
+            profile = UserProfile(room_id=room_id, user_id=user_id, membership="leave")
+            profile.insert()
+            self.profile_cache[key] = profile
+        return profile
+
+    def get_member(self, room: MatrixRoomID, user: MatrixUserID) -> Dict:
+        return self._get_user_profile(room, user).dict()
+
+    def set_member(self, room: MatrixRoomID, user: MatrixUserID, member: Dict) -> None:
+        profile = self._get_user_profile(room, user)
+        profile.membership = member.get("membership", profile.membership or "leave")
+        profile.displayname = member.get("displayname", profile.displayname)
+        profile.avatar_url = member.get("avatar_url", profile.avatar_url)
+        profile.update()
+
+    def set_membership(self, room: MatrixRoomID, user: MatrixUserID, membership: str) -> None:
+        self.set_member(room, user, {
+            "membership": membership,
+        })
+
+    def _get_room_state(self, room_id: MatrixRoomID, create: bool = True) -> RoomState:
+        try:
+            return self.room_state_cache[room_id]
+        except KeyError:
+            pass
+
+        room = RoomState.get(room_id)
+        if room:
+            self.room_state_cache[room_id] = room
+        elif create:
+            room = RoomState(room_id=room_id)
+            room.insert()
+            self.room_state_cache[room_id] = room
+        return room
+
+    def has_power_levels(self, room: MatrixRoomID) -> bool:
+        return bool(self._get_room_state(room).power_levels)
+
+    def get_power_levels(self, room: MatrixRoomID) -> Dict:
+        return self._get_room_state(room).power_levels
+
+    def set_power_level(self, room: MatrixRoomID, user: MatrixUserID, level: int) -> None:
+        room_state = self._get_room_state(room)
+        power_levels = room_state.power_levels
+        if not power_levels:
+            power_levels = {
+                "users": {},
+                "events": {},
+            }
+        power_levels[room]["users"][user] = level
+        room_state.power_levels = power_levels
+        room_state.update()
+
+    def set_power_levels(self, room: MatrixRoomID, content: Dict) -> None:
+        state = self._get_room_state(room)
+        state.power_levels = content
+        state.update()

mautrix_telegram/tgclient.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -14,47 +14,24 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
-from io import BytesIO
+from typing import List, Union, Optional
 
-from telethon import TelegramClient
-from telethon.tl.functions.messages import SendMessageRequest, SendMediaRequest
-from telethon.tl.types import *
-from telethon.extensions.markdown import parse as parse_md
+from telethon import TelegramClient, utils
+from telethon.tl.functions.messages import SendMediaRequest
+from telethon.tl.types import (
+    InputMediaUploadedDocument, InputMediaUploadedPhoto, TypeDocumentAttribute, TypeInputMedia,
+    TypeInputPeer, TypeMessageEntity, TypeMessageMedia, TypePeer)
+from telethon.tl.patched import Message
 
 
 class MautrixTelegramClient(TelegramClient):
-    async def send_message(self, entity, message, reply_to=None, entities=None, markdown=False,
-                           link_preview=True):
-        entity = await self.get_input_entity(entity)
-
-        if markdown:
-            message, entities = parse_md(message)
-
-        request = SendMessageRequest(
-            peer=entity,
-            message=message,
-            entities=entities,
-            no_webpage=not link_preview,
-            reply_to_msg_id=self._get_message_id(reply_to)
-        )
-        result = await self(request)
-        if isinstance(result, UpdateShortSentMessage):
-            return Message(
-                id=result.id,
-                to_id=entity,
-                message=message,
-                date=result.date,
-                out=result.out,
-                media=result.media,
-                entities=result.entities
-            )
-
-        return self._get_response_message(request, result)
-
-    async def upload_file(self, file, mime_type=None, attributes=None, file_name=None):
+    async def upload_file_direct(self, file: bytes, mime_type: str = None,
+                                 attributes: List[TypeDocumentAttribute] = None,
+                                 file_name: str = None, max_image_size: float = 10 * 1000 ** 2,
+                                 ) -> Union[InputMediaUploadedDocument, InputMediaUploadedPhoto]:
         file_handle = await super().upload_file(file, file_name=file_name, use_cache=False)
 
-        if mime_type == "image/png" or mime_type == "image/jpeg":
+        if (mime_type == "image/png" or mime_type == "image/jpeg") and len(file) < max_image_size:
             return InputMediaUploadedPhoto(file_handle)
         else:
             attributes = attributes or []
@@ -65,24 +42,12 @@ class MautrixTelegramClient(TelegramClient):
                 mime_type=mime_type or "application/octet-stream",
                 attributes=list(attr_dict.values()))
 
-    async def send_media(self, entity, media, caption=None, entities=None, reply_to=None):
+    async def send_media(self, entity: Union[TypeInputPeer, TypePeer],
+                         media: Union[TypeInputMedia, TypeMessageMedia],
+                         caption: str = None, entities: List[TypeMessageEntity] = None,
+                         reply_to: int = None) -> Optional[Message]:
         entity = await self.get_input_entity(entity)
-        reply_to = self._get_message_id(reply_to)
+        reply_to = utils.get_message_id(reply_to)
         request = SendMediaRequest(entity, media, message=caption or "", entities=entities or [],
                                    reply_to_msg_id=reply_to)
-        return self._get_response_message(request, await self(request))
-
-    async def download_file_bytes(self, location):
-        if isinstance(location, Document):
-            location = InputDocumentFileLocation(location.id, location.access_hash,
-                                                 location.version)
-        elif not isinstance(location, (InputFileLocation, InputDocumentFileLocation)):
-            location = InputFileLocation(location.volume_id, location.local_id, location.secret)
-
-        file = BytesIO()
-
-        await self.download_file(location, file)
-
-        data = file.getvalue()
-        file.close()
-        return data
+        return self._get_response_message(request, await self(request), entity)

mautrix_telegram/types.py

@@ -0,0 +1,9 @@
+from typing import Dict, NewType
+
+MatrixUserID = NewType('MatrixUserID', str)
+MatrixRoomID = NewType('MatrixRoomID', str)
+MatrixEventID = NewType('MatrixEventID', str)
+
+MatrixEvent = NewType('MatrixEvent', Dict)
+
+TelegramID = NewType('TelegramID', int)

mautrix_telegram/user.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -14,165 +14,238 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Awaitable, Dict, List, Iterable, Match, NewType, Optional, Tuple, TYPE_CHECKING
 import logging
 import asyncio
 import re
 
-from telethon.tl.types import *
+from telethon.tl.types import (
+    TypeUpdate, UpdateNewMessage, UpdateNewChannelMessage, PeerUser,
+    UpdateShortChatMessage, UpdateShortMessage, User as TLUser)
 from telethon.tl.types.contacts import ContactsNotModified
 from telethon.tl.functions.contacts import GetContactsRequest, SearchRequest
+from telethon.tl.functions.account import UpdateStatusRequest
 from mautrix_appservice import MatrixRequestError
 
-from .db import User as DBUser, Contact as DBContact
+from .types import MatrixUserID, TelegramID
+from .db import User as DBUser
 from .abstract_user import AbstractUser
 from . import portal as po, puppet as pu
 
-config = None
+if TYPE_CHECKING:
+    from .config import Config
+    from .context import Context
+
+config = None  # type: Config
+
+SearchResult = NewType('SearchResult', Tuple['pu.Puppet', int])
 
 
 class User(AbstractUser):
-    log = logging.getLogger("mau.user")
-    by_mxid = {}
-    by_tgid = {}
+    log = logging.getLogger("mau.user")  # type: logging.Logger
+    by_mxid = {}  # type: Dict[str, User]
+    by_tgid = {}  # type: Dict[int, User]
 
-    def __init__(self, mxid, tgid=None, username=None, db_contacts=None, saved_contacts=0,
-                 db_portals=None, db_instance=None):
+    def __init__(self, mxid: MatrixUserID, tgid: Optional[TelegramID] = None,
+                 username: Optional[str] = None, phone: Optional[str] = None,
+                 db_contacts: Optional[Iterable[TelegramID]] = None,
+                 saved_contacts: int = 0, is_bot: bool = False,
+                 db_portals: Optional[Iterable[Tuple[TelegramID, TelegramID]]] = None,
+                 db_instance: Optional[DBUser] = None) -> None:
         super().__init__()
-        self.mxid = mxid
-        self.tgid = tgid
-        self.username = username
-        self.contacts = []
-        self.saved_contacts = saved_contacts
+        self.mxid = mxid  # type: MatrixUserID
+        self.tgid = tgid  # type: TelegramID
+        self.is_bot = is_bot  # type: bool
+        self.username = username  # type: str
+        self.phone = phone  # type: str
+        self.contacts = []  # type: List[pu.Puppet]
+        self.saved_contacts = saved_contacts  # type: int
         self.db_contacts = db_contacts
-        self.portals = {}
-        self.db_portals = db_portals
-        self._db_instance = db_instance
+        self.portals = {}  # type: Dict[Tuple[TelegramID, TelegramID], po.Portal]
+        self.db_portals = db_portals or []
+        self._db_instance = db_instance  # type: Optional[DBUser]
 
-        self.command_status = None
+        self.command_status = None  # type: Optional[Dict]
 
         (self.relaybot_whitelisted,
          self.whitelisted,
-         self.is_admin) = config.get_permissions(self.mxid)
+         self.puppet_whitelisted,
+         self.matrix_puppet_whitelisted,
+         self.is_admin,
+         self.permissions) = config.get_permissions(self.mxid)
 
         self.by_mxid[mxid] = self
         if tgid:
             self.by_tgid[tgid] = self
 
     @property
-    def name(self):
+    def name(self) -> str:
         return self.mxid
 
     @property
-    def displayname(self):
-        # TODO show better username
-        match = re.compile("@(.+):(.+)").match(self.mxid)
+    def mxid_localpart(self) -> str:
+        match = re.compile("@(.+):(.+)").match(self.mxid)  # type: Match
         return match.group(1)
 
     @property
-    def db_contacts(self):
-        return [self.db.merge(DBContact(user=self.tgid, contact=puppet.id))
-                for puppet in self.contacts]
+    def human_tg_id(self) -> str:
+        return f"@{self.username}" if self.username else f"+{self.phone}" or None
 
-    @db_contacts.setter
-    def db_contacts(self, contacts):
-        if contacts:
-            self.contacts = [pu.Puppet.get(entry.contact) for entry in contacts]
-        else:
-            self.contacts = []
+    # TODO replace with proper displayname getting everywhere
+    @property
+    def displayname(self) -> str:
+        return self.mxid_localpart
 
     @property
-    def db_portals(self):
-        return [portal.db_instance for portal in self.portals.values()]
+    def plain_displayname(self) -> str:
+        return self.displayname
+
+    @property
+    def db_contacts(self) -> Iterable[TelegramID]:
+        return (puppet.id
+                for puppet in self.contacts
+                if puppet)
+
+    @db_contacts.setter
+    def db_contacts(self, contacts: Iterable[TelegramID]) -> None:
+        self.contacts = [pu.Puppet.get(entry) for entry in contacts] if contacts else []
+
+    @property
+    def db_portals(self) -> Iterable[Tuple[TelegramID, TelegramID]]:
+        return (portal.tgid_full
+                for portal in self.portals.values()
+                if portal and not portal.deleted)
 
     @db_portals.setter
-    def db_portals(self, portals):
-        if portals:
-            self.portals = {(portal.tgid, portal.tg_receiver):
-                                po.Portal.get_by_tgid(portal.tgid, portal.tg_receiver)
-                            for portal in portals}
-        else:
-            self.portals = {}
+    def db_portals(self, portals: Iterable[Tuple[TelegramID, TelegramID]]) -> None:
+        self.portals = {
+            tgid_full: po.Portal.get_by_tgid(*tgid_full)
+            for tgid_full in portals
+        } if portals else {}
 
     # region Database conversion
 
     @property
-    def db_instance(self):
+    def db_instance(self) -> DBUser:
         if not self._db_instance:
             self._db_instance = self.new_db_instance()
         return self._db_instance
 
-    def new_db_instance(self):
+    def new_db_instance(self) -> DBUser:
         return DBUser(mxid=self.mxid, tgid=self.tgid, tg_username=self.username,
-                      contacts=self.db_contacts, saved_contacts=self.saved_contacts,
-                      portals=self.db_portals)
+                      saved_contacts=self.saved_contacts, portals=self.db_portals)
 
-    def save(self):
-        self.db_instance.tgid = self.tgid
-        self.db_instance.username = self.username
-        self.db_instance.contacts = self.db_contacts
-        self.db_instance.saved_contacts = self.saved_contacts
-        self.db_instance.portals = self.db_portals
-        self.db.commit()
+    def save(self, contacts: bool = False, portals: bool = False) -> None:
+        self.db_instance.update(tgid=self.tgid, tg_username=self.username, tg_phone=self.phone,
+                                saved_contacts=self.saved_contacts)
+        if contacts:
+            self.db_instance.contacts = self.db_contacts
+        if portals:
+            self.db_instance.portals = self.db_portals
 
-    def delete(self):
+    def delete(self, delete_db: bool = True) -> None:
         try:
             del self.by_mxid[self.mxid]
             del self.by_tgid[self.tgid]
         except KeyError:
             pass
-        if self._db_instance:
-            self.db.delete(self._db_instance)
-            self.db.commit()
+        if delete_db and self._db_instance:
+            self._db_instance.delete()
 
     @classmethod
-    def from_db(cls, db_user):
-        return User(db_user.mxid, db_user.tgid, db_user.tg_username, db_user.contacts,
-                    db_user.saved_contacts, db_user.portals, db_instance=db_user)
+    def from_db(cls, db_user: DBUser) -> 'User':
+        return User(db_user.mxid, db_user.tgid, db_user.tg_username, db_user.tg_phone,
+                    db_user.contacts, db_user.saved_contacts, False, db_user.portals,
+                    db_instance=db_user)
 
     # endregion
     # region Telegram connection management
 
-    async def start(self, delete_unless_authenticated=False):
+    async def try_ensure_started(self) -> None:
+        try:
+            await self.ensure_started()
+        except Exception:
+            self.log.exception("Exception in ensure_started")
+
+    def ensure_started(self, even_if_no_session=False) -> Awaitable['User']:
+        return super().ensure_started(even_if_no_session)
+
+    async def start(self, delete_unless_authenticated: bool = False) -> 'User':
         await super().start()
-        if self.logged_in:
+        if await self.is_logged_in():
             self.log.debug(f"Ensuring post_login() for {self.name}")
             asyncio.ensure_future(self.post_login(), loop=self.loop)
         elif delete_unless_authenticated:
-            self.log.debug(f"Unauthenticated user {self.name} start()ed, deleting...")
-            # User not logged in -> forget user
-            self.client.disconnect()
-            # self.client.session.delete()
-            self.delete()
+            self.log.debug(f"Unauthenticated user {self.name} start()ed, deleting session...")
+            await self.client.disconnect()
+            self.client.session.delete()
         return self
 
-    async def post_login(self, info=None):
+    async def post_login(self, info: TLUser = None) -> None:
         try:
             await self.update_info(info)
-            await self.sync_dialogs()
-            await self.sync_contacts()
+            if not self.is_bot and config["bridge.startup_sync"]:
+                await self.sync_dialogs()
+                await self.sync_contacts()
             if config["bridge.catch_up"]:
                 await self.client.catch_up()
         except Exception:
-            self.log.exception("Failed to run post-login functions")
+            self.log.exception("Failed to run post-login functions for %s", self.mxid)
+
+    async def update(self, update: TypeUpdate) -> bool:
+        if not self.is_bot:
+            return False
+
+        if isinstance(update, (UpdateNewMessage, UpdateNewChannelMessage)):
+            message = update.message
+            if isinstance(message.to_id, PeerUser) and not message.out:
+                portal = po.Portal.get_by_tgid(message.from_id, peer_type="user",
+                                               tg_receiver=self.tgid)
+            else:
+                portal = po.Portal.get_by_entity(message.to_id, receiver_id=self.tgid)
+        elif isinstance(update, UpdateShortChatMessage):
+            portal = po.Portal.get_by_tgid(TelegramID(update.chat_id), peer_type="chat")
+        elif isinstance(update, UpdateShortMessage):
+            portal = po.Portal.get_by_tgid(TelegramID(update.user_id), self.tgid, "user")
+        else:
+            return False
+
+        if portal:
+            self.register_portal(portal)
+
+        return True
 
     # endregion
     # region Telegram actions that need custom methods
 
-    async def update_info(self, info=None):
+    async def set_presence(self, online: bool = True) -> None:
+        if not self.is_bot:
+            await self.client(UpdateStatusRequest(offline=not online))
+
+    async def update_info(self, info: TLUser = None) -> None:
         info = info or await self.client.get_me()
         changed = False
+        if self.is_bot != info.bot:
+            self.is_bot = info.bot
+            changed = True
         if self.username != info.username:
             self.username = info.username
             changed = True
+        if self.phone != info.phone:
+            self.phone = info.phone
+            changed = True
         if self.tgid != info.id:
             self.tgid = info.id
             self.by_tgid[self.tgid] = self
         if changed:
             self.save()
 
-    async def log_out(self):
+    async def log_out(self) -> bool:
+        puppet = pu.Puppet.get(self.tgid)
+        if puppet.is_real_user:
+            await puppet.switch_mxid(None, None)
         for _, portal in self.portals.items():
-            if portal.has_bot:
+            if not portal or portal.deleted or not portal.mxid or portal.has_bot:
                 continue
             try:
                 await portal.main_intent.kick(portal.mxid, self.mxid, "Logged out of Telegram.")
@@ -180,7 +253,7 @@ class User(AbstractUser):
                 pass
         self.portals = {}
         self.contacts = []
-        self.save()
+        self.save(portals=True, contacts=True)
         if self.tgid:
             try:
                 del self.by_tgid[self.tgid]
@@ -194,28 +267,30 @@ class User(AbstractUser):
         self.delete()
         return True
 
-    def _search_local(self, query, max_results=5, min_similarity=45):
-        results = []
+    def _search_local(self, query: str, max_results: int = 5, min_similarity: int = 45
+                      ) -> List[SearchResult]:
+        results = []  # type: List[SearchResult]
         for contact in self.contacts:
             similarity = contact.similarity(query)
             if similarity >= min_similarity:
-                results.append((contact, similarity))
+                results.append(SearchResult((contact, similarity)))
         results.sort(key=lambda tup: tup[1], reverse=True)
         return results[0:max_results]
 
-    async def _search_remote(self, query, max_results=5):
+    async def _search_remote(self, query: str, max_results: int = 5) -> List[SearchResult]:
         if len(query) < 5:
             return []
         server_results = await self.client(SearchRequest(q=query, limit=max_results))
-        results = []
+        results = []  # type: List[SearchResult]
         for user in server_results.users:
             puppet = pu.Puppet.get(user.id)
             await puppet.update_info(self, user)
-            results.append((puppet, puppet.similarity(query)))
+            results.append(SearchResult((puppet, puppet.similarity(query))))
         results.sort(key=lambda tup: tup[1], reverse=True)
         return results[0:max_results]
 
-    async def search(self, query, force_remote=False):
+    async def search(self, query: str, force_remote: bool = False
+                     ) -> Tuple[List[SearchResult], bool]:
         if force_remote:
             return await self._search_remote(query), True
 
@@ -225,83 +300,89 @@ class User(AbstractUser):
 
         return await self._search_remote(query), True
 
-    async def sync_dialogs(self, synchronous_create=False):
+    async def sync_dialogs(self, synchronous_create: bool = False) -> None:
         creators = []
-        for entity in await self._get_dialogs(limit=30):
+        for entity in await self.get_dialogs(limit=config["bridge.sync_dialog_limit"] or None):
             portal = po.Portal.get_by_entity(entity)
             self.portals[portal.tgid_full] = portal
             creators.append(
                 portal.create_matrix_room(self, entity, invites=[self.mxid],
                                           synchronous=synchronous_create))
-        self.save()
+        self.save(portals=True)
         await asyncio.gather(*creators, loop=self.loop)
 
-    def register_portal(self, portal):
+    def register_portal(self, portal: po.Portal) -> None:
         try:
             if self.portals[portal.tgid_full] == portal:
                 return
         except KeyError:
             pass
         self.portals[portal.tgid_full] = portal
-        self.save()
+        self.save(portals=True)
 
-    def unregister_portal(self, portal):
+    def unregister_portal(self, portal: po.Portal) -> None:
         try:
             del self.portals[portal.tgid_full]
-            self.save()
+            self.save(portals=True)
         except KeyError:
             pass
 
-    def _hash_contacts(self):
+    async def needs_relaybot(self, portal: po.Portal) -> bool:
+        return not await self.is_logged_in() or (
+            (portal.has_bot or self.bot) and portal.tgid_full not in self.portals)
+
+    def _hash_contacts(self) -> int:
         acc = 0
-        for id in sorted([self.saved_contacts] + [contact.id for contact in self.contacts]):
-            acc = (acc * 20261 + id) & 0xffffffff
+        for contact in sorted([self.saved_contacts] + [contact.id for contact in self.contacts]):
+            acc = (acc * 20261 + contact) & 0xffffffff
         return acc & 0x7fffffff
 
-    async def sync_contacts(self):
+    async def sync_contacts(self) -> None:
         response = await self.client(GetContactsRequest(hash=self._hash_contacts()))
         if isinstance(response, ContactsNotModified):
             return
-        self.log.debug("Updating contacts...")
+        self.log.debug(f"Updating contacts of {self.name}...")
         self.contacts = []
         self.saved_contacts = response.saved_count
         for user in response.users:
             puppet = pu.Puppet.get(user.id)
             await puppet.update_info(self, user)
             self.contacts.append(puppet)
-        self.save()
+        self.save(contacts=True)
 
     # endregion
     # region Class instance lookup
 
     @classmethod
-    def get_by_mxid(cls, mxid, create=True):
+    def get_by_mxid(cls, mxid: MatrixUserID, create: bool = True) -> Optional['User']:
+        if not mxid:
+            raise ValueError("Matrix ID can't be empty")
+
         try:
             return cls.by_mxid[mxid]
         except KeyError:
             pass
 
-        user = DBUser.query.get(mxid)
+        user = DBUser.get_by_mxid(mxid)
         if user:
             user = cls.from_db(user)
             return user
 
         if create:
             user = cls(mxid)
-            cls.db.add(user.db_instance)
-            cls.db.commit()
+            user.db_instance.insert()
             return user
 
         return None
 
     @classmethod
-    def get_by_tgid(cls, tgid):
+    def get_by_tgid(cls, tgid: TelegramID) -> Optional['User']:
         try:
             return cls.by_tgid[tgid]
         except KeyError:
             pass
 
-        user = DBUser.query.filter(DBUser.tgid == tgid).one_or_none()
+        user = DBUser.get_by_tgid(tgid)
         if user:
             user = cls.from_db(user)
             return user
@@ -309,7 +390,7 @@ class User(AbstractUser):
         return None
 
     @classmethod
-    def find_by_username(cls, username):
+    def find_by_username(cls, username: str) -> Optional['User']:
         if not username:
             return None
 
@@ -317,7 +398,7 @@ class User(AbstractUser):
             if user.username and user.username.lower() == username.lower():
                 return user
 
-        puppet = DBUser.query.filter(DBUser.tg_username == username).one_or_none()
+        puppet = DBUser.get_by_username(username)
         if puppet:
             return cls.from_db(puppet)
 
@@ -325,9 +406,9 @@ class User(AbstractUser):
     # endregion
 
 
-def init(context):
+def init(context: 'Context') -> List[Awaitable[None]]:
     global config
     config = context.config
 
-    users = [User.from_db(user) for user in DBUser.query.all()]
-    return [user.start(delete_unless_authenticated=True) for user in users]
+    users = [User.from_db(user) for user in DBUser.all()]
+    return [user.try_ensure_started() for user in users if user.tgid]

mautrix_telegram/util/__init__.py

@@ -1,2 +1,7 @@
 from .file_transfer import transfer_file_to_matrix, convert_image
 from .format_duration import format_duration
+from .signed_token import sign_token, verify_token
+from .recursive_dict import recursive_del, recursive_set, recursive_get
+
+def ignore_coro(coro):
+    pass

mautrix_telegram/util/file_transfer.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -14,6 +14,7 @@
 #
 # You should have received a copy of the GNU Affero General Public License
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Optional, Tuple, Union, Dict
 from io import BytesIO
 import time
 import logging
@@ -21,7 +22,17 @@ import asyncio
 
 import magic
 from sqlalchemy.exc import IntegrityError, InvalidRequestError
-from sqlalchemy.orm.exc import FlushError
+
+from telethon.tl.types import (Document, InputFileLocation, InputDocumentFileLocation,
+                               TypePhotoSize, PhotoSize, PhotoCachedSize, InputPhotoFileLocation,
+                               InputPeerPhotoFileLocation)
+from telethon.errors import (AuthBytesInvalidError, AuthKeyInvalidError, LocationInvalidError,
+                             SecurityError, FileIdInvalidError)
+from mautrix_appservice import IntentAPI
+
+from ..tgclient import MautrixTelegramClient
+from ..db import TelegramFile as DBTelegramFile
+from ..util import sane_mimetypes
 
 try:
     from PIL import Image
@@ -36,20 +47,19 @@ try:
 except ImportError:
     VideoFileClip = random = string = os = mimetypes = None
 
-from telethon.tl.types import (Document, FileLocation, InputFileLocation,
-                               InputDocumentFileLocation, PhotoSize, PhotoCachedSize)
-from telethon.errors import LocationInvalidError
+log = logging.getLogger("mau.util")  # type: logging.Logger
 
-from ..db import TelegramFile as DBTelegramFile
-
-log = logging.getLogger("mau.util")
+TypeLocation = Union[Document, InputDocumentFileLocation, InputPeerPhotoFileLocation,
+                     InputFileLocation, InputPhotoFileLocation]
 
 
-def convert_image(file, source_mime="image/webp", target_type="png", thumbnail_to=None):
+def convert_image(file: bytes, source_mime: str = "image/webp", target_type: str = "png",
+                  thumbnail_to: Optional[Tuple[int, int]] = None
+                  ) -> Tuple[str, bytes, Optional[int], Optional[int]]:
     if not Image:
         return source_mime, file, None, None
     try:
-        image = Image.open(BytesIO(file)).convert("RGBA")
+        image = Image.open(BytesIO(file)).convert("RGBA")  # type: Image.Image
         if thumbnail_to:
             image.thumbnail(thumbnail_to, Image.ANTIALIAS)
         new_file = BytesIO()
@@ -61,13 +71,14 @@ def convert_image(file, source_mime="image/webp", target_type="png", thumbnail_t
         return source_mime, file, None, None
 
 
-def _temp_file_name(ext):
+def _temp_file_name(ext: str) -> str:
     return ("/tmp/mxtg-video-"
             + "".join(random.choice(string.ascii_uppercase + string.digits) for _ in range(10))
             + ext)
 
 
-def _read_video_thumbnail(data, video_ext="mp4", frame_ext="png", max_size=(1024, 720)):
+def _read_video_thumbnail(data: bytes, video_ext: str = "mp4", frame_ext: str = "png",
+                          max_size: Tuple[int, int] = (1024, 720)) -> Tuple[bytes, int, int]:
     # We don't have any way to read the video from memory, so save it to disk.
     temp_file = _temp_file_name(video_ext)
     with open(temp_file, "wb") as file:
@@ -90,24 +101,28 @@ def _read_video_thumbnail(data, video_ext="mp4", frame_ext="png", max_size=(1024
     return thumbnail_file.getvalue(), w, h
 
 
-def _location_to_id(location):
-    if isinstance(location, (Document, InputDocumentFileLocation)):
-        return f"{location.id}-{location.version}"
-    elif isinstance(location, (FileLocation, InputFileLocation)):
+def _location_to_id(location: TypeLocation) -> str:
+    if isinstance(location, (Document, InputDocumentFileLocation, InputPhotoFileLocation)):
+        return f"{location.id}-{location.access_hash}"
+    elif isinstance(location, (InputFileLocation, InputPeerPhotoFileLocation)):
         return f"{location.volume_id}-{location.local_id}"
-    else:
-        return None
 
 
-async def transfer_thumbnail_to_matrix(client, intent, thumbnail_loc, video, mime):
+async def transfer_thumbnail_to_matrix(client: MautrixTelegramClient, intent: IntentAPI,
+                                       thumbnail_loc: TypeLocation, video: bytes,
+                                       mime: str) -> Optional[DBTelegramFile]:
     if not Image or not VideoFileClip:
         return None
 
-    id = _location_to_id(thumbnail_loc)
-    if not id:
+    loc_id = _location_to_id(thumbnail_loc)
+    if not loc_id:
         return None
 
-    video_ext = mimetypes.guess_extension(mime)
+    db_file = DBTelegramFile.get(loc_id)
+    if db_file:
+        return db_file
+
+    video_ext = sane_mimetypes.guess_extension(mime)
     if VideoFileClip and video_ext:
         try:
             file, width, height = _read_video_thumbnail(video, video_ext, frame_ext="png")
@@ -115,63 +130,81 @@ async def transfer_thumbnail_to_matrix(client, intent, thumbnail_loc, video, mim
             return None
         mime_type = "image/png"
     else:
-        file = await client.download_file_bytes(thumbnail_loc)
+        file = await client.download_file(thumbnail_loc)
         width, height = None, None
         mime_type = magic.from_buffer(file, mime=True)
 
-    uploaded = await intent.upload_file(file, mime_type)
+    content_uri = await intent.upload_file(file, mime_type)
 
-    return DBTelegramFile(id=id, mxc=uploaded["content_uri"], mime_type=mime_type,
-                          was_converted=False, timestamp=int(time.time()), size=len(file),
-                          width=width, height=height)
+    db_file = DBTelegramFile(id=loc_id, mxc=content_uri, mime_type=mime_type,
+                             was_converted=False, timestamp=int(time.time()), size=len(file),
+                             width=width, height=height)
+    try:
+        db_file.insert()
+    except (IntegrityError, InvalidRequestError) as e:
+        log.exception(f"{e.__class__.__name__} while saving transferred file thumbnail data. "
+                      "This was probably caused by two simultaneous transfers of the same file, "
+                      "and might (but probably won't) cause problems with thumbnails or something.")
+    return db_file
 
 
-transfer_locks = {}
-transfer_locks_lock = asyncio.Lock()
+transfer_locks = {}  # type: Dict[str, asyncio.Lock]
+
+TypeThumbnail = Optional[Union[TypeLocation, TypePhotoSize]]
 
 
-async def transfer_file_to_matrix(db, client, intent, location, thumbnail=None, is_sticker=False):
-    id = _location_to_id(location)
-    if not id:
+async def transfer_file_to_matrix(client: MautrixTelegramClient, intent: IntentAPI,
+                                  location: TypeLocation, thumbnail: TypeThumbnail = None,
+                                  is_sticker: bool = False) -> Optional[DBTelegramFile]:
+    location_id = _location_to_id(location)
+    if not location_id:
         return None
 
-    db_file = DBTelegramFile.query.get(id)
-    if db_file:
-        return db_file
-
-    async with transfer_locks_lock:
-        try:
-            lock = transfer_locks[id]
-        except KeyError:
-            lock = asyncio.Lock()
-            transfer_locks[id] = lock
-    async with lock:
-        return await _unlocked_transfer_file_to_matrix(db, client, intent, id, location, thumbnail, is_sticker)
-
-
-async def _unlocked_transfer_file_to_matrix(db, client, intent, id, location, thumbnail, is_sticker):
-    db_file = DBTelegramFile.query.get(id)
+    db_file = DBTelegramFile.get(location_id)
     if db_file:
         return db_file
 
     try:
-        file = await client.download_file_bytes(location)
-    except LocationInvalidError:
+        lock = transfer_locks[location_id]
+    except KeyError:
+        lock = asyncio.Lock()
+        transfer_locks[location_id] = lock
+    async with lock:
+        return await _unlocked_transfer_file_to_matrix(client, intent, location_id, location,
+                                                       thumbnail, is_sticker)
+
+
+async def _unlocked_transfer_file_to_matrix(client: MautrixTelegramClient, intent: IntentAPI,
+                                            loc_id: str, location: TypeLocation,
+                                            thumbnail: TypeThumbnail, is_sticker: bool
+                                            ) -> Optional[DBTelegramFile]:
+    db_file = DBTelegramFile.get(loc_id)
+    if db_file:
+        return db_file
+
+    try:
+        file = await client.download_file(location)
+    except (LocationInvalidError, FileIdInvalidError):
         return None
+    except (AuthBytesInvalidError, AuthKeyInvalidError, SecurityError) as e:
+        log.exception(f"{e.__class__.__name__} while downloading a file.")
+        return None
+
     width, height = None, None
     mime_type = magic.from_buffer(file, mime=True)
 
     image_converted = False
     if mime_type == "image/webp":
-        new_mime_type, file, width, height = convert_image(file, source_mime="image/webp", target_type="png", thumbnail_to=(
-            256, 256) if is_sticker else None)
+        new_mime_type, file, width, height = convert_image(
+            file, source_mime="image/webp", target_type="png",
+            thumbnail_to=(256, 256) if is_sticker else None)
         image_converted = new_mime_type != mime_type
         mime_type = new_mime_type
         thumbnail = None
 
-    uploaded = await intent.upload_file(file, mime_type)
+    content_uri = await intent.upload_file(file, mime_type)
 
-    db_file = DBTelegramFile(id=id, mxc=uploaded["content_uri"],
+    db_file = DBTelegramFile(id=loc_id, mxc=content_uri,
                              mime_type=mime_type, was_converted=image_converted,
                              timestamp=int(time.time()), size=len(file),
                              width=width, height=height)
@@ -182,16 +215,9 @@ async def _unlocked_transfer_file_to_matrix(db, client, intent, id, location, th
                                                                mime_type)
 
     try:
-        db.add(db_file)
-        db.commit()
-    except FlushError as e:
-        log.exception(f"{e.__class__.__name__} while saving transferred file data. "
-                      "This was probably caused by two simultaneous transfers of the same file, "
-                      "and should not cause any problems.")
+        db_file.insert()
     except (IntegrityError, InvalidRequestError) as e:
-        db.rollback()
         log.exception(f"{e.__class__.__name__} while saving transferred file data. "
                       "This was probably caused by two simultaneous transfers of the same file, "
                       "and should not cause any problems.")
-
     return db_file

mautrix_telegram/util/format_duration.py

@@ -1,6 +1,6 @@
 # -*- coding: future_fstrings -*-
 # mautrix-telegram - A Matrix-Telegram puppeting bridge
-# Copyright (C) 2018 Tulir Asokan
+# Copyright (C) 2019 Tulir Asokan
 #
 # This program is free software: you can redistribute it and/or modify
 # it under the terms of the GNU Affero General Public License as published by
@@ -16,10 +16,12 @@
 # along with this program.  If not, see <https://www.gnu.org/licenses/>.
 
 
-def format_duration(seconds):
-    def pluralize(count, singular): return singular if count == 1 else singular + "s"
+def format_duration(seconds: int) -> str:
+    def pluralize(count: int, singular: str) -> str:
+        return singular if count == 1 else singular + "s"
 
-    def include(count, word): return f"{count} {pluralize(count, word)}" if count > 0 else ""
+    def include(count: int, word: str) -> str:
+        return f"{count} {pluralize(count, word)}" if count > 0 else ""
 
     minutes, seconds = divmod(seconds, 60)
     hours, minutes = divmod(minutes, 60)

mautrix_telegram/util/recursive_dict.py

@@ -0,0 +1,54 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Dict, Any
+from ..config import DictWithRecursion
+
+
+def recursive_set(data: Dict[str, Any], key: str, value: Any) -> bool:
+    key, next_key = DictWithRecursion._parse_key(key)
+    if next_key is not None:
+        if key not in data:
+            data[key] = {}
+        next_data = data.get(key, {})
+        if not isinstance(next_data, dict):
+            return False
+        return recursive_set(next_data, next_key, value)
+    data[key] = value
+    return True
+
+
+def recursive_get(data: Dict[str, Any], key: str) -> Any:
+    key, next_key = DictWithRecursion._parse_key(key)
+    if next_key is not None:
+        next_data = data.get(key, None)
+        if not next_data:
+            return None
+        return recursive_get(next_data, next_key)
+    return data.get(key, None)
+
+
+def recursive_del(data: Dict[str, any], key: str) -> bool:
+    key, next_key = DictWithRecursion._parse_key(key)
+    if next_key is not None:
+        if key not in data:
+            return False
+        next_data = data.get(key, {})
+        return recursive_del(next_data, next_key)
+    if key in data:
+        del data[key]
+        return True
+    return False

mautrix_telegram/util/sane_mimetypes.py

@@ -0,0 +1,38 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+import mimetypes
+
+mimetypes.init()
+
+sanity_overrides = {
+    "image/jpeg": ".jpeg",
+    "image/tiff": ".tiff",
+    "text/plain": ".txt",
+    "text/html": ".html",
+    "audio/mpeg": ".mp3",
+    "audio/ogg": ".ogg",
+    "application/xml": ".xml",
+    "application/octet-stream": "",
+    "application/x-msdos-program": ".exe",
+}
+
+
+def guess_extension(mime: str) -> str:
+    try:
+        return sanity_overrides[mime]
+    except KeyError:
+        return mimetypes.guess_extension(mime)

mautrix_telegram/util/signed_token.py

@@ -0,0 +1,53 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Dict, Optional
+import json
+import base64
+import hashlib
+
+
+def _get_checksum(key: str, payload: bytes) -> str:
+    hasher = hashlib.sha256()
+    hasher.update(payload)
+    hasher.update(key.encode("utf-8"))
+    checksum = hasher.hexdigest()
+    return checksum
+
+
+def sign_token(key: str, payload: Dict) -> str:
+    payload_b64 = base64.urlsafe_b64encode(json.dumps(payload).encode("utf-8"))
+    checksum = _get_checksum(key, payload_b64)
+    return f"{checksum}:{payload_b64.decode('utf-8')}"
+
+
+def verify_token(key: str, data: str) -> Optional[Dict]:
+    if not data:
+        return None
+
+    try:
+        checksum, payload = data.split(":", 1)
+    except ValueError:
+        return None
+
+    if checksum != _get_checksum(key, payload.encode("utf-8")):
+        return None
+
+    payload = base64.urlsafe_b64decode(payload).decode("utf-8")
+    try:
+        return json.loads(payload)
+    except json.JSONDecodeError:
+        return None

mautrix_telegram/web/__init__.py

@@ -0,0 +1,2 @@
+from .provisioning import ProvisioningAPI
+from .public import PublicBridgeWebsite

mautrix_telegram/web/common/__init__.py

@@ -0,0 +1 @@
+from .auth_api import AuthAPI

mautrix_telegram/web/common/auth_api.py

@@ -0,0 +1,199 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from abc import abstractmethod
+from typing import Optional
+
+from aiohttp import web
+import abc
+import asyncio
+import logging
+
+from telethon.errors import *
+
+from ...commands.telegram.auth import enter_password
+from ...util import format_duration, ignore_coro
+from ...puppet import Puppet, PuppetError
+from ...user import User
+
+
+class AuthAPI(abc.ABC):
+    log = logging.getLogger("mau.web.auth")  # type: logging.Logger
+
+    def __init__(self, loop: asyncio.AbstractEventLoop):
+        self.loop = loop  # type: asyncio.AbstractEventLoop
+
+    @abstractmethod
+    def get_login_response(self, status: int = 200, state: str = "", username: str = "",
+                           phone: str = "", human_tg_id: str = "", mxid: str = "",
+                           message: str = "", error: str = "", errcode: str = "") -> web.Response:
+        raise NotImplementedError()
+
+    @abstractmethod
+    def get_mx_login_response(self, status: int = 200, state: str = "", username: str = "",
+                              phone: str = "", human_tg_id: str = "", mxid: str = "",
+                              message: str = "", error: str = "", errcode: str = ""
+                              ) -> web.Response:
+        raise NotImplementedError()
+
+    async def post_matrix_token(self, user: User, token: str) -> web.Response:
+        puppet = Puppet.get(user.tgid)
+        if puppet.is_real_user:
+            return self.get_mx_login_response(state="already-logged-in", status=409,
+                                              error="You have already logged in with your Matrix "
+                                                    "account.", errcode="already-logged-in")
+
+        resp = await puppet.switch_mxid(token.strip(), user.mxid)
+        if resp == PuppetError.OnlyLoginSelf:
+            return self.get_mx_login_response(status=403, errcode="only-login-self",
+                                              error="You can only log in as your own Matrix user.")
+        elif resp == PuppetError.InvalidAccessToken:
+            return self.get_mx_login_response(status=401, errcode="invalid-access-token",
+                                              error="Failed to verify access token.")
+        assert resp == PuppetError.Success, "Encountered an unhandled PuppetError."
+
+        return self.get_mx_login_response(mxid=user.mxid, status=200, state="logged-in")
+
+    async def post_matrix_password(self, user: User, password: str) -> web.Response:
+        return self.get_mx_login_response(mxid=user.mxid, status=501, error="Not yet implemented",
+                                          errcode="not-yet-implemented")
+
+    async def post_login_phone(self, user: User, phone: str) -> web.Response:
+        if not phone or not phone.strip():
+            return self.get_login_response(mxid=user.mxid, state="request", status=400,
+                                           errcode="phone_number_invalid",
+                                           error="Phone number not given.")
+        try:
+            await user.client.sign_in(phone.strip())
+            return self.get_login_response(mxid=user.mxid, state="code", status=200,
+                                           message="Code requested successfully. Check your SMS "
+                                                   "or Telegram client and enter the code below.")
+        except PhoneNumberInvalidError:
+            return self.get_login_response(mxid=user.mxid, state="request", status=400,
+                                           errcode="phone_number_invalid",
+                                           error="Invalid phone number.")
+        except PhoneNumberBannedError:
+            return self.get_login_response(mxid=user.mxid, state="request", status=403,
+                                           errcode="phone_number_banned",
+                                           error="Your phone number is banned from Telegram.")
+        except PhoneNumberAppSignupForbiddenError:
+            return self.get_login_response(mxid=user.mxid, state="request", status=403,
+                                           errcode="phone_number_app_signup_forbidden",
+                                           error="You have disabled 3rd party apps on your "
+                                                 "account.")
+        except PhoneNumberUnoccupiedError:
+            return self.get_login_response(mxid=user.mxid, state="request", status=404,
+                                           errcode="phone_number_unoccupied",
+                                           error="That phone number has not been registered.")
+        except PhoneNumberFloodError:
+            return self.get_login_response(
+                mxid=user.mxid, state="request", status=429, errcode="phone_number_flood",
+                error="Your phone number has been temporarily blocked for flooding. "
+                      "The ban is usually applied for around a day.")
+        except FloodWaitError as e:
+            return self.get_login_response(
+                mxid=user.mxid, state="request", status=429, errcode="flood_wait",
+                error="Your phone number has been temporarily blocked for flooding. "
+                      f"Please wait for {format_duration(e.seconds)} before trying again.")
+        except Exception:
+            self.log.exception("Error requesting phone code")
+            return self.get_login_response(mxid=user.mxid, state="request", status=500,
+                                           errcode="unknown_error",
+                                           error="Internal server error while requesting code.")
+
+    async def postprocess_login(self, user: User, user_info) -> None:
+        existing_user = User.get_by_tgid(user_info.id)
+        if existing_user and existing_user != user:
+            await existing_user.log_out()
+        ignore_coro(asyncio.ensure_future(user.post_login(user_info), loop=self.loop))
+        if user.command_status and user.command_status["action"] == "Login":
+            user.command_status = None
+
+    async def post_login_token(self, user: User, token: str) -> web.Response:
+        try:
+            user_info = await user.client.sign_in(bot_token=token.strip())
+            await self.postprocess_login(user, user_info)
+            return self.get_login_response(mxid=user.mxid, state="logged-in", status=200,
+                                           username=user_info.username, phone=None,
+                                           human_tg_id=f"@{user_info.username}")
+        except AccessTokenInvalidError:
+            return self.get_login_response(mxid=user.mxid, state="token", status=401,
+                                           errcode="bot_token_invalid",
+                                           error="Bot token invalid.")
+        except AccessTokenExpiredError:
+            return self.get_login_response(mxid=user.mxid, state="token", status=403,
+                                           errcode="bot_token_expired",
+                                           error="Bot token expired.")
+        except Exception:
+            self.log.exception("Error sending bot token")
+            return self.get_login_response(mxid=user.mxid, state="token", status=500,
+                                           error="Internal server error while sending token.")
+
+    async def post_login_code(self, user: User, code: int, password_in_data: bool
+                              ) -> Optional[web.Response]:
+        try:
+            user_info = await user.client.sign_in(code=code)
+            await self.postprocess_login(user, user_info)
+            human_tg_id = f"@{user_info.username}" if user_info.username else f"+{user_info.phone}"
+            return self.get_login_response(mxid=user.mxid, state="logged-in", status=200,
+                                           username=user_info.username, phone=user_info.phone,
+                                           human_tg_id=human_tg_id)
+        except PhoneCodeInvalidError:
+            return self.get_login_response(mxid=user.mxid, state="code", status=401,
+                                           errcode="phone_code_invalid",
+                                           error="Incorrect phone code.")
+        except PhoneCodeExpiredError:
+            return self.get_login_response(mxid=user.mxid, state="code", status=403,
+                                           errcode="phone_code_expired",
+                                           error="Phone code expired.")
+        except SessionPasswordNeededError:
+            if not password_in_data:
+                if user.command_status and user.command_status["action"] == "Login":
+                    user.command_status = {
+                        "next": enter_password,
+                        "action": "Login (password entry)",
+                    }
+                return self.get_login_response(
+                    mxid=user.mxid, state="password", status=202,
+                    message="Code accepted, but you have 2-factor authentication is enabled.")
+            return None
+        except Exception:
+            self.log.exception("Error sending phone code")
+            return self.get_login_response(mxid=user.mxid, state="code", status=500,
+                                           errcode="unknown_error",
+                                           error="Internal server error while sending code.")
+
+    async def post_login_password(self, user: User, password: str) -> web.Response:
+        try:
+            user_info = await user.client.sign_in(password=password.strip())
+            await self.postprocess_login(user, user_info)
+            human_tg_id = f"@{user_info.username}" if user_info.username else f"+{user_info.phone}"
+            return self.get_login_response(mxid=user.mxid, state="logged-in", status=200,
+                                           username=user_info.username, phone=user_info.phone,
+                                           human_tg_id=human_tg_id)
+        except PasswordEmptyError:
+            return self.get_login_response(mxid=user.mxid, state="password", status=400,
+                                           errcode="password_empty",
+                                           error="Empty password.")
+        except PasswordHashInvalidError:
+            return self.get_login_response(mxid=user.mxid, state="password", status=401,
+                                           errcode="password_invalid",
+                                           error="Incorrect password.")
+        except Exception:
+            self.log.exception("Error sending password")
+            return self.get_login_response(mxid=user.mxid, state="password", status=500,
+                                           errcode="unknown_error",
+                                           error="Internal server error while sending password.")

mautrix_telegram/web/provisioning/__init__.py

@@ -0,0 +1,480 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from aiohttp import web
+from typing import Awaitable, Callable, Dict, Optional, Tuple, TYPE_CHECKING
+import asyncio
+import logging
+import json
+
+from telethon.utils import get_peer_id, resolve_id
+from telethon.tl.types import ChatForbidden, ChannelForbidden, TypeChat
+from mautrix_appservice import AppService, MatrixRequestError, IntentError
+
+from ...types import MatrixUserID, TelegramID
+from ...user import User
+from ...portal import Portal
+from ...util import ignore_coro
+from ...commands.portal.util import user_has_power_level, get_initial_state
+from ..common import AuthAPI
+
+if TYPE_CHECKING:
+    from ...context import Context
+
+
+class ProvisioningAPI(AuthAPI):
+    log = logging.getLogger("mau.web.provisioning")  # type: logging.Logger
+
+    def __init__(self, context: "Context") -> None:
+        super().__init__(context.loop)
+        self.secret = context.config["appservice.provisioning.shared_secret"]  # type: str
+        self.az = context.az  # type: AppService
+        self.context = context  # type: Context
+
+        self.app = web.Application(loop=context.loop, middlewares=[self.error_middleware]
+                                   )  # type: web.Application
+
+        portal_prefix = "/portal/{mxid:![^/]+}"
+        self.app.router.add_route("GET", f"{portal_prefix}", self.get_portal_by_mxid)
+        self.app.router.add_route("GET", "/portal/{tgid:-[0-9]+}", self.get_portal_by_tgid)
+        self.app.router.add_route("POST", portal_prefix + "/connect/{chat_id:-[0-9]+}",
+                                  self.connect_chat)
+        self.app.router.add_route("POST", f"{portal_prefix}/create", self.create_chat)
+        self.app.router.add_route("POST", f"{portal_prefix}/disconnect", self.disconnect_chat)
+
+        user_prefix = "/user/{mxid:@[^:]*:[^/]+}"
+        self.app.router.add_route("GET", f"{user_prefix}", self.get_user_info)
+        self.app.router.add_route("GET", f"{user_prefix}/chats", self.get_chats)
+
+        self.app.router.add_route("POST", f"{user_prefix}/logout", self.logout)
+        self.app.router.add_route("POST", f"{user_prefix}/login/bot_token", self.send_bot_token)
+        self.app.router.add_route("POST", f"{user_prefix}/login/request_code", self.request_code)
+        self.app.router.add_route("POST", f"{user_prefix}/login/send_code", self.send_code)
+        self.app.router.add_route("POST", f"{user_prefix}/login/send_password", self.send_password)
+
+        self.app.router.add_route("GET", "/bridge", self.bridge_info)
+
+    async def get_portal_by_mxid(self, request: web.Request) -> web.Response:
+        err = self.check_authorization(request)
+        if err is not None:
+            return err
+
+        mxid = request.match_info["mxid"]
+        portal = Portal.get_by_mxid(mxid)
+        if not portal:
+            return self.get_error_response(404, "portal_not_found",
+                                           "Portal with given Matrix ID not found.")
+        user, _ = await self.get_user(request.query.get("user_id", None), expect_logged_in=None,
+                                      require_puppeting=False)
+        return web.json_response({
+            "mxid": portal.mxid,
+            "chat_id": get_peer_id(portal.peer),
+            "peer_type": portal.peer_type,
+            "title": portal.title,
+            "about": portal.about,
+            "username": portal.username,
+            "megagroup": portal.megagroup,
+            "can_unbridge": (await portal.can_user_perform(user, "unbridge")) if user else False,
+        })
+
+    async def get_portal_by_tgid(self, request: web.Request) -> web.Response:
+        err = self.check_authorization(request)
+        if err is not None:
+            return err
+
+        try:
+            tgid, _ = resolve_id(int(request.match_info["tgid"]))
+        except ValueError:
+            return self.get_error_response(400, "tgid_invalid",
+                                           "Given chat ID is not valid.")
+        portal = Portal.get_by_tgid(tgid)
+        if not portal:
+            return self.get_error_response(404, "portal_not_found",
+                                           "Portal to given Telegram chat not found.")
+        user, _ = await self.get_user(request.query.get("user_id", None), expect_logged_in=None,
+                                      require_puppeting=False)
+        return web.json_response({
+            "mxid": portal.mxid,
+            "chat_id": get_peer_id(portal.peer),
+            "peer_type": portal.peer_type,
+            "title": portal.title,
+            "about": portal.about,
+            "username": portal.username,
+            "megagroup": portal.megagroup,
+            "can_unbridge": (await portal.can_user_perform(user, "unbridge")) if user else False,
+        })
+
+    async def connect_chat(self, request: web.Request) -> web.Response:
+        err = self.check_authorization(request)
+        if err is not None:
+            return err
+
+        room_id = request.match_info["mxid"]
+        if Portal.get_by_mxid(room_id):
+            return self.get_error_response(409, "room_already_bridged",
+                                           "Room is already bridged to another Telegram chat.")
+
+        chat_id = request.match_info["chat_id"]
+        if chat_id.startswith("-100"):
+            tgid = TelegramID(int(chat_id[4:]))
+            peer_type = "channel"
+        elif chat_id.startswith("-"):
+            tgid = TelegramID(-int(chat_id))
+            peer_type = "chat"
+        else:
+            return self.get_error_response(400, "tgid_invalid", "Invalid Telegram chat ID.")
+
+        user, err = await self.get_user(request.query.get("user_id", None), expect_logged_in=None,
+                                        require_puppeting=False)
+        if err is not None:
+            return err
+        elif user and not await user_has_power_level(room_id, self.az.intent, user, "bridge"):
+            return self.get_error_response(403, "not_enough_permissions",
+                                           "You do not have the permissions to bridge that room.")
+
+        portal = Portal.get_by_tgid(tgid, peer_type=peer_type)
+        if portal.mxid == room_id:
+            return self.get_error_response(200, "bridge_exists",
+                                           "Telegram chat is already bridged to that Matrix room.")
+        elif portal.mxid:
+            force = request.query.get("force", None)
+            if force in ("delete", "unbridge"):
+                delete = force == "delete"
+                await portal.cleanup_room(portal.main_intent, portal.mxid, puppets_only=not delete,
+                                          message=("Portal deleted (moving to another room)"
+                                                   if delete
+                                                   else "Room unbridged (portal moving to another "
+                                                        "room)"))
+            else:
+                return self.get_error_response(409, "chat_already_bridged",
+                                               "Telegram chat is already bridged to another "
+                                               "Matrix room.")
+
+        is_logged_in = user is not None and await user.is_logged_in()
+        acting_user = user if is_logged_in else self.context.bot
+        if not acting_user:
+            return self.get_login_response(status=403, errcode="not_logged_in",
+                                           error="You are not logged in and there is no relay bot.")
+
+        entity = None  # type: Optional[TypeChat]
+        try:
+            entity = await acting_user.client.get_entity(portal.peer)
+        except Exception:
+            self.log.exception("Failed to get_entity(%s) for manual bridging.", portal.peer)
+
+        if not entity or isinstance(entity, (ChatForbidden, ChannelForbidden)):
+            if is_logged_in:
+                return self.get_error_response(403, "user_not_in_chat",
+                                               "Failed to get info of Telegram chat. "
+                                               "Are you in the chat?")
+            return self.get_error_response(403, "bot_not_in_chat",
+                                           "Failed to get info of Telegram chat. "
+                                           "Is the relay bot in the chat?")
+
+        direct = False
+
+        portal.mxid = room_id
+        portal.title, portal.about, levels = await get_initial_state(self.az.intent, room_id)
+        portal.photo_id = ""
+        portal.save()
+
+        ignore_coro(asyncio.ensure_future(portal.update_matrix_room(user, entity, direct,
+                                                                    levels=levels),
+                                          loop=self.loop))
+
+        return web.Response(status=202, body="{}")
+
+    async def create_chat(self, request: web.Request) -> web.Response:
+        err = self.check_authorization(request)
+        if err is not None:
+            return err
+
+        data = await self.get_data(request)
+        if not data:
+            return self.get_error_response(400, "json_invalid", "Invalid JSON.")
+
+        room_id = request.match_info["mxid"]
+        if Portal.get_by_mxid(room_id):
+            return self.get_error_response(409, "room_already_bridged",
+                                           "Room is already bridged to another Telegram chat.")
+
+        user, err = await self.get_user(request.query.get("user_id", None), expect_logged_in=None,
+                                        require_puppeting=False)
+        if err is not None:
+            return err
+        elif not await user.is_logged_in() or user.is_bot:
+            return self.get_error_response(403, "not_logged_in_real_account",
+                                           "You are not logged in with a real account.")
+        elif not await user_has_power_level(room_id, self.az.intent, user, "bridge"):
+            return self.get_error_response(403, "not_enough_permissions",
+                                           "You do not have the permissions to bridge that room.")
+
+        try:
+            title, about, _ = await get_initial_state(self.az.intent, room_id)
+        except (MatrixRequestError, IntentError):
+            return self.get_error_response(403, "bot_not_in_room",
+                                           "The bridge bot is not in the given room.")
+
+        about = data.get("about", about)
+
+        title = data.get("title", title)
+        if len(title) == 0:
+            return self.get_error_response(400, "body_value_invalid", "Title can not be empty.")
+
+        type = data.get("type", "")
+        if type not in ("group", "chat", "supergroup", "channel"):
+            return self.get_error_response(400, "body_value_invalid",
+                                           "Given chat type is not valid.")
+
+        supergroup = type == "supergroup"
+        type = {
+            "supergroup": "channel",
+            "channel": "channel",
+            "chat": "chat",
+            "group": "chat",
+        }[type]
+
+        portal = Portal(tgid=TelegramID(0), mxid=room_id, title=title, about=about, peer_type=type)
+        try:
+            await portal.create_telegram_chat(user, supergroup=supergroup)
+        except ValueError as e:
+            portal.delete()
+            return self.get_error_response(500, "unknown_error", e.args[0])
+
+        return web.json_response({
+            "chat_id": portal.tgid,
+        }, status=201)
+
+    async def disconnect_chat(self, request: web.Request) -> web.Response:
+        err = self.check_authorization(request)
+        if err is not None:
+            return err
+
+        portal = Portal.get_by_mxid(request.match_info["mxid"])
+        if not portal or not portal.tgid:
+            return self.get_error_response(404, "portal_not_found",
+                                           "Room is not a portal.")
+
+        user, err = await self.get_user(request.query.get("user_id", None), expect_logged_in=None,
+                                        require_puppeting=False, require_user=False)
+        if err is not None:
+            return err
+        elif user and not await user_has_power_level(portal.mxid, self.az.intent, user, "unbridge"):
+            return self.get_error_response(403, "not_enough_permissions",
+                                           "You do not have the permissions to unbridge that room.")
+
+        delete = request.query.get("delete", "").lower() in ("true", "t", "1", "yes", "y")
+        sync = request.query.get("delete", "").lower() in ("true", "t", "1", "yes", "y")
+
+        coro = portal.cleanup_and_delete() if delete else portal.unbridge()
+        if sync:
+            try:
+                await coro
+            except Exception:
+                self.log.exception("Failed to disconnect chat")
+                return self.get_error_response(500, "exception", "Failed to disconnect chat")
+        else:
+            ignore_coro(asyncio.ensure_future(coro, loop=self.loop))
+        return web.json_response({}, status=200 if sync else 202)
+
+    async def get_user_info(self, request: web.Request) -> web.Response:
+        data, user, err = await self.get_user_request_info(request, expect_logged_in=None,
+                                                           require_puppeting=False)
+        if err is not None:
+            return err
+
+        user_data = None
+        if await user.is_logged_in():
+            me = await user.client.get_me()
+            await user.update_info(me)
+            user_data = {
+                "id": user.tgid,
+                "username": user.username,
+                "first_name": me.first_name,
+                "last_name": me.last_name,
+                "phone": me.phone,
+                "is_bot": user.is_bot,
+            }
+        return web.json_response({
+            "telegram": user_data,
+            "mxid": user.mxid,
+            "permissions": user.permissions,
+        })
+
+    async def get_chats(self, request: web.Request) -> web.Response:
+        data, user, err = await self.get_user_request_info(request, expect_logged_in=True)
+        if err is not None:
+            return err
+
+        if not user.is_bot:
+            chats = await user.get_dialogs()
+            return web.json_response([{
+                "id": get_peer_id(chat),
+                "title": chat.title,
+            } for chat in chats])
+        else:
+            return web.json_response([{
+                "id": get_peer_id(chat.peer),
+                "title": chat.title,
+            } for chat in user.portals.values() if chat.tgid])
+
+    async def send_bot_token(self, request: web.Request) -> web.Response:
+        data, user, err = await self.get_user_request_info(request)
+        if err is not None:
+            return err
+        return await self.post_login_token(user, data.get("token", ""))
+
+    async def request_code(self, request: web.Request) -> web.Response:
+        data, user, err = await self.get_user_request_info(request)
+        if err is not None:
+            return err
+        return await self.post_login_phone(user, data.get("phone", ""))
+
+    async def send_code(self, request: web.Request) -> web.Response:
+        data, user, err = await self.get_user_request_info(request)
+        if err is not None:
+            return err
+        return await self.post_login_code(user, data.get("code", 0), password_in_data=False)
+
+    async def send_password(self, request: web.Request) -> web.Response:
+        data, user, err = await self.get_user_request_info(request)
+        if err is not None:
+            return err
+        return await self.post_login_password(user, data.get("password", ""))
+
+    async def logout(self, request: web.Request) -> web.Response:
+        _, user, err = await self.get_user_request_info(request, expect_logged_in=True,
+                                                        require_puppeting=False,
+                                                        want_data=False)
+        if err is not None:
+            return err
+        await user.log_out()
+
+    async def bridge_info(self, request: web.Request) -> web.Response:
+        return web.json_response({
+            "relaybot_username": self.context.bot.username if self.context.bot is not None else None,
+        }, status=200)
+
+    @staticmethod
+    async def error_middleware(_, handler: Callable[[web.Request], Awaitable[web.Response]]
+                               ) -> Callable[[web.Request], Awaitable[web.Response]]:
+        async def middleware_handler(request: web.Request) -> web.Response:
+            try:
+                return await handler(request)
+            except web.HTTPException as ex:
+                return web.json_response({
+                    "error": f"Unhandled HTTP {ex.status}",
+                    "errcode": f"unhandled_http_{ex.status}",
+                }, status=ex.status)
+
+        return middleware_handler
+
+    @staticmethod
+    def get_error_response(status=200, errcode="", error="") -> web.Response:
+        return web.json_response({
+            "error": error,
+            "errcode": errcode,
+        }, status=status)
+
+    def get_mx_login_response(self, status=200, state="", username="", phone="", human_tg_id="",
+                              mxid="", message="", error="", errcode=""):
+        raise NotImplementedError()
+
+    def get_login_response(self, status=200, state="", username="", phone: str = "",
+                           human_tg_id: str = "", mxid="", message="", error="", errcode=""
+                           ) -> web.Response:
+        if username or phone:
+            resp = {
+                "state": "logged-in",
+                "username": username,
+                "phone": phone,
+            }
+        elif message:
+            resp = {
+                "state": state,
+                "message": message,
+            }
+        else:
+            resp = {
+                "error": error,
+                "errcode": errcode,
+            }
+            if state:
+                resp["state"] = state
+        return web.json_response(resp, status=status)
+
+    def check_authorization(self, request: web.Request) -> Optional[web.Response]:
+        auth = request.headers.get("Authorization", "")
+        if auth != f"Bearer {self.secret}":
+            return self.get_error_response(error="Shared secret is not valid.",
+                                           errcode="shared_secret_invalid",
+                                           status=401)
+        return None
+
+    @staticmethod
+    async def get_data(request: web.Request) -> Optional[dict]:
+        try:
+            return await request.json()
+        except json.JSONDecodeError:
+            return None
+
+    async def get_user(self, mxid: MatrixUserID, expect_logged_in: Optional[bool] = False,
+                       require_puppeting: bool = True, require_user: bool = True
+                       ) -> Tuple[Optional[User], Optional[web.Response]]:
+        if not mxid:
+            if not require_user:
+                return None, None
+            return None, self.get_login_response(error="User ID not given.",
+                                                 errcode="mxid_empty", status=400)
+
+        user = await User.get_by_mxid(mxid).ensure_started(even_if_no_session=True)
+        if require_puppeting and not user.puppet_whitelisted:
+            return user, self.get_login_response(error="You are not whitelisted.",
+                                                 errcode="mxid_not_whitelisted", status=403)
+        if expect_logged_in is not None:
+            logged_in = await user.is_logged_in()
+            if not expect_logged_in and logged_in:
+                return user, self.get_login_response(username=user.username, phone=user.phone,
+                                                     status=409,
+                                                     error="You are already logged in.",
+                                                     errcode="already_logged_in")
+            elif expect_logged_in and not logged_in:
+                return user, self.get_login_response(status=403, error="You are not logged in.",
+                                                     errcode="not_logged_in")
+        return user, None
+
+    async def get_user_request_info(self, request: web.Request,
+                                    expect_logged_in: Optional[bool] = False,
+                                    require_puppeting: bool = False,
+                                    want_data: bool = True,
+                                    ) -> (Tuple[Optional[Dict],
+                                                Optional[User],
+                                                Optional[web.Response]]):
+        err = self.check_authorization(request)
+        if err is not None:
+            return err
+
+        data = None
+        if want_data and (request.method == "POST" or request.method == "PUT"):
+            data = await self.get_data(request)
+            if not data:
+                return None, None, self.get_login_response(error="Invalid JSON.",
+                                                           errcode="json_invalid", status=400)
+
+        mxid = request.match_info["mxid"]
+        user, err = await self.get_user(mxid, expect_logged_in, require_puppeting)
+
+        return data, user, err

mautrix_telegram/web/provisioning/spec.yaml

@@ -0,0 +1,895 @@
+swagger: "2.0"
+
+info:
+  title: Mautrix-Telegram provisioning
+  version: 0.3.0
+  description: The provisioning API for Mautrix-Telegram, the Matrix-Telegram puppeting/relaybot bridge.
+  license:
+    name: AGPLv3
+    url: https://github.com/tulir/mautrix-telegram/blob/master/LICENSE
+
+externalDocs:
+  description: Provisioning API wiki page on GitHub
+  url: https://github.com/tulir/mautrix-telegram/wiki/Provisioning-API
+
+basePath: /_matrix/provision/v1
+
+schemes: [https]
+consumes: [application/json]
+produces: [application/json]
+
+tags:
+- name: User info
+- name: Authentication
+- name: Bridging
+- name: Misc
+
+paths:
+  /bridge:
+    get:
+      operationId: get_bridge
+      summary: Get the bridge's information
+      tags: [Misc]
+      responses:
+        200:
+          description: The bridge information
+          schema:
+            type: object
+            properties:
+              relaybot_username:
+                type: string
+                description: The relay bot's username on Telegram
+  /portal/{room_id}:
+    get:
+      operationId: get_portal
+      summary: Get the bridging status and info of the connected Telegram chat
+      tags: [Bridging]
+      responses:
+        200:
+          description: Room is bridged
+          schema:
+            $ref: "#/definitions/PortalInfo"
+        400:
+          $ref: "#/responses/BadRequest"
+        404:
+          description: Unknown portal
+          schema:
+            type: object
+            title: Error
+            properties:
+              errcode:
+                type: string
+                title: Error code
+                description: A machine-readable error code
+                enum:
+                - portal_not_found
+              error:
+                $ref: "#/definitions/HumanReadableError"
+      parameters:
+        - name: room_id
+          in: path
+          description: The Matrix ID of the room whose bridging status to get
+          required: true
+          type: string
+          pattern: "![^/]+"
+        - name: user_id
+          in: query
+          description: Optional Matrix user ID to check if the user has permissions to do bridging.
+          required: false
+          type: string
+  /portal/{chat_id}:
+    get:
+      operationId: get_portal_by_tgid
+      summary: Get the bridging status and info of the connected Telegram chat
+      tags: [Bridging]
+      responses:
+        200:
+          description: Chat is bridged
+          schema:
+            $ref: "#/definitions/PortalInfo"
+        400:
+          description: Invalid Telegram chat ID
+          schema:
+            type: object
+            title: Error
+            properties:
+              errcode:
+                type: string
+                title: Error code
+                description: A machine-readable error code
+                enum:
+                - tgid_invalid
+              error:
+                $ref: "#/definitions/HumanReadableError"
+        404:
+          description: Unknown portal
+          schema:
+            type: object
+            title: Error
+            properties:
+              errcode:
+                type: string
+                title: Error code
+                description: A machine-readable error code
+                enum:
+                - portal_not_found
+              error:
+                $ref: "#/definitions/HumanReadableError"
+      parameters:
+        - name: chat_id
+          in: path
+          description: The Matrix ID of the room whose bridging status to get
+          required: true
+          type: integer
+          pattern: "-[0-9]+"
+        - name: user_id
+          in: query
+          description: Optional Matrix user ID to check if the user has permissions to do bridging.
+          required: false
+          type: string
+  /portal/{room_id}/connect/{chat_id}:
+    post:
+      operationId: connect_portal
+      summary: Connect an existing Telegram chat to the given room
+      tags: [Bridging]
+      parameters:
+        - name: room_id
+          in: path
+          description: The Matrix ID of the room to which the Telegram chat should be connected
+          required: true
+          type: string
+        - name: chat_id
+          in: path
+          description: The ID of the Telegram chat to connect
+          required: true
+          type: integer
+          pattern: "-[0-9]+"
+        - name: force
+          in: query
+          description: Set to force bridging by unbridging or deleting existing portal rooms.
+          required: false
+          type: string
+          enum:
+          - delete
+          - unbridge
+        - name: user_id
+          in: query
+          description: Optional Matrix user ID to check if the user has permissions to do the bridging.
+          required: false
+          type: string
+      responses:
+        200:
+          description: Telegram chat was already bridged to given room.
+        202:
+          description: Room bridging initiated
+        400:
+          $ref: "#/responses/BadRequest"
+        403:
+          description: "Given user doesn't have permission to bridge the room, or the bridge bot is not in the room"
+          schema:
+            type: object
+            title: Error
+            properties:
+              errcode:
+                type: string
+                title: Error code
+                description: A machine-readable error code
+                enum:
+                - not_enough_permissions
+                - bot_not_in_room
+                - bot_not_in_chat
+                - not_logged_in
+              error:
+                $ref: "#/definitions/HumanReadableError"
+        409:
+          description: Matrix room or Telegram chat is already bridged to another chat/room
+          schema:
+            type: object
+            title: Error
+            properties:
+              errcode:
+                type: string
+                title: Error code
+                description: A machine-readable error code
+                example: <room|chat>_already_bridged
+                enum:
+                - room_already_bridged
+                - chat_already_bridged
+              error:
+                $ref: "#/definitions/HumanReadableError"
+  /portal/{room_id}/create:
+    post:
+      operationId: create_portal
+      summary: Create a new Telegram chat for the given room
+      tags: [Bridging]
+      responses:
+        201:
+          description: Telegram chat created
+          schema:
+            type: object
+            properties:
+              chat_id:
+                type: integer
+        400:
+          $ref: "#/responses/BadRequest"
+        403:
+          description: "Given user isn't logged in with a real account or doesn't have permission to bridge the room, or the bridge bot is not in the room"
+          schema:
+            type: object
+            title: Error
+            properties:
+              errcode:
+                type: string
+                title: Error code
+                description: A machine-readable error code
+                enum:
+                - not_logged_in_real_account
+                - not_enough_permissions
+                - bot_not_in_room
+              error:
+                $ref: "#/definitions/HumanReadableError"
+        409:
+          description: Room is already bridged
+          schema:
+            type: object
+            title: Error
+            properties:
+              errcode:
+                type: string
+                title: Error code
+                description: A machine-readable error code
+                enum:
+                - room_already_bridged
+              error:
+                $ref: "#/definitions/HumanReadableError"
+      parameters:
+        - name: room_id
+          in: path
+          description: The Matrix ID of the room whose bridging status to get
+          required: true
+          type: string
+        - name: body
+          in: body
+          required: true
+          schema:
+            type: object
+            required: [type]
+            properties:
+              type:
+                description: The type of chat to create
+                type: string
+                example: supergroup
+                enum:
+                - chat
+                - supergroup
+                - channel
+              title:
+                description: Title for the new chat
+                type: string
+                example: Mautrix-Telegram Bridge
+              about:
+                description: About text for the new chat
+                type: string
+                example: Discussion about mautrix-telegram
+        - name: user_id
+          in: query
+          description: Matrix user to create the chat as.
+          required: true
+          type: string
+  /portal/{room_id}/disconnect:
+    post:
+      operationId: disconnect_portal
+      summary: Disconnect the Telegram chat from the room
+      tags: [Bridging]
+      responses:
+        202:
+          description: Room unbridging initiated
+        400:
+          $ref: "#/responses/BadRequest"
+        403:
+          $ref: "#/responses/PermissionError"
+        404:
+          description: Unknown portal
+          schema:
+            type: object
+            title: Error
+            properties:
+              errcode:
+                type: string
+                title: Error code
+                description: A machine-readable error code
+                enum:
+                - portal_not_found
+              error:
+                $ref: "#/definitions/HumanReadableError"
+      parameters:
+        - name: room_id
+          in: path
+          description: The Matrix ID of the room whose bridging status to get
+          required: true
+          type: string
+        - name: user_id
+          in: query
+          description: Optional Matrix user ID to check if the user has permissions to do the bridging.
+          required: false
+          type: string
+        - name: delete
+          in: query
+          description: Whether or not to delete the room completely (kick all users instead of just Telegram puppets)
+          required: false
+          type: boolean
+          default: false
+        - name: sync
+          in: query
+          description: Whether or not to wait for the unbridging to be completed before responding. **Could cause timeouts in large rooms**
+          required: false
+          type: boolean
+          default: false
+
+  /user/{user_id}:
+    get:
+      operationId: get_me
+      summary: Get the info of the Telegram user the given Matrix user is logged in as
+      tags: [User info]
+      responses:
+        200:
+          description: User found
+          schema:
+            $ref: "#/definitions/UserInfo"
+        400:
+          $ref: "#/responses/BadRequest"
+        403:
+          $ref: "#/responses/NotWhitelistedError"
+        500:
+          $ref: "#/responses/UnknownError"
+      parameters:
+        - name: user_id
+          in: path
+          description: The Matrix ID of the user who to log in as
+          required: true
+          type: string
+  /user/{user_id}/chats:
+    get:
+      operationId: get_chats
+      summary: Get the list of Telegram chats the given Matrix user has access to
+      tags: [User info]
+      responses:
+        200:
+          description: User is logged in
+          schema:
+            $ref: "#/definitions/UserChats"
+        400:
+          $ref: "#/responses/BadRequest"
+        403:
+          description: User is not logged in or not whitelisted
+          schema:
+            type: object
+            title: Error
+            properties:
+              errcode:
+                type: string
+                title: Error code
+                description: A machine-readable error code
+                enum:
+                - not_logged_in
+                - mxid_not_whitelisted
+              error:
+                $ref: "#/definitions/HumanReadableError"
+        500:
+          $ref: "#/responses/UnknownError"
+      parameters:
+        - name: user_id
+          in: path
+          description: The Matrix ID of the user who to log in as
+          required: true
+          type: string
+
+  /user/{user_id}/login/bot_token:
+    post:
+      operationId: post_bot_token
+      summary: Log in with a bot token
+      tags: [Authentication]
+      responses:
+        200:
+          description: Login successful
+          schema:
+            $ref: "#/definitions/AuthSuccess"
+        400:
+          $ref: "#/responses/BadRequest"
+        401:
+          description: Invalid or expired bot token or invalid shared secret
+          schema:
+            type: object
+            title: Error
+            properties:
+              errcode:
+                type: string
+                title: Error code
+                description: A machine-readable error code
+                example: bot_token_<error>
+                enum:
+                - bot_token_invalid
+                - bot_token_expired
+                - shared_secret_invalid
+              error:
+                $ref: "#/definitions/HumanReadableError"
+        403:
+          $ref: "#/responses/NotWhitelistedError"
+        409:
+          $ref: "#/responses/AlreadyLoggedInError"
+        500:
+          $ref: "#/responses/UnknownError"
+      parameters:
+        - name: user_id
+          in: path
+          description: The Matrix ID of the user who to log in as
+          required: true
+          type: string
+        - name: body
+          in: body
+          required: true
+          schema:
+            type: object
+            properties:
+              token:
+                type: string
+                description: The access token of the bot to log in as
+                example: "297900271:IXjeGEcAN61zHnjPgkWnYWyvVp9K4ulHBEv"
+  /user/{user_id}/login/request_code:
+    post:
+      operationId: post_login_phone
+      summary: Request a phone code from Telegram
+      tags: [Authentication]
+      responses:
+        200:
+          description: Code requested successfully
+          schema:
+            $ref: "#/definitions/AuthSuccess"
+        400:
+          description: Invalid phone number or JSON
+          schema:
+            type: object
+            title: Error
+            properties:
+              errcode:
+                type: string
+                title: Error code
+                description: A machine-readable error code
+                example: machine_readable_error
+                enum:
+                - phone_number_invalid
+                - json_invalid
+              error:
+                $ref: "#/definitions/HumanReadableError"
+        401:
+          description: Invalid shared secret
+          schema:
+            type: object
+            title: Error
+            properties:
+              errcode:
+                type: string
+                title: Error code
+                description: A machine-readable error code
+                enum:
+                - shared_secret_invalid
+              error:
+                $ref: "#/definitions/HumanReadableError"
+        403:
+          description: Matrix ID is not whitelisted or phone number is banned or has forbidden 3rd party apps
+          schema:
+            type: object
+            title: Error
+            properties:
+              errcode:
+                type: string
+                title: Error code
+                description: A machine-readable error code
+                example: machine_readable_error
+                enum:
+                - mxid_not_whitelisted
+                - phone_number_banned
+                - phone_number_app_signup_forbidden
+              error:
+                $ref: "#/definitions/HumanReadableError"
+        404:
+          description: Unregistered phone number
+          schema:
+            type: object
+            title: Error
+            properties:
+              errcode:
+                type: string
+                title: Error code
+                description: A machine-readable error code
+                enum:
+                - phone_number_unoccupied
+              error:
+                $ref: "#/definitions/HumanReadableError"
+        409:
+          $ref: "#/responses/AlreadyLoggedInError"
+        429:
+          description: Phone number has been temporarily blocked for flooding
+          schema:
+            type: object
+            title: Error
+            properties:
+              errcode:
+                type: string
+                title: Error code
+                description: A machine-readable error code
+                enum:
+                - flood_wait
+                - phone_number_flood
+              error:
+                $ref: "#/definitions/HumanReadableError"
+        500:
+          $ref: "#/responses/UnknownError"
+      parameters:
+        - name: user_id
+          in: path
+          description: The Matrix ID of the user who to log in as
+          required: true
+          type: string
+        - name: body
+          in: body
+          required: true
+          schema:
+            type: object
+            properties:
+              phone:
+                type: string
+                description: The phone number to log in as.
+                example: "+123456789"
+  /user/{user_id}/login/send_code:
+    post:
+      operationId: post_login_code
+      summary: Send the login code
+      tags: [Authentication]
+      responses:
+        200:
+          description: Login successful
+          schema:
+            $ref: "#/definitions/AuthSuccess"
+        202:
+          description: Correct code, but two-factor authentication is enabled
+          schema:
+            $ref: "#/definitions/AuthSuccess"
+        400:
+          $ref: "#/responses/BadRequest"
+        401:
+          description: Invalid phone code or shared secret
+          schema:
+            type: object
+            title: Error
+            properties:
+              errcode:
+                type: string
+                title: Error code
+                description: A machine-readable error code
+                enum:
+                - phone_code_invalid
+                - shared_secret_invalid
+              error:
+                $ref: "#/definitions/HumanReadableError"
+        403:
+          description: Matrix ID not whitelisted or phone code expired
+          schema:
+            type: object
+            title: Error
+            properties:
+              errcode:
+                type: string
+                title: Error code
+                description: A machine-readable error code
+                example: machine_readable_error
+                enum:
+                - mxid_not_whitelisted
+                - phone_code_expired
+              error:
+                $ref: "#/definitions/HumanReadableError"
+        409:
+          $ref: "#/responses/AlreadyLoggedInError"
+        500:
+          $ref: "#/responses/UnknownError"
+      parameters:
+        - name: user_id
+          in: path
+          description: The Matrix ID of the user who to log in as
+          required: true
+          type: string
+        - name: body
+          in: body
+          required: true
+          schema:
+            type: object
+            properties:
+              code:
+                type: integer
+                description: The phone code from Telegram.
+                format: int32
+                example: 123456
+  /user/{user_id}/login/send_password:
+    post:
+      operationId: post_login_password
+      summary: Send the two-factor auth password
+      tags: [Authentication]
+      responses:
+        200:
+          description: Login successful
+          schema:
+            $ref: "#/definitions/AuthSuccess"
+        400:
+          description: Missing password or invalid JSON
+          schema:
+            type: object
+            title: Error
+            properties:
+              errcode:
+                type: string
+                title: Error code
+                description: A machine-readable error code
+                example: <field>_empty
+                enum:
+                - password_empty
+                - json_invalid
+              error:
+                $ref: "#/definitions/HumanReadableError"
+        401:
+          description: Incorrect password or invalid shared secret
+          schema:
+            type: object
+            title: Error
+            properties:
+              errcode:
+                type: string
+                title: Error code
+                description: A machine-readable error code
+                enum:
+                - password_invalid
+                - shared_secret_invalid
+              error:
+                $ref: "#/definitions/HumanReadableError"
+        403:
+          $ref: "#/responses/NotWhitelistedError"
+        409:
+          $ref: "#/responses/AlreadyLoggedInError"
+        500:
+          $ref: "#/responses/UnknownError"
+      parameters:
+        - name: user_id
+          in: path
+          description: The Matrix ID of the user who to log in as
+          required: true
+          type: string
+        - name: body
+          in: body
+          required: true
+          schema:
+            type: object
+            properties:
+              password:
+                type: string
+                description: The two-factor auth password
+                format: password
+                example: hunter2
+  /user/{user_id}/logout:
+    post:
+      operationId: logout
+      summary: Log out
+      tags: [Authentication]
+      responses:
+        200:
+          description: Logout successful
+        403:
+          description: User was not logged in
+          schema:
+            type: object
+            title: Error
+            properties:
+              errcode:
+                type: string
+                title: Error code
+                description: A machine-readable error code
+                enum:
+                - not_logged_in
+              error:
+                $ref: "#/definitions/HumanReadableError"
+        500:
+          $ref: "#/responses/UnknownError"
+      parameters:
+        - name: user_id
+          in: path
+          description: The Matrix ID of the user who to log out as
+          required: true
+          type: string
+
+responses:
+  NotWhitelistedError:
+    description: Matrix ID not whitelisted for puppeting
+    schema:
+      type: object
+      title: Error
+      properties:
+        errcode:
+          type: string
+          title: Error code
+          description: A machine-readable error code
+          enum:
+          - mxid_not_whitelisted
+        error:
+          $ref: "#/definitions/HumanReadableError"
+  AlreadyLoggedInError:
+    description: The Matrix user is already logged in
+    schema:
+      type: object
+      properties:
+        state:
+          type: string
+          enum:
+          - logged-in
+        username:
+          type: string
+          description: The Telegram username the user is logged in as.
+        phone:
+          type: string
+          description: The phone number of the account the user is logged into.
+  BadRequest:
+    description: Invalid JSON.
+    schema:
+      type: object
+      title: Error
+      properties:
+        errcode:
+          type: string
+          title: Error code
+          description: A machine-readable error code
+          enum:
+          - json_invalid
+          - mxid_empty
+          - body_value_missing
+          - body_value_invalid
+        error:
+          $ref: "#/definitions/HumanReadableError"
+  UnknownError:
+    description: Unknown error
+    schema:
+      type: object
+      title: UnknownError
+      properties:
+        errcode:
+          type: string
+          title: Error code
+          description: A machine-readable error code
+          enum:
+          - unknown_error
+          - unhandled_error
+        error:
+          type: string
+          title: Error
+          description: A human-readable description of the error
+          example: Internal server error while <action>.
+  PermissionError:
+    description: The given Matrix user doesn't have the permissions to do that.
+    schema:
+      type: object
+      title: Error
+      properties:
+        errcode:
+          type: string
+          title: Error code
+          description: A machine-readable error code
+          example: not_enough_permissions
+          enum:
+          - not_enough_permissions
+        error:
+          $ref: "#/definitions/HumanReadableError"
+
+definitions:
+  UserInfo:
+    type: object
+    properties:
+      mxid:
+        type: string
+        example: "@usern:example.com"
+      permissions:
+        type: string
+        example: user
+        enum:
+        - none
+        - relaybot
+        - user
+        - full
+        - admin
+      telegram:
+        type: object
+        properties:
+          id:
+            type: integer
+            example: 123456789
+          username:
+            type: string
+            example: username
+          first_name:
+            type: string
+            example: Usern
+          last_name:
+            type: string
+            example: A.
+          phone:
+            type: string
+            example: 123456789
+          is_bot:
+            type: boolean
+            example: false
+  UserChats:
+    type: array
+    items:
+      type: object
+      properties:
+        id:
+          type: integer
+          example: -123456789
+          description: A bot API style chat ID.
+        title:
+          type: string
+
+  PortalInfo:
+    type: object
+    properties:
+      mxid:
+        type: string
+        example: "!foo:example.com"
+      chat_id:
+        type: integer
+        example: -100123456789
+      peer_type:
+        type: string
+        enum:
+        - user
+        - chat
+        - channel
+      megagroup:
+        type: boolean
+      username:
+        type: string
+      title:
+        type: string
+      about:
+        type: string
+      can_unbridge:
+        type: boolean
+        description: If a user ID was provided with the request, this will indicate whether or not the user can unbridge the room.
+
+  AuthSuccess:
+    type: object
+    properties:
+      state:
+        type: string
+        description: The state/next step after the successful operation.
+        enum:
+        - code
+        - request
+        - password
+        - token
+        - logged-in
+      username:
+        type: string
+        description: The Telegram username the user is logged in as. Only applicable if state=logged-in
+      phone:
+        type: string
+        description: The phone number of the account the user logged into. Only applicable if state=logged-in
+
+  HumanReadableError:
+    type: string
+    description: A human-readable description of the error
+    example: A human-readable description of the error
+
+security:
+  - Bearer: []
+securityDefinitions:
+  Bearer:
+    description: Required authentication for all endpoints
+    name: Authorization
+    in: header
+    type: apiKey

mautrix_telegram/web/public/__init__.py

@@ -0,0 +1,187 @@
+# -*- coding: future_fstrings -*-
+# mautrix-telegram - A Matrix-Telegram puppeting bridge
+# Copyright (C) 2019 Tulir Asokan
+#
+# This program is free software: you can redistribute it and/or modify
+# it under the terms of the GNU Affero General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU Affero General Public License for more details.
+#
+# You should have received a copy of the GNU Affero General Public License
+# along with this program.  If not, see <https://www.gnu.org/licenses/>.
+from typing import Optional
+from aiohttp import web
+from mako.template import Template
+import pkg_resources
+import asyncio
+import logging
+import random
+import string
+import time
+
+from ...types import MatrixUserID
+from ...util import sign_token, verify_token
+from ...user import User
+from ...puppet import Puppet
+from ..common import AuthAPI
+
+
+class PublicBridgeWebsite(AuthAPI):
+    log = logging.getLogger("mau.web.public")  # type: logging.Logger
+
+    def __init__(self, loop: asyncio.AbstractEventLoop):
+        super().__init__(loop)
+        self.secret_key = "".join(
+            random.choice(string.ascii_lowercase + string.digits) for _ in range(64))  # type: str
+
+        self.login = Template(pkg_resources.resource_string(
+            "mautrix_telegram", "web/public/login.html.mako"))  # type: Template
+
+        self.mx_login = Template(pkg_resources.resource_string(
+            "mautrix_telegram", "web/public/matrix-login.html.mako"))  # type: Template
+
+        self.app = web.Application(loop=loop)  # type: web.Application
+        self.app.router.add_route("GET", "/login", self.get_login)
+        self.app.router.add_route("POST", "/login", self.post_login)
+        self.app.router.add_route("GET", "/matrix-login", self.get_matrix_login)
+        self.app.router.add_route("POST", "/matrix-login", self.post_matrix_login)
+        self.app.router.add_static("/", pkg_resources.resource_filename("mautrix_telegram",
+                                                                        "web/public/"))
+
+    def make_token(self, mxid: str, endpoint: str = "/login", expires_in: int = 900) -> str:
+        return sign_token(self.secret_key, {
+            "mxid": mxid,
+            "endpoint": endpoint,
+            "expiry": int(time.time()) + expires_in,
+        })
+
+    def verify_token(self, token: str, endpoint: str = "/login") -> Optional[MatrixUserID]:
+        token = verify_token(self.secret_key, token)
+        if token and (token.get("expiry", 0) > int(time.time()) and
+                      token.get("endpoint", None) == endpoint):
+            return MatrixUserID(token.get("mxid", None))
+        return None
+
+    async def get_login(self, request: web.Request) -> web.Response:
+        state = "bot_token" if request.rel_url.query.get("mode", "") == "bot" else "request"
+
+        mxid = self.verify_token(request.rel_url.query.get("token", None), endpoint="/login")
+        if not mxid:
+            return self.get_login_response(status=401, state="invalid-token")
+        user = User.get_by_mxid(mxid, create=False) if mxid else None
+
+        if not user:
+            return self.get_login_response(mxid=mxid, state=state)
+        elif not user.puppet_whitelisted:
+            return self.get_login_response(mxid=user.mxid, error="You are not whitelisted.",
+                                           status=403)
+        await user.ensure_started()
+        if not await user.is_logged_in():
+            return self.get_login_response(mxid=user.mxid, state=state)
+
+        return self.get_login_response(mxid=user.mxid, human_tg_id=user.human_tg_id)
+
+    async def get_matrix_login(self, request: web.Request) -> web.Response:
+        mxid = self.verify_token(request.rel_url.query.get("token", None),
+                                 endpoint="/matrix-login")
+        if not mxid:
+            return self.get_mx_login_response(status=401, state="invalid-token")
+        user = User.get_by_mxid(mxid, create=False) if mxid else None
+
+        if not user:
+            return self.get_mx_login_response(mxid=mxid)
+        elif not user.puppet_whitelisted:
+            return self.get_mx_login_response(mxid=user.mxid, error="You are not whitelisted.",
+                                              status=403)
+        await user.ensure_started()
+        if not await user.is_logged_in():
+            return self.get_mx_login_response(mxid=user.mxid, status=403,
+                                              error="You are not logged in to Telegram.")
+
+        puppet = Puppet.get(user.tgid)
+        if puppet.is_real_user:
+            return self.get_mx_login_response(state="already-logged-in", status=409)
+
+        return self.get_mx_login_response(mxid=user.mxid)
+
+    def get_login_response(self, status: int = 200, state: str = "", username: str = "",
+                           phone: str = "", human_tg_id: str = "", mxid: str = "",
+                           message: str = "", error: str = "", errcode: str = "") -> web.Response:
+        return web.Response(status=status, content_type="text/html",
+                            text=self.login.render(human_tg_id=human_tg_id, state=state,
+                                                   error=error, message=message, mxid=mxid))
+
+    def get_mx_login_response(self, status: int = 200, state: str = "", username: str = "",
+                              phone: str = "", human_tg_id: str = "", mxid: str = "",
+                              message: str = "", error: str = "", errcode: str = ""
+                              ) -> web.Response:
+        return web.Response(status=status, content_type="text/html",
+                            text=self.mx_login.render(human_tg_id=human_tg_id, state=state,
+                                                      error=error, message=message, mxid=mxid))
+
+    async def post_matrix_login(self, request: web.Request) -> web.Response:
+        mxid = self.verify_token(request.rel_url.query.get("token", None),
+                                 endpoint="/matrix-login")
+        if not mxid:
+            return self.get_mx_login_response(status=401, state="invalid-token")
+
+        data = await request.post()
+
+        user = await User.get_by_mxid(mxid).ensure_started()
+        if not user.puppet_whitelisted:
+            return self.get_mx_login_response(mxid=user.mxid, error="You are not whitelisted.",
+                                              status=403)
+        elif not await user.is_logged_in():
+            return self.get_mx_login_response(mxid=user.mxid, status=403,
+                                              error="You are not logged in to Telegram.")
+        mode = data.get("mode", "access_token")
+        if mode == "password":
+            return await self.post_matrix_password(user, data["value"])
+        elif mode == "access_token":
+            return await self.post_matrix_token(user, data["value"])
+        return self.get_mx_login_response(mxid=user.mxid, status=400,
+                                          error="You must provide an access token or "
+                                                "password.")
+
+    async def post_login(self, request: web.Request) -> web.Response:
+        mxid = self.verify_token(request.rel_url.query.get("token", None), endpoint="/login")
+        if not mxid:
+            return self.get_login_response(status=401, state="invalid-token")
+
+        data = await request.post()
+
+        user = await User.get_by_mxid(mxid).ensure_started(even_if_no_session=True)
+        if not user.puppet_whitelisted:
+            return self.get_login_response(mxid=user.mxid, error="You are not whitelisted.",
+                                           status=403)
+        elif await user.is_logged_in():
+            return self.get_login_response(mxid=user.mxid, human_tg_id=user.human_tg_id)
+
+        await user.ensure_started(even_if_no_session=True)
+
+        if "phone" in data:
+            return await self.post_login_phone(user, data["phone"])
+        elif "bot_token" in data:
+            return await self.post_login_token(user, data["bot_token"])
+        elif "code" in data:
+            try:
+                code = int(data["code"].strip())
+            except ValueError:
+                return self.get_login_response(mxid=user.mxid, state="code", status=400,
+                                               errcode="phone_code_invalid",
+                                               error="Phone code must be a number.")
+            resp = await self.post_login_code(user, code,
+                                              password_in_data="password" in data)
+            if resp or "password" not in data:
+                return resp
+        elif "password" not in data:
+            return self.get_login_response(error="No data given.", status=400)
+
+        if "password" in data:
+            return await self.post_login_password(user, data["password"])
+        return self.get_login_response(error="This should never happen.", status=500)